# Website Sitemap
Sitemap: https://pastorieroermond.nl/sitemap.xml
Sitemap: https://pastorieroermond.nl/de/sitemap.xml
Sitemap: https://pastorieroermond.nl/en/sitemap.xml
Sitemap: https://pastorieroermond.nl/fr/sitemap.xml


# Crawlers Setup
User-agent: *

Allow: /
Crawl-delay: 25

# ----------------------------------------------------
# Block bad crawlers
# ----------------------------------------------------
# third party search engine bot
User-agent: MJ12bot
Disallow: /

# Web-crawler by the company 'WiseGuys' to gather unstructured webdata
User-agent: Vagabondo
Disallow: /

# Chinese Search Engine
User-agent: BaiDuSpider
Disallow: /

# French Search Engine
User-agent: Exabot
Disallow: /

# Russian Search Engine
User-agent: Yandex
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: CFNetwork
Disallow: /

User-agent: TwengaBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: FatBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: seoscanners.net
Disallow: /

User-agent: SpeedySpider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

# Bots
User-agent: LeGuideImgServer
Disallow: /

User-Agent: Baiduspider
Disallow: /

User-Agent: CazoodleBot
Disallow: /

User-Agent: Fasterfox
Disallow: /

User-Agent: Ingrid
Crawl-delay: 20

User-Agent: Jyxobot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: ShopWiki
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: CatchBot
Disallow: /

User-agent: ccbot
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: Zeus 32297 Webster Pro V2.9 Win32
Disallow: /

User-agent: AMZNKAssocBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: wbot
Disallow: /

User-agent: JobdiggerSpider
Disallow: /

User-agent: sistrix crawler
Disallow: /

User-agent: 008
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: *
Crawl-delay: 10

User-Agent: msnbot
Crawl-Delay: 20

User-Agent: Slurp
Crawl-delay: 20

User-Agent: Yahoo! Slurp
Crawl-delay: 20

User-agent: rogerbot
Crawl-delay: 5

User-agent: Screaming Frog SEO Spider
Crawl-delay: 5