# My-Jewellery.com
# Edited: 2023-11-08

# Website Sitemaps
# NL
Sitemap: https://www.my-jewellery.com/nl-nl/media/sitemaps/sitemap.xml
# EN
Sitemap: https://www.my-jewellery.com/en/media/sitemaps/sitemap_en.xml
# DE
Sitemap: https://www.my-jewellery.com/de/media/sitemaps/sitemap_de.xml
# FR
Sitemap: https://www.my-jewellery.com/fr/media/sitemaps/sitemap_fr.xml

# Crawlers Setup
User-agent: *

Allow: /media/catalog/product/cache/*.jpg
Allow: /media/catalog/product/cache/*.png

Allow: /static/version*/frontend/*

# Default instructions
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /*/*-test
Disallow: /*/test-*
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*/review/
Disallow: /listAjax/
# Disallow: /*SID=

# Specific (Temp) Restrictions
# Disallow: /*/influencers.html
# Disallow: /*/influencers/*
Disallow: /*/*-zalando.html
Disallow: /*/*zalando*
# Disallow: /en/be/*
# Disallow: /de/be/*
Disallow: /*/my-jewellery-deals.html
Disallow: /nl-nl/duurzaamheid/reshare-inzamelactie

# B2B Restrictions
Disallow: /groothandel/
Disallow: /wholesale/

# Restrict CMS Directories
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pub/
Disallow: /404/
Disallow: /*/app/
Disallow: /*/bin/
Disallow: /*/dev/
Disallow: /*/lib/
Disallow: /*/phpserver/
Disallow: /*/pub/
Disallow: /*/404/

# Restrict User Account & Checkout Pages
Disallow: /checkout/
Disallow: /*/checkout/
Disallow: /onestepcheckout/
Disallow: /*/onestepcheckout/
Disallow: /customer/
Disallow: /*/customer/
Disallow: /customer/account/
Disallow: /*/customer/account/
Disallow: /customer/account/login/
Disallow: /*/customer/account/login/
Disallow: /wishlist/
Disallow: /*/wishlist/

# Disallow URL Filter Searches
# Disallow: /catalogsearch/
# Disallow: /*/catalogsearch/
# Disallow: /catalogsearch/*
# Disallow: /*/catalogsearch/*
# Disallow: /*?dir*
# Disallow: /*?dir=desc
# Disallow: /*?dir=asc
# Disallow: /*?limit=all
# Disallow: /*?limit=50
Disallow: /*/show/all.html
# Disallow: /*?mode*
# Disallow: /*?q=*
# Disallow: /*?cat=*
# Disallow: /*?for_who=*
# Disallow: /*?price=*
# Disallow: /*?collectie=*
# Disallow: /*&q=*
# Disallow: /*zanpid=*
# Disallow: /*xnpe_tifc=*
# Disallow: /*utm_medium=email*
# Disallow: /*featuredShow=*
# Disallow: /*channable= #Breaks G Merchants

# Disallow Pagination
# Now in CF: Noindex, Follow via X-Robots-Tag
# Disallow: /*?p=3
#etc...

# Disallow Filter-combinations
# Site-wide
Allow: /*/flapbox-*-*.html
Allow: /*/*-l-amour*.html

# NL (Now handled through robots HTTP Header)
# EN (Now handled through robots HTTP Header)
# DE (Now handled through robots HTTP Header)
# FR (Now handled through robots HTTP Header)

# Disallow Store Switcher URLs
# Disallow: /*?___from_store=
# Disallow: /*/___from_store/
# Disallow: /*/uenc/
# Disallow: /en/stores/store/redirect/___store/con_*/___from_store/con_*/uenc/*/
# Disallow: /de/stores/store/redirect/___store/con_*/___from_store/con_*/uenc/*/

# Bot Management Exceptions by Yawiss

# User-agent: Googlebot
# User-agent: AdsBot-Google
# User-agent: AdsBot-Google-Mobile
# User-agent: AdsBot-Google-Mobile-Apps
# User-agent: Googlebot-Image

User-agent: Bytespider
Crawl-Delay: 1
User-agent: AhrefsSiteAudit
Crawl-Delay: 2
User-agent: Pinterestbot
Crawl-Delay: 3
User-agent: AhrefsBot
Crawl-Delay: 5
User-agent: UptimeRobot
Crawl-Delay: 12
User-agent: BLEXBot
Crawl-Delay: 20
User-agent: MJ12bot
Crawl-Delay: 25
User-agent: DotBot
Crawl-Delay: 30
User-agent: MegaIndex
Crawl-Delay: 40

User-agent: Seekport Crawler
Disallow: /
User-agent: AspiegelBot
Disallow: /
User-agent: PetalBot
Disallow: /