# Allow all web crawlers
User-agent: *
Allow: /

# Make sure search engines can access the sitemap
Sitemap: https://scanlytic.co/sitemap.xml

# No duplicate content indexing
# Prevent indexing of language query parameter versions
Disallow: /*?lang=*

# Allow all languages main paths
Allow: /en/
Allow: /ru/
Allow: /de/
Allow: /fr/
Allow: /es/

# Explicitly allow specific paths that were having issues
Allow: /about/
Allow: /product/
Allow: /blog/
Allow: /contact/
Allow: /qr-generator/

# Allow important static resources
Allow: /img/
Allow: /assets/
Allow: /*.css
Allow: /*.js

# Speed up crawling
Crawl-delay: 1