Sitemap: https://www.thestandard.com.hk/sitemap.xml
Sitemap: https://www.thestandard.com.hk/horse-racing-sitemap.xml

User-agent: *
Allow: /ads.txt
Disallow: /api/
Disallow: /search
Disallow: /search?
Disallow: /_next/
Crawl-delay: 5

# Major search engines and social media bots
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: Slurp
User-agent: Twitterbot
User-agent: Facebookexternalhit
User-agent: Applebot
User-agent: baiduspider
Allow: /
Crawl-delay: 1