# robots.txt for CLSTR News Aggregator
# Content Signals (https://contentsignals.org): search and AI answering with
# attribution are welcome; using this content to train models is not.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
Disallow: /cronjob/
Disallow: /signin
Disallow: /settings
Disallow: /automations
Disallow: /welcome
Disallow: /briefing/
Disallow: /alerts/
Disallow: /account-error
Disallow: /auth/
Disallow: /debug/
Disallow: /stripe/
Disallow: /internal/
Disallow: /unsubscribe/
Disallow: /billing
Disallow: /profile
Disallow: /user/

# SEO-backlink crawlers: no reader ever reaches CLSTR through one, and they
# walk the whole corpus. See config/seo.js robots.blockedCrawlers.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Sitemaps
Sitemap: https://clstr.news/sitemap.xml