# www.ironscout.ai robots.txt

User-agent: *
# Search lives on app.ironscout.ai, not www
Disallow: /search
# Cloudflare internal paths
Disallow: /cdn-cgi/
# Raw chart-data JSON consumed client-side by www pages; not indexable content
# (GSC was crawling these into "Crawled - currently not indexed").
# /market-snapshots/monthly/ stays crawlable: it is the declared DataDownload
# contentUrl in the monthly-archive Dataset JSON-LD.
Disallow: /market-snapshots/30d/
Disallow: /market-snapshots/today.json
# Next.js RSC text payloads — internal client-navigation transport, not content.
# Googlebot was fetching them and 404ing: 10 in the 2026-08-03 crawl export, e.g.
# /ids/schemas/product_detail_v1.schema.json/__next._tree.txt?_rsc=7ykokwRHQrbiBx4J
# Same crawl-budget rationale as /market-snapshots/30d/ above.
# NOTE: `__next.` (two underscores, then a dot) is the payload naming and does NOT
# match the build assets under /_next/ (one underscore). Those must stay crawlable
# or Google cannot render the pages.
Disallow: /*__next.
Disallow: /*_rsc=
# AI content usage preferences (Content Signals — contentsignals.org,
# draft-romm-aipref-contentsignals). Group-member record applying to
# User-agent: * above. Search indexing allowed; AI systems may reference
# content at query time (that is where assistant referrals come from);
# model training is not permitted. Owner decision 2026-10-14, experiment E8
# in context/operations/acquisition-experiments-2026-10-14.md. The signal is
# advisory; enforcement for the training crawlers is the Cloudflare AI Crawl
# Control rule listed there.
Content-Signal: ai-train=no, search=yes, ai-input=yes

Sitemap: https://www.ironscout.ai/sitemap.xml
Sitemap: https://www.ironscout.ai/news-sitemap.xml