# https://www.shoutt.ai/robots.txt
# See https://www.robotstxt.org/robotstxt.html
#
# NOTE: no blank lines inside a group. A blank line terminates the record in the
# original spec, which would orphan every Disallow below it. Comments are safe.
#
# NOTE: robots.txt has no inheritance. A crawler obeys only the single most
# specific group matching its user agent. If a "User-agent: GPTBot" group is
# ever added below, it must repeat every Disallow in this group or it will have
# been granted access to the product surface.
User-agent: *
# Product surface - job listings and the application flow are part of the
# product, not crawlable content.
Disallow: /jobs
Disallow: /gigs
Disallow: /opportunities
Disallow: /leads
Disallow: /apply/
Disallow: /applications
Disallow: /preview-form/
Disallow: /chrome-required
Disallow: /testextension
Disallow: /newjob
Disallow: /newbrief
Disallow: /info
# Accounts, auth and admin. /login and /signup are deliberately NOT disallowed -
# people search for "shoutt login" and the signup page is marketing copy, so both
# should be findable. /signup/verification stays blocked: prefix matching means
# dropping "Disallow: /signup" would otherwise have un-blocked it too, and it is
# an email-link landing, not a page worth indexing.
Disallow: /signup/verification
Disallow: /beta-signup
Disallow: /closed-beta
Disallow: /google-callback
Disallow: /password/
Disallow: /users/
Disallow: /admin/
Disallow: /rewards
Disallow: /quiet-notifications
# One-off links from emails - nothing worth indexing
Disallow: /unsubscribe
Disallow: /newsletter
Disallow: /sourcetracking
# Everything not matched above stays crawlable - robots.txt is allow-by-default,
# so the marketing surface is open because nothing here matches it. Deliberately
# left crawlable: /, /about, /privacy, /terms, /contactus, /login, /signup,
# /blog/, /sitemap.xml and /llms.txt. The Allow below states the same thing
# explicitly.
Allow: /
# AI usage preferences (IETF AIPREF draft). Not enforceable, documents intent:
# don't train on the product, do train on the content.
Content-Usage: train-ai=n
Content-Usage: /blog/ train-ai=y

Sitemap: https://www.shoutt.ai/sitemap.xml