# robots.txt for URLcut.ai

# AI search crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, PerplexityBot, ClaudeBot,
# Google-Extended, Amazonbot) are welcome, and are governed by the rules below.
#
# They deliberately have no group of their own. A crawler matches exactly ONE group in this
# file and groups do not inherit, so a bare "User-agent: GPTBot / Allow: /" block exempted
# those crawlers from every Disallow here (dashboard, admin, /go/, /verify/, /api/).
# Keeping one set of rules in the wildcard group below is what makes those Disallows apply
# to them as well.

# Rules for all crawlers
User-agent: *

# Allow all public content by default
Allow: /

# LLM optimization files
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /changelog.txt

# Disallow authenticated/private pages under /shortener/
Disallow: /shortener/dashboard
Disallow: /shortener/links
Disallow: /shortener/analytics
Disallow: /shortener/settings
Disallow: /shortener/billing
Disallow: /shortener/login
Disallow: /shortener/register
Disallow: /shortener/logout
Disallow: /shortener/forgot-password
Disallow: /shortener/reset-password
Disallow: /shortener/admin
Disallow: /shortener/utm-builder

# Disallow internal testing/utility pages
Disallow: /shortener/email-test.html
Disallow: /shortener/counter
Disallow: /shortener/weather
Disallow: /shortener/Error
Disallow: /shortener/404

# Disallow interstitial and password verification pages (short link ecosystem)
Disallow: /go/
Disallow: /go-quick/
Disallow: /verify/

# Disallow API endpoints
Disallow: /api/

# Sitemap location (auto-generated with correct URLs)
Sitemap: https://urlcut.ai/sitemap.xml