# robots.txt for suffolkgolfunion.com
# Generated by nooQ Golf CMS

# Standard search engine crawlers
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /api/
Disallow: /_next/
Disallow: /preview/
# Second URL for every page — /t/{tenant}/{slug} renders the same content as
# /{slug}. Crawling it splits ranking signals across two URLs per page.
Disallow: /t/
# Preview/admin query variant of every page.
Disallow: /*?tenant=
# Member-only and transactional pages: nothing to rank, and thin or gated.
Disallow: /downloads/login
Disallow: /downloads/reset-password
Disallow: /support/tickets
Disallow: /forms/
# Internal site search results — thin, duplicative, and a classic crawl trap.
# Paired with this, the WebSite JSON-LD deliberately emits no SearchAction:
# advertising a search endpoint that crawlers are told not to fetch is
# self-contradictory. Change both together or neither.
Disallow: /search

# AI/LLM crawlers - welcome to index our content
User-agent: GPTBot
Allow: /
Disallow: /admin

User-agent: ChatGPT-User
Allow: /
Disallow: /admin

User-agent: Claude-Web
Allow: /
Disallow: /admin

User-agent: Anthropic-AI
Allow: /
Disallow: /admin

User-agent: Google-Extended
Allow: /
Disallow: /admin

User-agent: PerplexityBot
Allow: /
Disallow: /admin

User-agent: Cohere-AI
Allow: /
Disallow: /admin

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin

User-agent: ClaudeBot
Allow: /
Disallow: /admin

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin

User-agent: Bingbot
Allow: /
Disallow: /admin

# LLM guidance: /llms.txt and /llms-full.txt
# LLM content files (https://llmstxt.org/)
# These files provide structured content for LLMs:
# - /llms.txt - Summary of site structure and links
# - /llms-full.txt - Full page content for indexing
# - /agents.md - How AI agents should interact with this site
LLM-Content: https://suffolkgolfunion.com/llms.txt
LLM-Full-Content: https://suffolkgolfunion.com/llms-full.txt

# Sitemaps
Sitemap: https://suffolkgolfunion.com/sitemap.xml

# Host
Host: suffolkgolfunion.com