# AI Crawler Permissions - REQUIRED FOR AI VISIBILITY
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Extended
Allow: /

User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: Bytespider
Allow: /
Crawl-delay: 2

# Bing crawler - PRIORITY ACCESS
User-agent: bingbot
Allow: /
Crawl-delay: 0

User-agent: msnbot
Allow: /
Crawl-delay: 0

User-agent: BingPreview
Allow: /

# Traditional crawlers
User-agent: *
Allow: /

# PRIORITY: Allow high-value out-of-state location pages (before state-level blocks)
Allow: /web-design-maumee-ohio/
Allow: /web-design-burleson-texas/
Allow: /web-design-dacula-georgia/
Allow: /web-design-winder-georgia/
Allow: /web-design-jarrettsville-maryland/
Allow: /web-design-sebastian-florida/
Allow: /web-design-lecanto-florida/

# Block download pages and secure areas
Disallow: /shop/download.php
Disallow: /shop/api/
Disallow: /uploads/shop-products/*.pdf
Disallow: /uploads/shop-products/*.zip
Disallow: /uploads/shop-products/*.doc*

# Block admin and development areas
Disallow: /admin/
Disallow: /beta/
Disallow: /.private/
Disallow: /appwt-private/
Disallow: /tools/*.php
Disallow: /tools/*/*.php

# Block outdated content and redirects
Disallow: /tag/
Disallow: /feed
Disallow: /feed/
Disallow: /*/feed/
Disallow: /calc/
Disallow: /?p=
Disallow: /amp
Disallow: /amp/
Disallow: /2023/
Disallow: /2024/
Disallow: /our-clients
Disallow: /what-is-interservernet/
Disallow: /how-do-you-grow-organic-traffic/
Disallow: /programmatic-advertising
Disallow: /usa-payment-solutions
Disallow: /ai-models
Disallow: /elementor-
Disallow: /free-consultation
Disallow: /animations-and-videos
Disallow: /websites/
Disallow: /qr
Disallow: /privacy-policy
Disallow: /terms

# NATIONWIDE EXPANSION: State/Province/Country Landing Pages
# Now allowing all 75 state/province/country pages created Oct 27, 2025
# These are high-quality template pages (not spammy dynamic pages)
# Strategy: Tier 2 semi-custom state pages for nationwide coverage

# Allow all state-level pages (web-design-[state]/)
# Examples: /web-design-california/, /web-design-texas/, /web-design-ontario/
Allow: /web-design-alabama/
Allow: /web-design-alaska/
Allow: /web-design-arizona/
Allow: /web-design-arkansas/
Allow: /web-design-california/
Allow: /web-design-colorado/
Allow: /web-design-connecticut/
Allow: /web-design-delaware/
Allow: /web-design-florida/
Allow: /web-design-georgia/
Allow: /web-design-hawaii/
Allow: /web-design-idaho/
Allow: /web-design-illinois/
Allow: /web-design-indiana/
Allow: /web-design-iowa/
Allow: /web-design-kansas/
Allow: /web-design-kentucky/
Allow: /web-design-louisiana/
Allow: /web-design-maine/
Allow: /web-design-maryland/
Allow: /web-design-massachusetts/
Allow: /web-design-michigan/
Allow: /web-design-minnesota/
Allow: /web-design-mississippi/
Allow: /web-design-missouri/
Allow: /web-design-montana/
Allow: /web-design-nebraska/
Allow: /web-design-nevada/
Allow: /web-design-new-hampshire/
Allow: /web-design-new-jersey/
Allow: /web-design-new-mexico/
Allow: /web-design-new-york/
Allow: /web-design-north-carolina/
Allow: /web-design-north-dakota/
Allow: /web-design-ohio/
Allow: /web-design-oklahoma/
Allow: /web-design-oregon/
Allow: /web-design-pennsylvania/
Allow: /web-design-rhode-island/
Allow: /web-design-south-carolina/
Allow: /web-design-south-dakota/
Allow: /web-design-tennessee/
Allow: /web-design-texas/
Allow: /web-design-utah/
Allow: /web-design-vermont/
Allow: /web-design-virginia/
Allow: /web-design-washington/
Allow: /web-design-west-virginia/
Allow: /web-design-wisconsin/
Allow: /web-design-wyoming/

# Allow Canadian provinces/territories
Allow: /web-design-alberta/
Allow: /web-design-british-columbia/
Allow: /web-design-manitoba/
Allow: /web-design-new-brunswick/
Allow: /web-design-newfoundland-and-labrador/
Allow: /web-design-northwest-territories/
Allow: /web-design-nova-scotia/
Allow: /web-design-nunavut/
Allow: /web-design-ontario/
Allow: /web-design-prince-edward-island/
Allow: /web-design-quebec/
Allow: /web-design-saskatchewan/
Allow: /web-design-yukon/

# Allow UK countries
Allow: /web-design-england/
Allow: /web-design-scotland/
Allow: /web-design-wales/
Allow: /web-design-northern-ireland/

# Allow Australian states/territories
Allow: /web-design-new-south-wales/
Allow: /web-design-victoria/
Allow: /web-design-queensland/
Allow: /web-design-south-australia/
Allow: /web-design-western-australia/
Allow: /web-design-tasmania/
Allow: /web-design-northern-territory/
Allow: /web-design-australian-capital-territory/

# STILL BLOCK: Dynamic service+city+state pages (spammy, thin content)
# These are the 1,500+ auto-generated pages from location-page.php
# Format: /web-design-[service]-[city]-[state]/ or /*-[city]/
# We allow direct access but not crawling to avoid doorway page penalty

Sitemap: https://appwt.com/sitemap.xml
Sitemap: https://appwt.com/sitemap-locations.xml
Sitemap: https://appwt.com/sitemap-keyword-pages.xml
Sitemap: https://appwt.com/sitemap-state-pages.xml
Sitemap: https://appwt.com/blog-sitemap.xml
Sitemap: https://appwt.com/llms.txt