# robots.txt for HTML Free Codes
# Updated: 2025-10-10 - SEO optimization

# ===================================
# GENERAL CRAWLERS
# ===================================
User-agent: *

# Block query parameters (prevent duplicate content indexing)
Disallow: /*?header=*
Disallow: /*?*header=*

# Block old WordPress patterns (these redirect via .htaccess, but prevent crawling)
Disallow: /blog/
Disallow: /category/
Disallow: /tag/
Disallow: /author/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /type/
Disallow: /features/
Disallow: /features-2/
Disallow: /home-*/
Disallow: /explore-our-categories/
Disallow: /apps-integrations/
Disallow: /classic/
Disallow: /nosidebar/
Disallow: /text-only/
Disallow: /vertical/
Disallow: /fullwidth/

# Block old content paths
Disallow: /inspiration/
Disallow: /travel/
Disallow: /health/
Disallow: /lifestyle/
Disallow: /uncategorized/
Disallow: /code/

# Block pagination with query params
Disallow: /*page=*

# Block WordPress REST API (if any remnants)
Disallow: /wp-json/
Disallow: /wp-admin/
Disallow: /wp-includes/

# Block common WordPress files (if any remnants)
Disallow: /readme.html
Disallow: /license.txt

# Allow all valid pages
Allow: /
Allow: /tutorial/
Allow: /reference/
Allow: /apis/
Allow: /forms/
Allow: /graphics/
Allow: /media/
Allow: /examples/
Allow: /*.html$

# ===================================
# AI SEARCH ENGINE CRAWLERS (2025)
# ===================================

# ChatGPT
User-agent: ChatGPT-User
Allow: /

# OpenAI SearchGPT
User-agent: OAI-SearchBot
Allow: /

# Anthropic Claude
User-agent: Claude-Web
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI (Bard/Gemini)
User-agent: GoogleOther
Allow: /

# Bing AI/Copilot
User-agent: Bingbot
Allow: /

# ===================================
# SITEMAP & CRAWL SETTINGS
# ===================================
Sitemap: https://htmlfreecodes.com/sitemap.xml

# Crawl delay (helps prevent server overload)
Crawl-delay: 1