# robots.txt for thumbtail.com
# Updated: March 8, 2025

User-agent: *
Allow: /

# Explicitly allow blog indexing
Allow: /blog/

# Sitemap location
Sitemap: https://thumbtail.com/sitemap.xml

# Block access to administrative areas
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /admin/
Disallow: /phpmyadmin/
Disallow: /cpanel/

# Block access to system directories
Disallow: /vendor/
Disallow: /config/
Disallow: /backup/
Disallow: /logs/

# Block individual files
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$

# Allow JavaScript files for better indexing
Allow: /js/*.js

# If using WordPress
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/*/readme.txt
Disallow: /wp-content/debug.log

# Allow Let's Encrypt SSL verification
Allow: /.well-known/acme-challenge/

# Sitemap
Sitemap: https://thumbtail.com/sitemap.xml

# Encourage crawling of important pages
Allow: /index.html
Allow: /download.html
Allow: /blog/
Allow: /privacy-policy.html
Allow: /terms.html

# Crawl delay for intensive bots (optional)
# Crawl-delay: 10