# robots.txt file for petvblog.com
# Author: Your Name
# Date: 2024-06-03

# Allow all search engines full access to blog content
User-agent: *
Disallow:

# Block specific folders not intended for public access
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /admin/
Disallow: /scripts/
Disallow: /backend/

# Block specific files not intended for public access
Disallow: /secret.html
Disallow: /login.html
Disallow: /register.html

# Allow full access to specific bots for images and ads
User-agent: Googlebot-Image
Disallow:

User-agent: Mediapartners-Google
Disallow:

# Sitemap location
Sitemap: https://www.petvblog.com/sitemaps.xml

# Crawl-delay directive for all bots (remove or update value if necessary)
# Crawl-delay: 10

# Specific directives for major search engines
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

# Block specific user agents if needed
User-agent: BadBot
Disallow: /