# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
# The only search engine we like is Google. Why anybody uses anything else
# is beyond us. Quite frankly, we just as soon not have people hanging
# around our site if they haven't achieved some competency with the web.
User-agent: Googlebot
Disallow: /home/feedback
Disallow: /home/email
Disallow: /home/mailing_lists_subscribe
Disallow: /meal_mini_calendar
# Googlebot, please use /recipes/list instead :)
Disallow: /cookbooks
Disallow: /recipes/search
Disallow: /recipes/search_by_category
Disallow:
# It *sorely* pains us to do this, but we're getting far too many people
# either linking to our images for utterly inane forum discussions or
# outright stealing (in the RIAA sense) our images for various purposes.
# Since no one has yet to ask permission, we're sadly forced to:
User-agent: Googlebot-Image
Disallow: /
# All right, the way-back machine is pretty cool too
User-agent: ia_archiver
Disallow: /home/feedback
Disallow: /home/email
Disallow: /home/mailing_lists_subscribe
Disallow: /cookbooks/
Disallow:
# Yahoo!'s feed reader uses robots.txt (erroneously, IOHO), so we must
# Allow it here. Hopefully no other RSS readers will make this mistake
User-agent: YahooFeedSeeker
Disallow:
User-agent: *
Disallow: /