# This is the recommended default robot.txt file. Please feel free to modify it to # your liking. Please keep in mind that a blank line is required between each # User-agent you specify. # # You can check the syntax of your file at: # http://tool.motoricerca.info/robots-checker.phtml # # More information can be found at http://www.robotstxt.org/wc/norobots.html # or http://www.w3.org/TR/html4/appendix/notes.html#h-B.4.1 #-------------------------------------------------------------------------------------- # Image and blog Bots # Ban all image bots. We don't want family photos searched. #-------------------------------------------------------------------------------------- User-agent: Googlebot-Image #--- Google Image User-agent: yahoo-mmcrawler #--- Yahoo MM User-agent: psbot #--- MSN PicSearch User-agent: asterias #--- SingingFish User-agent: yahoo-blogs/v3.9 #--- Yahoo Blogs Crawl-delay: 60 Disallow: / #-------------------------------------------------------------------------------------- # Mainstream Bots # Allow all mainstream bot to search most directories # "Disallow: " specifies that nothing is disallowed #-------------------------------------------------------------------------------------- User-agent: Googlebot #--- Google User-agent: MSNBot #--- MSN Search User-agent: Slurp #--- Yahoo User-agent: Teoma #--- Ask/Teoma User-agent: Nutch #--- Nutch User-agent: Checkbot #--- http://degraaff.org/checkbot/ Crawl-delay: 10 Disallow: /cgi-bin/ Disallow: /images/ Disallow: /image/ Disallow: /test/ Disallow: /tmp/ Disallow: /admin/ Disallow: /administrator/ Disallow: /phpCalendar/ Disallow: /gallery/ Disallow: /calendar/ Disallow: com_event Disallow: /guestbook/ #-------------------------------------------------------------------------------------- # Other less well know and perhaps questionable Bots # Ban other troublesome bots # More info at http://www.robotstxt.org/wc/active/html/index.html #-------------------------------------------------------------------------------------- User-agent: Gigabot/2.0 #--- GigaBlast User-agent: Gigabot #--- GigaBlast User-agent: Scrubby #--- Scrub The Web User-agent: ia_archiver #--- Alexa/Waybac User-agent: baiduspider #--- Baidu User-agent: BlackWidow #--- Disallow: / #-------------------------------------------------------------------------------------- # Default (if not specified above) #-------------------------------------------------------------------------------------- User-agent: * Crawl-delay: 60 Disallow: /cgi-bin/ Disallow: /images/ Disallow: /image/ Disallow: /test/ Disallow: /tmp/ Disallow: /admin/ Disallow: /administrator/ Disallow: /phpCalendar/ Disallow: /gallery/ Disallow: /calendar/ Disallow: com_event Disallow: /guestbook/