############################### # # robots.txt file for this website # unique and interesting # addresses all robots by using wild card * # User-agent: ia_archiver Disallow: / User-agent: Titan Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-Agent: Scooter Disallow: / User-agent: * # list folders robots are not allowed to index # Disallow: /images Disallow: /_private Disallow: /stats Disallow: /login Disallow: /outbound Disallow: /admin Disallow: /usa Disallow: /ssfm Disallow: /includes # # list specific files robots are not allowed to index # Disallow: CellSignsRSS.asp Disallow: imageVerification.asp # # # # End of robots.txt file # ###############################