# For domain: http://www.findallabout.com

# Robots that will not spider the domain
User-agent: psbot
Disallow: /
User-agent: Twiceler
Disallow: /
User-agent: IRLbot 
Disallow: /
User-agent: HouxouCrawler 
Disallow: / 

# All robots will spider the domain
User-agent: *
Disallow: /cgi-bin/
Disallow: /fpdf/
Disallow: /images/
Disallow: /import_scripts/
Disallow: /private/
Disallow: /templates/
Disallow: /ads.php/
Disallow: /ads_wide.php/
Disallow: /author_activate.php/
Disallow: /author_change_email.php/
Disallow: /author_confirm.php/
Disallow: /author_edit_article.php/
Disallow: /author_footer.php/
Disallow: /author_header.php/
Disallow: /author_login.php/
Disallow: /author_logoff.php/
Disallow: /author_manage_pennames.php/
Disallow: /author_profile.php/
Disallow: /author_register.php/
Disallow: /author_submit.php/
Disallow: /author_view_articles.php/
Disallow: /captcha.php/
Disallow: /captcha_image.php/
Disallow: /config.php/
Disallow: /confirm.php/
Disallow: /create_report.php/
Disallow: /error_log/
Disallow: /footer.php/
Disallow: /form.html/
Disallow: /form.php/
Disallow: /functions.php/
Disallow: /header.php/
Disallow: /hurryup.tif/
Disallow: /import.php/
Disallow: /import_settings.php/
Disallow: /newsfeed.php/
Disallow: /php.ini/
Disallow: /privacy_policy.php/
Disallow: /sample_article_mail.html/
Disallow: /search.php/
Disallow: /submission_guideline.php/
Disallow: /terms_of_service.php/
Disallow: /user_unsubscribe.php/
Disallow: /view_article.php/
Disallow: /view_category.php/