# This robots.txt was stolen from http://sebastians-pamphlets.com/
# Ok, steal it, but leave the comments intact. 
# Don't forget to change the sitemap URL below.

# This site serves machine-readable disclosures, e.g. crawler directives 
# like rel-nofollow applied to links with commercial intent, to Web robots only.
# http://sebastians-pamphlets.com/links/full-disclosure/

# You should cloak your robots.txt file. Here's the manual for smart webmasters:
# http://sebastians-pamphlets.com/smart-robots-txt/

User-agent: *

# Archives shouldn't be crawlable IMO. Optimize your category pages instead.
Disallow: /2005/
Disallow: /2006/
Disallow: /2007/
Disallow: /2008/
Disallow: /2009/
Disallow: /2010/
Disallow: /2011/
Disallow: /2012/
Disallow: /2013/
Disallow: /2014/
Disallow: /2015/

# I don't recommend sitemaps autodiscovery for each and every site!
# For this blog it makes sense.
Sitemap: http://sebastians-pamphlets.com/sitemap.xml

# Don't disallow your feeds. That doesn't fix possible dupe issues.
# You can apply rel-nofollow to XML links, but usually that's not necessary.
# If they outrank your contents, your optimization failed badly.




# Testing, do not copy the statements below:

# Inspired by http://googlewebmastercentral.blogspot.com/2008/02/cross-submissions-via-robotstxt-on.html :
Sitemap: http://google.com/sitemap.xml

# 'Noindex:' is an undocumented directive supported only by Googlebot.
# It means 'do not crawl and do not index from 3rd-party signals'.
# If you don't know what you're doing WRT to REP directives, do not use it! 

Noindex: /noindex/

Noindex: /repstuff/noindex.php
Nofollow: /repstuff/nofollow.php
Noarchive: /repstuff/noarchive.php

Nopreview: /repstuff/nopreview.pdf
Noarchive: /repstuff/nopreview.pdf

Noindex: /repstuff/noindex-nofollow.php
Nofollow: /repstuff/noindex-nofollow.php

Noindex: /repstuff/noindex-follow.php
Follow: /repstuff/noindex-follow.php

Index: /repstuff/index-nofollow.php
Nofollow: /repstuff/index-nofollow.php

Allow: /porn/
Sitemap: http://sebastians-pamphlets.com/htpasswd-sitemap.txt

Disallow: /smut/