# /robots.txt file for http://midiextender.com/
#The basic idea is that if you include a tag like: 
#	<META NAME="ROBOTS" CONTENT="NOINDEX">
#		in your HTML document, that document won't be indexed. 
#
# 	<META NAME="ROBOTS" CONTENT="NOFOLLOW">
#		the links in that document will not be parsed by the robot. 
#	<META NAME="ROBOTS" CONTENT="NOINDEX, NOFOLLOW">
#
# example the two robots can get only ok in tmp
# User-agent: webcrawler
# User-agent: infoseek
# Allow:    /tmp/ok.html
# Disallow: /tmp

# disallow back end stuff
User-agent: *
Disallow: /cgi-bin
Disallow: /logs

# disallow stats pages
Disallow: /webalizer
Disallow: /modlogan


#To remove all info from the wayback machine
User-agent: ia_archiver
Disallow: /