robots.txt
# robots.txt generiert mit profiSUBMIT - Professional Plus
User-agent: Googlebot
Disallow:
User-agent: MSNBot
Disallow:
User-agent: yahoo-slurp
Disallow:
User-agent: ia_archiver
Disallow:
User-agent: robozilla
Disallow:
User-agent: baiduspider
Disallow:
User-agent: googlebot-image
Disallow:
User-agent: psbot
Disallow:
User-agent: yahoo-mmcrawler
Disallow:
User-agent: *
Disallow:
Disallow: /cgi-bin/
Disallow: /stats/
Sitemap: http://www.milliardenmaerkte.info/sitemap.xml
