KURTZ

Originally Posted by KURTZ

thanks Brian just tested this:

Code:

# Allow Archive.org to save snapshots of everything
User-agent: ia_archiver
Allow: /

# Tame yahoo... it tends to eat a ton of resources without a delay
User-agent: Slurp
Crawl-delay: 60


#list individual pages and files here that all bots should ignore, as well as group extentions.
#If you re-write everything to .html, you can disallow *.php, but note that if you don't have a CRR for custom pages, those will be blocked.

User-agent: *
Disallow: *.js
Disallow: /board/clientscript/
Disallow: /board/cpstyles/
Disallow: /board/customavatars/
Disallow: /board/customprofilepics/
Disallow: /board/images/
Disallow: /board/ajax.php
Disallow: /board/attachment.php
Disallow: /board/calendar.php
Disallow: /board/cron.php
Disallow: /board/editpost.php
Disallow: /board/global.php
Disallow: /board/image.php
Disallow: /board/inlinemod.php
Disallow: /board/joinrequests.php
Disallow: /board/login.php
Disallow: /board/member.php
Disallow: /board/memberlist.php
Disallow: /board/misc.php
Disallow: /board/moderator.php
Disallow: /board/newattachment.php
Disallow: /board/newreply.php
Disallow: /board/newthread.php
Disallow: /board/online.php
Disallow: /board/poll.php
Disallow: /board/postings.php
Disallow: /board/printthread.php
Disallow: /board/private.php
Disallow: /board/profile.php
Disallow: /board/register.php
Disallow: /board/report.php
Disallow: /board/reputation.php
Disallow: /board/search.php
Disallow: /board/sendmessage.php
Disallow: /board/showgroups.php
Disallow: /board/subscription.php
Disallow: /board/threadrate.php
Disallow: /board/usercp.php
Disallow: /board/usernote.php

#Finally, list the path to your sitemap:
Sitemap: http://www.fniv.it/board/sitemap_index.xml.gz