User-agent: * # disallow all files in these directories Disallow: /map1/ Disallow: /map2/ Disallow: /map3/ Disallow: /map4/ Disallow: /map5/ Disallow: /map6/ Disallow: /map7/ Disallow: /map8/ Disallow: /map9/ Disallow: /map10/ User-agent: Googlebot # disallow all files ending with these extensions Disallow: /*.extension$ Disallow: /*.extension$ Disallow: /*.txt$ # allow google image bot to search all images User-agent: Googlebot-Image Allow: /* # allow adsense bot on entire site User-agent: Mediapartners-Google* Disallow: /*?* # allow archiving site User-agent: ia_archiver Disallow: