From Botched Teal, 10 Years ago, written in Plain Text.
Embed
  1. Sitemap: http://www.fbi.gov/sitemap.xml
  2.  
  3. User-agent: Googlebot
  4. Disallow: /*.js$
  5. Disallow: /*.css$
  6.  
  7. User-agent: *
  8. Crawl-delay: 5
  9. Disallow: /audio/
  10. Disallow: /cgi-bin/
  11. Disallow: /includes/
  12. Disallow: /Includes/
  13. Disallow: /images/
  14. Disallow: /scripts/
  15. Disallow: /test/
  16. Disallow: /login
  17. Disallow: /help/
  18. Disallow: /javascript/
  19. Disallow: /css/
  20. Disallow: /redirects/
  21. Disallow: /cgi/
  22. Disallow: /error/
  23. Disallow: /esi/
  24. Disallow: /graphics/
  25. Disallow: /js/
  26. Disallow: /swf/
  27. Disallow: /xml/
  28. Disallow: /sitemap.html
  29. Disallow: /*sendto_form$
  30. Disallow: /*folder_factories$
  31. Disallow: /collections/
  32. Disallow: /news/pressrel/press-releases/collections-and-portlet/
  33. Disallow: /*?
  34. Disallow: /*?*
  35. Disallow: /news/stories/story-index/
  36. Disallow: /news/current/
  37. Disallow: /ipaccess
  38. Disallow: /ipaccess/
  39. Disallow: /news/videos/mp4*
  40. Disallow: /foia/privacy-impact-assessments/IBPF*
  41.  
  42. User-agent: Yandex
  43. Crawl-delay: 5
  44. Disallow: /*.js$
  45. Disallow: /*.css$
  46. Disallow: /audio/
  47. Disallow: /cgi-bin/
  48. Disallow: /includes/
  49. Disallow: /Includes/
  50. Disallow: /images/
  51. Disallow: /scripts/
  52. Disallow: /test/
  53. Disallow: /login
  54. Disallow: /help/
  55. Disallow: /javascript/
  56. Disallow: /css/
  57. Disallow: /redirects/
  58. Disallow: /cgi/
  59. Disallow: /error/
  60. Disallow: /esi/
  61. Disallow: /graphics/
  62. Disallow: /js/
  63. Disallow: /swf/
  64. Disallow: /xml/
  65. Disallow: /sitemap.html
  66. Disallow: /*sendto_form$
  67. Disallow: /*folder_factories$
  68. Disallow: /collections/
  69. Disallow: /news/pressrel/press-releases/collections-and-portlet/
  70. Disallow: /*?
  71. Disallow: /*?*
  72. Disallow: /news/stories/story-index/
  73. Disallow: /news/current/
  74. Disallow: /ipaccess
  75. Disallow: /ipaccess/
  76. Disallow: /news/videos/mp4*
  77.  
  78. User-agent: 008
  79. Disallow: /