robots.txt 2.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used: http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/robotstxt.html
  15. User-agent: *
  16. Crawl-delay: 10
  17. # CSS, JS, Images
  18. Allow: /misc/*.css$
  19. Allow: /misc/*.css?
  20. Allow: /misc/*.js$
  21. Allow: /misc/*.js?
  22. Allow: /misc/*.gif
  23. Allow: /misc/*.jpg
  24. Allow: /misc/*.jpeg
  25. Allow: /misc/*.png
  26. Allow: /modules/*.css$
  27. Allow: /modules/*.css?
  28. Allow: /modules/*.js$
  29. Allow: /modules/*.js?
  30. Allow: /modules/*.gif
  31. Allow: /modules/*.jpg
  32. Allow: /modules/*.jpeg
  33. Allow: /modules/*.png
  34. Allow: /profiles/*.css$
  35. Allow: /profiles/*.css?
  36. Allow: /profiles/*.js$
  37. Allow: /profiles/*.js?
  38. Allow: /profiles/*.gif
  39. Allow: /profiles/*.jpg
  40. Allow: /profiles/*.jpeg
  41. Allow: /profiles/*.png
  42. Allow: /themes/*.css$
  43. Allow: /themes/*.css?
  44. Allow: /themes/*.js$
  45. Allow: /themes/*.js?
  46. Allow: /themes/*.gif
  47. Allow: /themes/*.jpg
  48. Allow: /themes/*.jpeg
  49. Allow: /themes/*.png
  50. # Directories
  51. Disallow: /includes/
  52. Disallow: /misc/
  53. Disallow: /modules/
  54. Disallow: /profiles/
  55. Disallow: /scripts/
  56. Disallow: /themes/
  57. # Files
  58. Disallow: /CHANGELOG.txt
  59. Disallow: /cron.php
  60. Disallow: /INSTALL.mysql.txt
  61. Disallow: /INSTALL.pgsql.txt
  62. Disallow: /INSTALL.sqlite.txt
  63. Disallow: /install.php
  64. Disallow: /INSTALL.txt
  65. Disallow: /LICENSE.txt
  66. Disallow: /MAINTAINERS.txt
  67. Disallow: /update.php
  68. Disallow: /UPGRADE.txt
  69. Disallow: /xmlrpc.php
  70. # Paths (clean URLs)
  71. Disallow: /admin/
  72. Disallow: /comment/reply/
  73. Disallow: /filter/tips/
  74. Disallow: /node/add/
  75. Disallow: /search/
  76. Disallow: /user/register/
  77. Disallow: /user/password/
  78. Disallow: /user/login/
  79. Disallow: /user/logout/
  80. # Paths (no clean URLs)
  81. Disallow: /?q=admin/
  82. Disallow: /?q=comment/reply/
  83. Disallow: /?q=filter/tips/
  84. Disallow: /?q=node/add/
  85. Disallow: /?q=search/
  86. Disallow: /?q=user/password/
  87. Disallow: /?q=user/register/
  88. Disallow: /?q=user/login/
  89. Disallow: /?q=user/logout/