robots.txt 1.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used: http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/robotstxt.html
  15. #
  16. # For syntax checking, see:
  17. # http://www.frobee.com/robots-txt-check
  18. User-agent: *
  19. Crawl-delay: 10
  20. # Directories
  21. Disallow: /includes/
  22. Disallow: /misc/
  23. Disallow: /modules/
  24. Disallow: /profiles/
  25. Disallow: /scripts/
  26. Disallow: /themes/
  27. # Files
  28. Disallow: /CHANGELOG.txt
  29. Disallow: /cron.php
  30. Disallow: /INSTALL.mysql.txt
  31. Disallow: /INSTALL.pgsql.txt
  32. Disallow: /INSTALL.sqlite.txt
  33. Disallow: /install.php
  34. Disallow: /INSTALL.txt
  35. Disallow: /LICENSE.txt
  36. Disallow: /MAINTAINERS.txt
  37. Disallow: /update.php
  38. Disallow: /UPGRADE.txt
  39. Disallow: /xmlrpc.php
  40. # Paths (clean URLs)
  41. Disallow: /admin/
  42. Disallow: /comment/reply/
  43. Disallow: /filter/tips/
  44. Disallow: /node/add/
  45. Disallow: /search/
  46. Disallow: /user/register/
  47. Disallow: /user/password/
  48. Disallow: /user/login/
  49. Disallow: /user/logout/
  50. # Paths (no clean URLs)
  51. Disallow: /?q=admin/
  52. Disallow: /?q=comment/reply/
  53. Disallow: /?q=filter/tips/
  54. Disallow: /?q=node/add/
  55. Disallow: /?q=search/
  56. Disallow: /?q=user/password/
  57. Disallow: /?q=user/register/
  58. Disallow: /?q=user/login/
  59. Disallow: /?q=user/logout/