robotstxt.origin 1.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used: http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/robotstxt.html
  15. User-agent: *
  16. # sitemap
  17. Allow: /sitemap.xml
  18. # CSS, JS, Images
  19. Allow: /core/*.css$
  20. Allow: /core/*.css?
  21. Allow: /core/*.js$
  22. Allow: /core/*.js?
  23. Allow: /core/*.gif
  24. Allow: /core/*.jpg
  25. Allow: /core/*.jpeg
  26. Allow: /core/*.png
  27. Allow: /core/*.svg
  28. Allow: /profiles/*.css$
  29. Allow: /profiles/*.css?
  30. Allow: /profiles/*.js$
  31. Allow: /profiles/*.js?
  32. Allow: /profiles/*.gif
  33. Allow: /profiles/*.jpg
  34. Allow: /profiles/*.jpeg
  35. Allow: /profiles/*.png
  36. Allow: /profiles/*.svg
  37. # Directories
  38. Disallow: /core/
  39. Disallow: /profiles/
  40. # Files
  41. Disallow: /README.txt
  42. Disallow: /web.config
  43. # Paths (clean URLs)
  44. Disallow: /admin/
  45. Disallow: /comment/reply/
  46. Disallow: /filter/tips
  47. Disallow: /node/add/
  48. Disallow: /search/
  49. Disallow: /user/register
  50. Disallow: /user/password
  51. Disallow: /user/login
  52. Disallow: /user/logout
  53. # Paths (no clean URLs)
  54. Disallow: /index.php/admin/
  55. Disallow: /index.php/comment/reply/
  56. Disallow: /index.php/filter/tips
  57. Disallow: /index.php/node/add/
  58. Disallow: /index.php/search/
  59. Disallow: /index.php/user/password
  60. Disallow: /index.php/user/register
  61. Disallow: /index.php/user/login
  62. Disallow: /index.php/user/logout
  63. # Paths Hacks
  64. Disallow: /bafeshop/
  65. Disallow: /bafishop/
  66. Disallow: /baffshop/
  67. Disallow: /bafyshop/
  68. Disallow: /bafoshop/
  69. Disallow: /bafnshop/
  70. Disallow: /bafgshop/
  71. Disallow: /bafsshop/