robots.txt 1.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used: http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/robotstxt.html
  15. User-agent: *
  16. # CSS, JS, Images
  17. Allow: /core/*.css$
  18. Allow: /core/*.css?
  19. Allow: /core/*.js$
  20. Allow: /core/*.js?
  21. Allow: /core/*.gif
  22. Allow: /core/*.jpg
  23. Allow: /core/*.jpeg
  24. Allow: /core/*.png
  25. Allow: /core/*.svg
  26. Allow: /profiles/*.css$
  27. Allow: /profiles/*.css?
  28. Allow: /profiles/*.js$
  29. Allow: /profiles/*.js?
  30. Allow: /profiles/*.gif
  31. Allow: /profiles/*.jpg
  32. Allow: /profiles/*.jpeg
  33. Allow: /profiles/*.png
  34. Allow: /profiles/*.svg
  35. # Directories
  36. Disallow: /core/
  37. Disallow: /profiles/
  38. # Files
  39. Disallow: /README.txt
  40. Disallow: /web.config
  41. # Paths (clean URLs)
  42. Disallow: /admin/
  43. Disallow: /comment/reply/
  44. Disallow: /filter/tips
  45. Disallow: /node/add/
  46. Disallow: /search/
  47. Disallow: /user/register
  48. Disallow: /user/password
  49. Disallow: /user/login
  50. Disallow: /user/logout
  51. # Paths (no clean URLs)
  52. Disallow: /index.php/admin/
  53. Disallow: /index.php/comment/reply/
  54. Disallow: /index.php/filter/tips
  55. Disallow: /index.php/node/add/
  56. Disallow: /index.php/search/
  57. Disallow: /index.php/user/password
  58. Disallow: /index.php/user/register
  59. Disallow: /index.php/user/login
  60. Disallow: /index.php/user/logout