Commit 6ea79950c6db043995464c5f8cc2bd76840919ec

Authored by Paulo Graça
1 parent aaec77acfd
Exists in DSpace52++

remove other crawlers

Showing 1 changed file with 67 additions and 0 deletions   Show diff stats
dspace-jspui/src/main/webapp/robots.txt
... ... @@ -149,3 +149,70 @@ Disallow: /
149 149 # http://www.webreaper.net/
150 150 User-agent: WebReaper
151 151 Disallow: /
  152 +
  153 +
  154 +# RCAAP: remove other crawlers
  155 +User-agent: adbeat_bot
  156 +Disallow: /
  157 +User-agent: AhrefsBot
  158 +Disallow: /
  159 +User-agent: AITCSRobot
  160 +Disallow: /
  161 +User-agent: Alexibot
  162 +Disallow: /
  163 +User-agent: Baiduspider
  164 +Disallow: /
  165 +User-agent: BLEXBot
  166 +Disallow: /
  167 +User-agent: Cliqzbot
  168 +Disallow: /
  169 +User-agent: DotBot
  170 +Disallow: /
  171 +User-agent: dotbot
  172 +Disallow: /
  173 +User-agent: Exabot
  174 +Disallow: /
  175 +User-agent: expo9
  176 +Disallow: /
  177 +User-agent: Huaweisymantecspider
  178 +Disallow: /
  179 +User-agent: InfluenceBot
  180 +Disallow: /
  181 +User-agent: ltx71
  182 +Disallow: /
  183 +User-agent: ltx71 - (http://ltx71.com/)
  184 +Disallow: /
  185 +User-agent: MaxPointCrawler
  186 +Disallow: /
  187 +User-agent: MJ12bot
  188 +Disallow: /
  189 +User-agent: rogerbot
  190 +Disallow: /
  191 +User-agent: SemrushBot
  192 +Disallow: /
  193 +User-agent: SemrushBot-SA
  194 +Disallow: /
  195 +User-agent: seoscanners.net
  196 +Disallow: /
  197 +User-agent: SiteSnagger
  198 +Disallow: /
  199 +User-agent: SurveyBot
  200 +Disallow: /
  201 +User-agent: turnitinbot
  202 +Disallow: /
  203 +User-agent: WebReaper
  204 +Disallow: /
  205 +User-agent: Xaldon_WebSpider
  206 +Disallow: /
  207 +User-agent: Xenu’s
  208 +Disallow: /
  209 +User-agent: Xenu’s Link Sleuth 1.1c
  210 +Disallow: /
  211 +User-agent: YandexBot
  212 +Disallow: /
  213 +
  214 +# RCAAP delay crallers
  215 +User-agent: bingbot
  216 +Crawl-delay: 10
  217 +User-agent: Yahoo
  218 +Crawl-delay: 10
152 219 \ No newline at end of file
... ...