Commit 6ea79950c6db043995464c5f8cc2bd76840919ec
1 parent
aaec77acfd
Exists in
DSpace52++
remove other crawlers
Showing
1 changed file
with
67 additions
and
0 deletions
Show diff stats
dspace-jspui/src/main/webapp/robots.txt
| ... | ... | @@ -149,3 +149,70 @@ Disallow: / |
| 149 | 149 | # http://www.webreaper.net/ |
| 150 | 150 | User-agent: WebReaper |
| 151 | 151 | Disallow: / |
| 152 | + | |
| 153 | + | |
| 154 | +# RCAAP: remove other crawlers | |
| 155 | +User-agent: adbeat_bot | |
| 156 | +Disallow: / | |
| 157 | +User-agent: AhrefsBot | |
| 158 | +Disallow: / | |
| 159 | +User-agent: AITCSRobot | |
| 160 | +Disallow: / | |
| 161 | +User-agent: Alexibot | |
| 162 | +Disallow: / | |
| 163 | +User-agent: Baiduspider | |
| 164 | +Disallow: / | |
| 165 | +User-agent: BLEXBot | |
| 166 | +Disallow: / | |
| 167 | +User-agent: Cliqzbot | |
| 168 | +Disallow: / | |
| 169 | +User-agent: DotBot | |
| 170 | +Disallow: / | |
| 171 | +User-agent: dotbot | |
| 172 | +Disallow: / | |
| 173 | +User-agent: Exabot | |
| 174 | +Disallow: / | |
| 175 | +User-agent: expo9 | |
| 176 | +Disallow: / | |
| 177 | +User-agent: Huaweisymantecspider | |
| 178 | +Disallow: / | |
| 179 | +User-agent: InfluenceBot | |
| 180 | +Disallow: / | |
| 181 | +User-agent: ltx71 | |
| 182 | +Disallow: / | |
| 183 | +User-agent: ltx71 - (http://ltx71.com/) | |
| 184 | +Disallow: / | |
| 185 | +User-agent: MaxPointCrawler | |
| 186 | +Disallow: / | |
| 187 | +User-agent: MJ12bot | |
| 188 | +Disallow: / | |
| 189 | +User-agent: rogerbot | |
| 190 | +Disallow: / | |
| 191 | +User-agent: SemrushBot | |
| 192 | +Disallow: / | |
| 193 | +User-agent: SemrushBot-SA | |
| 194 | +Disallow: / | |
| 195 | +User-agent: seoscanners.net | |
| 196 | +Disallow: / | |
| 197 | +User-agent: SiteSnagger | |
| 198 | +Disallow: / | |
| 199 | +User-agent: SurveyBot | |
| 200 | +Disallow: / | |
| 201 | +User-agent: turnitinbot | |
| 202 | +Disallow: / | |
| 203 | +User-agent: WebReaper | |
| 204 | +Disallow: / | |
| 205 | +User-agent: Xaldon_WebSpider | |
| 206 | +Disallow: / | |
| 207 | +User-agent: Xenu’s | |
| 208 | +Disallow: / | |
| 209 | +User-agent: Xenu’s Link Sleuth 1.1c | |
| 210 | +Disallow: / | |
| 211 | +User-agent: YandexBot | |
| 212 | +Disallow: / | |
| 213 | + | |
| 214 | +# RCAAP delay crallers | |
| 215 | +User-agent: bingbot | |
| 216 | +Crawl-delay: 10 | |
| 217 | +User-agent: Yahoo | |
| 218 | +Crawl-delay: 10 | |
| 152 | 219 | \ No newline at end of file | ... | ... |