Commit 6ea79950c6db043995464c5f8cc2bd76840919ec
1 parent
aaec77acfd
Exists in
DSpace52++
remove other crawlers
Showing
1 changed file
with
67 additions
and
0 deletions
Show diff stats
dspace-jspui/src/main/webapp/robots.txt
... | ... | @@ -149,3 +149,70 @@ Disallow: / |
149 | 149 | # http://www.webreaper.net/ |
150 | 150 | User-agent: WebReaper |
151 | 151 | Disallow: / |
152 | + | |
153 | + | |
154 | +# RCAAP: remove other crawlers | |
155 | +User-agent: adbeat_bot | |
156 | +Disallow: / | |
157 | +User-agent: AhrefsBot | |
158 | +Disallow: / | |
159 | +User-agent: AITCSRobot | |
160 | +Disallow: / | |
161 | +User-agent: Alexibot | |
162 | +Disallow: / | |
163 | +User-agent: Baiduspider | |
164 | +Disallow: / | |
165 | +User-agent: BLEXBot | |
166 | +Disallow: / | |
167 | +User-agent: Cliqzbot | |
168 | +Disallow: / | |
169 | +User-agent: DotBot | |
170 | +Disallow: / | |
171 | +User-agent: dotbot | |
172 | +Disallow: / | |
173 | +User-agent: Exabot | |
174 | +Disallow: / | |
175 | +User-agent: expo9 | |
176 | +Disallow: / | |
177 | +User-agent: Huaweisymantecspider | |
178 | +Disallow: / | |
179 | +User-agent: InfluenceBot | |
180 | +Disallow: / | |
181 | +User-agent: ltx71 | |
182 | +Disallow: / | |
183 | +User-agent: ltx71 - (http://ltx71.com/) | |
184 | +Disallow: / | |
185 | +User-agent: MaxPointCrawler | |
186 | +Disallow: / | |
187 | +User-agent: MJ12bot | |
188 | +Disallow: / | |
189 | +User-agent: rogerbot | |
190 | +Disallow: / | |
191 | +User-agent: SemrushBot | |
192 | +Disallow: / | |
193 | +User-agent: SemrushBot-SA | |
194 | +Disallow: / | |
195 | +User-agent: seoscanners.net | |
196 | +Disallow: / | |
197 | +User-agent: SiteSnagger | |
198 | +Disallow: / | |
199 | +User-agent: SurveyBot | |
200 | +Disallow: / | |
201 | +User-agent: turnitinbot | |
202 | +Disallow: / | |
203 | +User-agent: WebReaper | |
204 | +Disallow: / | |
205 | +User-agent: Xaldon_WebSpider | |
206 | +Disallow: / | |
207 | +User-agent: Xenu’s | |
208 | +Disallow: / | |
209 | +User-agent: Xenu’s Link Sleuth 1.1c | |
210 | +Disallow: / | |
211 | +User-agent: YandexBot | |
212 | +Disallow: / | |
213 | + | |
214 | +# RCAAP delay crallers | |
215 | +User-agent: bingbot | |
216 | +Crawl-delay: 10 | |
217 | +User-agent: Yahoo | |
218 | +Crawl-delay: 10 | |
152 | 219 | \ No newline at end of file | ... | ... |