diff --git a/src/main/resources/urlfilters.json b/src/main/resources/urlfilters.json index 5e9c856..eb5e546 100644 --- a/src/main/resources/urlfilters.json +++ b/src/main/resources/urlfilters.json @@ -57,13 +57,6 @@ "params": { "isSitemap": "false" } - }, - { - "class": "com.digitalpebble.stormcrawler.filtering.robots.RobotsFilter", - "name": "RobotsFilter", - "params": { - "limitToSameHost": "true" - } } ] }