Removed robots filter as not part of initial benchmark + hurts perfor…

…mance
DigitalPebble · Jan 19, 2017 · d1c3fb5 · d1c3fb5
1 parent 8e659dc
commit d1c3fb5
Showing 1 changed file with 0 additions and 7 deletions.
diff --git a/src/main/resources/urlfilters.json b/src/main/resources/urlfilters.json
@@ -57,13 +57,6 @@
       "params": {
         "isSitemap": "false"
       }
-    },
-    {
-      "class": "com.digitalpebble.stormcrawler.filtering.robots.RobotsFilter",
-      "name": "RobotsFilter",
-      "params": {
-        "limitToSameHost": "true"
-      }
     }
   ]
 }