commit b8d1e4f84086cd8bfe411e0ab5564957ab6e18ef
author Sebastian Nagel <snagel@apache.org> Mon Dec 23 11:57:16 2019 +0100
committer GitHub <noreply@github.com> Mon Dec 23 11:57:16 2019 +0100
tree 523b7f2f09a8473a2b24c9bb9abfb0e94ea3b57d
parent 80b8a1c7de913b96f717a9b8a55985c24d1ffb20
parent 4c74bcece7f743a4ec008550f709c259317c5aa4

Merge pull request #487 from commoncrawl/NUTCH-2754-max-crawl-delay

NUTCH-2754 fetcher.max.crawl.delay ignored if exceeding 5 min. / 300 sec.

src/java/org/apache/nutch/protocol/RobotRulesParser.java

diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
index 0671a8f..159f34f 100644
--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
@@ -78,6 +78,10 @@
       RobotRulesMode.ALLOW_NONE);
 
   private static SimpleRobotRulesParser robotParser = new SimpleRobotRulesParser();
+  static {
+    robotParser.setMaxCrawlDelay(Long.MAX_VALUE);
+  }
+
   protected Configuration conf;
   protected String agentNames;