commit d046b469550715ded6db88934461ba63f17a5ee5
author YossiTamari <33034182+YossiTamari@users.noreply.github.com> Tue Oct 15 12:41:25 2019 +0300
committer GitHub <noreply@github.com> Tue Oct 15 12:41:25 2019 +0300
tree 0414cbefc76c53ce5f1c1da3a296ab0dadbacb02
parent 968fc7ec01f8cba27f9fa19b52bfa9033619d1dc

Add sitemap.size.max

Overrides http.content.limit and file.content.limit when fetching sitemaps (which can be pretty big).

conf/nutch-default.xml

diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index 2ba04d2..97fcbe4 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -2671,4 +2671,12 @@
     Maximum number of redirects to follow.
    </description>
 </property>
+
+<property>
+  <name>sitemap.size.max</name>
+  <value>52428800</value>
+  <description>
+    Maximum sitemap size in bytes.
+   </description>
+</property>
 </configuration>