commit aff04c8484990545edb5e997894438d04033d1bb
author Jurian Broertjes <jurian.broertjes@openindex.io> Thu May 16 16:01:16 2019 +0000
committer Sebastian Nagel <snagel@apache.org> Fri May 24 15:30:19 2019 +0200
tree 8a61eae2f7cbc5fb8e26b5b9b8cdc5e3e9d3407d
parent a2b944f89514b391e0a2eb0fbdac59cd0d01d455

fix for NUTCH-2717

src/java/org/apache/nutch/crawl/Generator.java

diff --git a/src/java/org/apache/nutch/crawl/Generator.java b/src/java/org/apache/nutch/crawl/Generator.java
index e4afbe9..bc6a3aa 100644
--- a/src/java/org/apache/nutch/crawl/Generator.java
+++ b/src/java/org/apache/nutch/crawl/Generator.java
@@ -55,7 +55,7 @@
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
@@ -72,6 +72,7 @@
 import org.apache.nutch.util.NutchConfiguration;
 import org.apache.nutch.util.NutchJob;
 import org.apache.nutch.util.NutchTool;
+import org.apache.nutch.util.SegmentReaderUtil;
 import org.apache.nutch.util.TimingUtil;
 import org.apache.nutch.util.URLUtil;
 
@@ -292,7 +293,7 @@
       private boolean byDomain = false;
       private URLNormalizers normalizers;
       private static boolean normalise;
-      private MapFile.Reader[] hostdbReaders = null;
+      private SequenceFile.Reader[] hostdbReaders = null;
       private Expression maxCountExpr = null;
       private Expression fetchDelayExpr = null;
 
@@ -300,7 +301,7 @@
         if (conf.get(GENERATOR_HOSTDB) != null) {
           try {
             Path path = new Path(conf.get(GENERATOR_HOSTDB), "current");
-            hostdbReaders = MapFileOutputFormat.getReaders(path, conf);
+            hostdbReaders = SegmentReaderUtil.getReaders(path, conf);
           } catch (IOException e) {
             LOG.error("Error reading HostDB because {}", e.getMessage());
           }