commit 7840cb6fabe68564aa5742594c653490ffc2ca4e
author Sebastian Nagel <snagel@apache.org> Thu Feb 20 15:09:02 2020 +0100
committer Sebastian Nagel <snagel@apache.org> Thu Feb 20 15:09:02 2020 +0100
tree 549bd9e200457b0ddf6b4ef89dcc22ea2f99e9ad
parent 8e5837f58f28f1b5bf9a714513dac926eddce3f8

NUTCH-2767 Fetcher to stop filling queues skipped due to repeated exception
- use array to count queuing status

src/java/org/apache/nutch/fetcher/QueueFeeder.java

diff --git a/src/java/org/apache/nutch/fetcher/QueueFeeder.java b/src/java/org/apache/nutch/fetcher/QueueFeeder.java
index 6c2ad39..8f64118 100644
--- a/src/java/org/apache/nutch/fetcher/QueueFeeder.java
+++ b/src/java/org/apache/nutch/fetcher/QueueFeeder.java
@@ -89,14 +89,14 @@
   public void run() {
     boolean hasMore = true;
     int cnt = 0;
-    Map<QueuingStatus, Integer> queuingStatus = new TreeMap<>();
+    int[] queuingStatus = new int[QueuingStatus.values().length];
     while (hasMore) {
       if (System.currentTimeMillis() >= timelimit && timelimit != -1) {
         // enough ... lets' simply read all the entries from the input without
         // processing them
         try {
           hasMore = context.nextKeyValue();
-          queuingStatus.compute(QueuingStatus.HIT_BY_TIMELIMIT, (k, v) -> v == null ? 1 : v + 1);
+          queuingStatus[QueuingStatus.HIT_BY_TIMELIMIT.ordinal()]++;
         } catch (IOException e) {
           LOG.error("QueueFeeder error reading input, record " + cnt, e);
           return;
@@ -140,7 +140,7 @@
             CrawlDatum datum = new CrawlDatum();
             datum.set((CrawlDatum) context.getCurrentValue());
             QueuingStatus status = queues.addFetchItem(url, datum);
-            queuingStatus.compute(status, (k, v) -> v == null ? 1 : v + 1);
+            queuingStatus[status.ordinal()]++;
             if (status == QueuingStatus.ABOVE_EXCEPTION_THRESHOLD) {
               context
                   .getCounter("FetcherStatus", "AboveExceptionThresholdInQueue")
@@ -158,11 +158,9 @@
       }
     }
     LOG.info("QueueFeeder finished: total {} records", cnt);
-    if (queuingStatus.size() > 0) {
-      LOG.info("QueueFeeder queuing status:");
-      for (Map.Entry<QueuingStatus, Integer> e : queuingStatus.entrySet()) {
-        LOG.info("\t{}\t{}", e.getValue(), e.getKey());
-      }
+    LOG.info("QueueFeeder queuing status:");
+    for (QueuingStatus status : QueuingStatus.values()) {
+      LOG.info("\t{}\t{}", queuingStatus[status.ordinal()], status);
     }
   }
 }