docs/attachments/LUCENE-5516/LUCENE-5516.patch

Index: lucene/CHANGES.txt
===================================================================
--- lucene/CHANGES.txt	(revision 1576367)
+++ lucene/CHANGES.txt	(working copy)
@@ -116,6 +116,10 @@
   and a separate Sort for ordering of children within a block. 
   (Robert Muir, Mike McCandless, Adrien Grand)
 
+* LUCENE-5516: MergeScheduler#merge() now accepts a MergeTrigger as well as
+  a boolean that indicates if a new merge was found in the caller thread before
+  the scheduler was called. (Simon Willnauer)
+
 Optimizations
 
 * LUCENE-5468: HunspellStemFilter uses 10 to 100x less RAM. It also loads
Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java
===================================================================
--- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java	(revision 1576367)
+++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicy.java	(working copy)
@@ -28,6 +28,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.MergeTrigger;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentCommitInfo;
Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java	(revision 1576367)
+++ lucene/test-framework/src/java/org/apache/lucene/index/BaseMergePolicyTestCase.java	(working copy)
@@ -17,15 +17,15 @@
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 /**
  * Base test case for {@link MergePolicy}.
  */
@@ -39,11 +39,11 @@
     final AtomicBoolean mayMerge = new AtomicBoolean(true);
     final MergeScheduler mergeScheduler = new SerialMergeScheduler() {
       @Override
-      synchronized public void merge(IndexWriter writer) throws IOException {
+      synchronized public void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException {
         if (!mayMerge.get() && writer.getNextMerge() != null) {
           throw new AssertionError();
         }
-        super.merge(writer);
+        super.merge(writer, trigger, newMergesFound);
       }
     };
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergeScheduler(mergeScheduler).setMergePolicy(mergePolicy()));
Index: lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java	(revision 1576367)
+++ lucene/core/src/test/org/apache/lucene/index/TestNoMergeScheduler.java	(working copy)
@@ -17,21 +17,22 @@
  * limitations under the License.
  */
 
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.Arrays;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.junit.Test;
-
 public class TestNoMergeScheduler extends LuceneTestCase {
 
   @Test
   public void testNoMergeScheduler() throws Exception {
     MergeScheduler ms = NoMergeScheduler.INSTANCE;
     ms.close();
-    ms.merge(null);
+    ms.merge(null, RandomPicks.randomFrom(random(), MergeTrigger.values()), random().nextBoolean());
   }
 
   @Test
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java	(revision 1576367)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java	(working copy)
@@ -311,8 +311,7 @@
   // merging a segment with >= 20 (maxMergeDocs) docs
   private class MyMergeScheduler extends MergeScheduler {
     @Override
-    synchronized public void merge(IndexWriter writer)
-      throws IOException {
+    synchronized public void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException {
 
       while(true) {
         MergePolicy.OneMerge merge = writer.getNextMerge();
Index: lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java	(revision 1576367)
+++ lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java	(working copy)
@@ -16,23 +16,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import java.io.IOException;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LogMergePolicy;
 import org.apache.lucene.index.MergePolicy;
-import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.MergePolicy.OneMerge;
 import org.apache.lucene.index.MergeScheduler;
-import org.apache.lucene.index.MergePolicy.OneMerge;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.index.MergeTrigger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
 
+import java.io.IOException;
+
 /**
  * Holds tests cases to verify external APIs are accessible
  * while not being in org.apache.lucene.index package.
@@ -113,7 +114,7 @@
   private static class ReportingMergeScheduler extends MergeScheduler {
 
     @Override
-    public void merge(IndexWriter writer) throws IOException {
+    public void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException {
       OneMerge merge = null;
       while ((merge = writer.getNextMerge()) != null) {
         if (VERBOSE) {
Index: lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/SerialMergeScheduler.java	(working copy)
@@ -31,7 +31,7 @@
    * "synchronized" so that even if the application is using
    * multiple threads, only one merge may run at a time. */
   @Override
-  synchronized public void merge(IndexWriter writer) throws IOException {
+  synchronized public void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException {
 
     while(true) {
       MergePolicy.OneMerge merge = writer.getNextMerge();
Index: lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java	(working copy)
@@ -17,7 +17,6 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.index.MergePolicy.MergeTrigger;
 import org.apache.lucene.util.Constants;
 
 import java.io.IOException;
Index: lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java	(working copy)
@@ -308,7 +308,7 @@
   }
 
   @Override
-  public synchronized void merge(IndexWriter writer) throws IOException {
+  public synchronized void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException {
 
     assert !Thread.holdsLock(writer);
 
Index: lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/MergePolicy.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/MergePolicy.java	(working copy)
@@ -17,18 +17,18 @@
  * limitations under the License.
  */
 
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MergeInfo;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SetOnce;
+import org.apache.lucene.util.SetOnce.AlreadySetException;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MergeInfo;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.SetOnce.AlreadySetException;
-import org.apache.lucene.util.SetOnce;
-
 /**
  * <p>Expert: a MergePolicy determines the sequence of
  * primitive merge operations.</p>
@@ -566,29 +566,4 @@
     this.maxCFSSegmentSize = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
   }
 
-  /**
-   * MergeTrigger is passed to
-   * {@link MergePolicy#findMerges(MergeTrigger, SegmentInfos)} to indicate the
-   * event that triggered the merge.
-   */
-  public static enum MergeTrigger {
-    /**
-     * Merge was triggered by a segment flush.
-     */
-    SEGMENT_FLUSH, 
-    /**
-     * Merge was triggered by a full flush. Full flushes
-     * can be caused by a commit, NRT reader reopen or a close call on the index writer.
-     */
-    FULL_FLUSH,
-    /**
-     * Merge has been triggered explicitly by the user.
-     */
-    EXPLICIT,
-    
-    /**
-     * Merge was triggered by a successfully finished merge.
-     */
-    MERGE_FINISHED,
-  }
 }
Index: lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java	(revision 0)
+++ lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java	(working copy)
@@ -0,0 +1,49 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * MergeTrigger is passed to
+ * {@link org.apache.lucene.index.MergePolicy#findMerges(MergeTrigger, org.apache.lucene.index.SegmentInfos)} to indicate the
+ * event that triggered the merge.
+ */
+public enum MergeTrigger {
+  /**
+   * Merge was triggered by a segment flush.
+   */
+  SEGMENT_FLUSH,
+  /**
+   * Merge was triggered by a full flush. Full flushes
+   * can be caused by a commit, NRT reader reopen or a close call on the index writer.
+   */
+  FULL_FLUSH,
+  /**
+   * Merge has been triggered explicitly by the user.
+   */
+  EXPLICIT,
+
+  /**
+   * Merge was triggered by a successfully finished merge.
+   */
+  MERGE_FINISHED,
+
+  /**
+   * Merge was triggered by a closing IndexWriter.
+   */
+  CLOSING
+}

Property changes on: lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Date Author Id Revision HeadURL
\ No newline at end of property
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/NoMergeScheduler.java	(working copy)
@@ -40,7 +40,7 @@
   public void close() {}
 
   @Override
-  public void merge(IndexWriter writer) {}
+  public void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) {}
 
   @Override
   public MergeScheduler clone() {
Index: lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java	(working copy)
@@ -24,9 +24,7 @@
 import java.util.Locale;
 import java.util.Map;
 
-import org.apache.lucene.index.MergePolicy.MergeTrigger;
 
-
 /**
  * <p>This class implements a {@link MergePolicy} that tries
  * to merge segments into levels of exponentially
Index: lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java	(working copy)
@@ -36,8 +36,12 @@
   protected MergeScheduler() {
   }
 
-  /** Run the merges provided by {@link IndexWriter#getNextMerge()}. */
-  public abstract void merge(IndexWriter writer) throws IOException;
+  /** Run the merges provided by {@link IndexWriter#getNextMerge()}.
+   * @param writer the {@link IndexWriter} to obtain the merges from.
+   * @param trigger the {@link MergeTrigger} that caused this merge to happen
+   * @param newMergesFound <code>true</code> iff any new merges were found by the caller otherwise <code>false</code>
+   * */
+  public abstract void merge(IndexWriter writer, MergeTrigger trigger, boolean newMergesFound) throws IOException;
 
   /** Close this MergeScheduler. */
   @Override
Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java	(revision 1576367)
+++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -17,30 +17,11 @@
  * limitations under the License.
  */
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.FieldInfos.FieldNumbers;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.MergePolicy.MergeTrigger;
 import org.apache.lucene.index.MergeState.CheckAbort;
 import org.apache.lucene.index.NumericFieldUpdates.UpdatesIterator;
 import org.apache.lucene.search.Query;
@@ -58,6 +39,24 @@
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.ThreadInterruptedException;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
   An <code>IndexWriter</code> creates and maintains an index.
 
@@ -994,7 +993,7 @@
             try {
               // Give merge scheduler last chance to run, in case
               // any pending merges are waiting:
-              mergeScheduler.merge(this);
+              mergeScheduler.merge(this, MergeTrigger.CLOSING, false);
             } catch (ThreadInterruptedException tie) {
               // ignore any interruption, does not matter
               interrupted = true;
@@ -1830,17 +1829,18 @@
     }
 
     MergePolicy.MergeSpecification spec;
-
+    boolean newMergesFound = false;
     synchronized(this) {
       spec = mergePolicy.findForcedDeletesMerges(segmentInfos);
-      if (spec != null) {
+      newMergesFound = spec != null;
+      if (newMergesFound) {
         final int numMerges = spec.merges.size();
         for(int i=0;i<numMerges;i++)
           registerMerge(spec.merges.get(i));
       }
     }
 
-    mergeScheduler.merge(this);
+    mergeScheduler.merge(this, MergeTrigger.EXPLICIT, newMergesFound);
 
     if (spec != null && doWait) {
       final int numMerges = spec.merges.size();
@@ -1932,29 +1932,30 @@
 
   private final void maybeMerge(MergeTrigger trigger, int maxNumSegments) throws IOException {
     ensureOpen(false);
-    updatePendingMerges(trigger, maxNumSegments);
-    mergeScheduler.merge(this);
+    boolean newMergesFound = updatePendingMerges(trigger, maxNumSegments);
+    mergeScheduler.merge(this, trigger, newMergesFound);
   }
 
-  private synchronized void updatePendingMerges(MergeTrigger trigger, int maxNumSegments)
+  private synchronized boolean updatePendingMerges(MergeTrigger trigger, int maxNumSegments)
     throws IOException {
     assert maxNumSegments == -1 || maxNumSegments > 0;
     assert trigger != null;
     if (stopMerges) {
-      return;
+      return false;
     }
 
     // Do not start new merges if we've hit OOME
     if (hitOOM) {
-      return;
+      return false;
     }
-
+    boolean newMergesFound = false;
     final MergePolicy.MergeSpecification spec;
     if (maxNumSegments != UNBOUNDED_MAX_MERGE_SEGMENTS) {
       assert trigger == MergeTrigger.EXPLICIT || trigger == MergeTrigger.MERGE_FINISHED :
         "Expected EXPLICT or MERGE_FINISHED as trigger even with maxNumSegments set but was: " + trigger.name();
       spec = mergePolicy.findForcedMerges(segmentInfos, maxNumSegments, Collections.unmodifiableMap(segmentsToMerge));
-      if (spec != null) {
+      newMergesFound = spec != null;
+      if (newMergesFound) {
         final int numMerges = spec.merges.size();
         for(int i=0;i<numMerges;i++) {
           final MergePolicy.OneMerge merge = spec.merges.get(i);
@@ -1964,13 +1965,14 @@
     } else {
       spec = mergePolicy.findMerges(trigger, segmentInfos);
     }
-
-    if (spec != null) {
+    newMergesFound = spec != null;
+    if (newMergesFound) {
       final int numMerges = spec.merges.size();
       for(int i=0;i<numMerges;i++) {
         registerMerge(spec.merges.get(i));
       }
     }
+    return newMergesFound;
   }
 
   /** Expert: to be used by a {@link MergePolicy} to avoid
Index: solr/test-framework/src/java/org/apache/solr/util/RandomMergePolicy.java
===================================================================
--- solr/test-framework/src/java/org/apache/solr/util/RandomMergePolicy.java	(revision 1576367)
+++ solr/test-framework/src/java/org/apache/solr/util/RandomMergePolicy.java	(working copy)
@@ -18,7 +18,6 @@
 package org.apache.solr.util;
 
 import org.apache.lucene.index.*;
-import org.apache.lucene.index.MergePolicy.MergeSpecification;
 import org.apache.lucene.util.LuceneTestCase;
 
 import org.slf4j.Logger;