docs/attachments/LUCENE-3786/LUCENE-3786-3x-nocommit.patch

Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java
===================================================================
--- lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java	(revision 0)
+++ lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java	(working copy)
@@ -0,0 +1,88 @@
+package org.apache.lucene.facet.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.search.SearcherTaxoManager.TaxonomyTimestamp;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
+public class SearcherTaxoManagerTest extends LuceneTestCase {
+
+	private void touchIndex(Directory searchDir, Directory taxoDir) throws IOException {
+		String timeString = Long.toString(System.currentTimeMillis());
+		TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+		tw.addCategory(new CategoryPath("a/"+Integer.toString(tw.getSize()), '/'));
+		Map<String, String> commitData = new HashMap<String, String>();
+		commitData.put(TaxonomyTimestamp.INDEX_UPDATE_TIME, timeString);
+		commitData.put(TaxonomyTimestamp.INDEX_CREATE_TIME, timeString);
+		tw.commit(commitData);
+		tw.close();
+
+		IndexWriter writer = new IndexWriter(searchDir, new IndexWriterConfig(
+				TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+		writer.addDocument(new Document());
+		commitData.clear();
+		commitData.put(TaxonomyTimestamp.TAXO_UPDATE_TIME, timeString);
+		commitData.put(TaxonomyTimestamp.TAXO_CREATE_TIME, timeString);
+		writer.commit(commitData);
+		writer.close();
+	}
+
+	private void touchIndexNonSynched(Directory dir, Directory taxoDir) throws IOException {
+		TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+		tw.addCategory(new CategoryPath("a/"+Integer.toString(tw.getSize()), '/'));
+		tw.commit();
+		tw.close();
+
+		IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
+				TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+		writer.addDocument(new Document());
+		long time = System.nanoTime();
+		// TODO: read createTime from the taxoDir following the commit
+		writer.commit(new TaxonomyTimestamp(-1, time + 1000).toSearchCommitData());
+		writer.close();
+	}
+
+	@Test(expected=IOException.class)
+	public void testCtorWithNonSynchedIndexes() throws Exception {
+		Directory dir = newDirectory();
+		Directory taxoDir = newDirectory();
+		touchIndexNonSynched(dir, taxoDir);
+		// assert just to avoid the "object not used" compile warning...
+		assertNotNull(new SearcherTaxoManager(dir, taxoDir, null));
+		fail("should not have reached here - SearcherTaxoManager should fail if search and taxonomy indexes are out of sync");
+	}
+
+}

Property changes on: lucene/contrib/facet/src/test/org/apache/lucene/facet/search/SearcherTaxoManagerTest.java
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
Added: svn:eol-style
## -0,0 +1 ##
+native
Index: lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
===================================================================
--- lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java	(revision 1326275)
+++ lucene/contrib/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java	(working copy)
@@ -200,5 +200,29 @@
     
     dir.close();
   }
+
+  @Test
+  public void testTimestamps() throws Exception {
+    // ensures that an application cannot override the create/update time stamps.
+    Directory dir = newDirectory();
+    
+    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, new NoOpCache());
+    taxoWriter.addCategory(new CategoryPath("a"));
+    HashMap<String, String> commitUserData = new HashMap<String, String>();
+    // set these properties -- they should not be committed like that !
+    commitUserData.put(DirectoryTaxonomyWriter.INDEX_CREATE_TIME, "create");
+    commitUserData.put(DirectoryTaxonomyWriter.INDEX_UPDATE_TIME, "update");
+    taxoWriter.commit(commitUserData);
+    taxoWriter.close();
+    
+    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
+    Map<String, String> commitData = taxoReader.getCommitUserData();
+    taxoReader.close();
+    dir.close();
+    
+    // parse the timestamp properties -- they should be parseable (i.e. not overridden above).
+    Long.parseLong(commitData.get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME));
+    Long.parseLong(commitData.get(DirectoryTaxonomyWriter.INDEX_UPDATE_TIME));
+  }
   
 }
Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java
===================================================================
--- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java	(revision 0)
+++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java	(working copy)
@@ -0,0 +1,286 @@
+package org.apache.lucene.facet.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.lucene.facet.search.SearcherTaxoManager.SearcherTaxoPair;
+import org.apache.lucene.facet.taxonomy.InconsistentTaxonomyException;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ReferenceManager;
+import org.apache.lucene.search.SearcherFactory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+
+public class SearcherTaxoManager extends ReferenceManager<SearcherTaxoPair> {
+  
+  public static class SearcherTaxoPair {
+    public final IndexSearcher searcher;
+    public final TaxonomyReader taxoReader;
+
+    public SearcherTaxoPair(IndexSearcher is, TaxonomyReader tr) {
+      searcher = is;
+      taxoReader = tr;
+    }
+  }
+
+  private static final Logger logger = Logger.getLogger(SearcherTaxoManager.class.getName());
+  
+  /** Holds the create and last update times of a taxonomy index. */
+  public static final class TaxonomyTimestamp {
+    
+    /**
+     * The property name that exists in the commit data of the search index, and
+     * denotes the taxonomy index update time.
+     */
+    public static final String TAXO_UPDATE_TIME = "TAXONOMY_INDEX_UPDATE_TIME";
+
+    /**
+     * The property name that exists in the commit data of the search index, and
+     * denotes the taxonomy index creation time.
+     */
+    public static final String TAXO_CREATE_TIME = "TAXONOMY_INDEX_CREATE_TIME";
+
+    public final long createTime;
+    public final long lastUpdateTime;
+    
+    /** Creates a {@link TaxonomyTimestamp} from the commit data of a taxonomy index. */
+    public static TaxonomyTimestamp fromTaxonomyCommitData(Map<String, String> commitData) {
+      long createTime = -1, updateTime = -1;
+      if (commitData != null) {
+        String createTimeProp = commitData.get(DirectoryTaxonomyWriter.INDEX_CREATE_TIME);
+        if (createTimeProp != null) {
+          createTime = Long.parseLong(createTimeProp);
+        }
+        
+        String updateTimeProp = commitData.get(DirectoryTaxonomyWriter.INDEX_UPDATE_TIME);
+        if (updateTimeProp != null) {
+          updateTime = Long.parseLong(updateTimeProp);
+        }
+      }
+      return new TaxonomyTimestamp(createTime, updateTime);
+    }
+
+    /** Creates a {@link TaxonomyTimestamp} from the commit data of a search index. */
+    public static TaxonomyTimestamp fromSearchCommitData(Map<String, String> commitData) {
+      long createTime = -1, updateTime = -1;
+      if (commitData != null) {
+        String createTimeProp = commitData.get(TAXO_CREATE_TIME);
+        if (createTimeProp != null) {
+          createTime = Long.parseLong(createTimeProp);
+        }
+        
+        String updateTimeProp = commitData.get(TAXO_UPDATE_TIME);
+        if (updateTimeProp != null) {
+          updateTime = Long.parseLong(updateTimeProp);
+        }
+      }
+      return new TaxonomyTimestamp(createTime, updateTime);
+    }
+
+    public TaxonomyTimestamp(long createTime, long lastUpdateTime) {
+      this.createTime = createTime;
+      this.lastUpdateTime = lastUpdateTime;
+    }
+    
+    public Map<String, String> toSearchCommitData() {
+      return new HashMap<String, String>() {{
+        put(TAXO_CREATE_TIME, Long.toString(createTime));
+        put(TAXO_UPDATE_TIME, Long.toString(lastUpdateTime));
+      }};
+    }
+    
+    @Override
+    public String toString() {
+      return "createTime=" + createTime + " lastUpdateTime=" + lastUpdateTime;
+    }
+    
+  }
+
+  private final SearcherFactory searcherFactory;
+  private final Directory taxoDir;
+  
+  public SearcherTaxoManager(Directory indexDir, Directory taxoDir,
+      SearcherFactory searcherFactory) throws IOException {
+    if (indexDir == null || taxoDir == null) {
+      throw new IllegalArgumentException("indexReader, taxoReader and taxoFactory cannot be null !");
+    }
+
+    IndexReader indexReader = null;
+    TaxonomyReader taxoReader = null;
+    boolean success = false;
+    try {
+      indexReader = IndexReader.open(taxoDir);
+      taxoReader = new DirectoryTaxonomyReader(taxoDir);
+
+      // validate that the IndexReader and TaxoReader are in sync
+      TaxonomyTimestamp searchTaxoToken = getTimestampToken(indexReader);
+      TaxonomyTimestamp taxoIndexToken = getTimestampToken(taxoReader);
+      if (searchTaxoToken.createTime != taxoIndexToken.createTime
+          || searchTaxoToken.lastUpdateTime > taxoIndexToken.lastUpdateTime) {
+        String msg = "Search and Taxonomy indexes times do not match: searchIndex=["
+            + searchTaxoToken + "]; taxoIndex=[" + taxoIndexToken + "]";
+        if (logger.isLoggable(Level.FINEST)) {
+          logger.finest(msg);
+        }
+        throw new IOException(msg);
+      }
+
+      if (searcherFactory == null) {
+        searcherFactory = new SearcherFactory();
+      }
+      
+      this.searcherFactory = searcherFactory;
+      current = new SearcherTaxoPair(searcherFactory.newSearcher(indexReader), taxoReader);
+      this.taxoDir = taxoDir;
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(indexReader, taxoReader);
+      }
+    }
+  }
+  
+  @Override
+  protected void decRef(SearcherTaxoPair reference) throws IOException {
+    boolean success1 = false, success2 = false;
+    try {
+      reference.searcher.getIndexReader().decRef();
+      success1 = true;
+      reference.taxoReader.decRef();
+      success2 = true;
+    } finally {
+      if (!success1) {
+        reference.searcher.getIndexReader().incRef();
+      }
+      if (!success2) {
+        reference.taxoReader.incRef();
+      }
+    }
+  }
+
+  @Override
+  protected boolean tryIncRef(SearcherTaxoPair reference) {
+    if (reference.searcher.getIndexReader().tryIncRef()) {
+      reference.taxoReader.incRef();
+      return true;
+    }
+    return false;
+  }
+  
+  /** Returns the taxonomy index create/update timestamps that are registered on the search index. */
+  private TaxonomyTimestamp getTimestampToken(IndexReader reader) throws IOException {
+    return TaxonomyTimestamp.fromSearchCommitData(reader.getIndexCommit().getUserData());
+  }
+  
+  /** Returns the taxonomy index create/update timestamps that are registered on the taxonomy index. */
+  private TaxonomyTimestamp getTimestampToken(TaxonomyReader taxoReader) throws IOException {
+    return TaxonomyTimestamp.fromTaxonomyCommitData(taxoReader.getCommitUserData());
+  }
+
+  /**
+   * Returns a new {@link SearcherTaxoPair} if the search and taxonomy index are
+   * in sync, otherwise returns {@code null}.
+   */
+  private SearcherTaxoPair getNewPair(TaxonomyReader taxoReader,
+      IndexReader reader, TaxonomyTimestamp readerTaxoToken,
+      TaxonomyTimestamp taxoReaderToken) throws IOException {
+    SearcherTaxoPair pair = null;
+    try {
+      // search and taxonomy indexes are in the same epoch. Refresh taxoReader
+      // and return a new pair if appropriate
+      taxoReader.refresh();
+      
+      // The taxonomy wasn't recreated - make sure that its lastUpdate is >=
+      // indexReader's lastUpdate (otherwise it means that the indexReader
+      // contains categories that are not yet visible to the taxonomy reader).
+      taxoReaderToken = getTimestampToken(taxoReader);
+      if (readerTaxoToken.lastUpdateTime <= taxoReaderToken.lastUpdateTime) {
+        pair = new SearcherTaxoPair(searcherFactory.newSearcher(reader), taxoReader);
+      }
+    } catch (InconsistentTaxonomyException e) {
+      // The taxonomy cannot be refreshed -- could be that someone just
+      // committed a 'recreate' on the taxonomy - we cannot return a new pair.
+    }
+    return pair;
+  }
+  
+  @Override
+  protected SearcherTaxoPair refreshIfNeeded(SearcherTaxoPair referenceToRefresh) throws IOException {
+    IndexReader newIndexReader = IndexReader.openIfChanged(referenceToRefresh.searcher.getIndexReader());
+    if (newIndexReader == null) {
+      // search index was not updated, don't update the pair (even if the
+      // taxonomy index has been updated)
+      return null;
+    }
+
+    SearcherTaxoPair newPair = null;
+    TaxonomyTimestamp newReaderTaxoToken = null;
+    TaxonomyTimestamp taxoReaderToken = null;
+    TaxonomyReader newTaxoReader = null;
+    boolean success = false;
+    try {
+      newReaderTaxoToken = getTimestampToken(newIndexReader);
+      taxoReaderToken = getTimestampToken(referenceToRefresh.taxoReader);
+      if (newReaderTaxoToken.createTime == taxoReaderToken.createTime) {
+        newPair = getNewPair(referenceToRefresh.taxoReader,
+            newIndexReader, newReaderTaxoToken, taxoReaderToken);
+        if (newPair != null) {
+          // need to incRef() because we return a new instance of
+          // SearcherTaxoPair, but share the taxoReader instance, and
+          // ReferenceManager will soon 'release' us, so need to adjust the 
+          // reference count accordingly.
+          newPair.taxoReader.incRef();
+        }
+      } else {
+        // the taxonomy index and search index's createTime differs. Open a new
+        // TaxonomyReader and try to get a matching pair with it.
+        newTaxoReader = new DirectoryTaxonomyReader(taxoDir);
+        taxoReaderToken = getTimestampToken(newTaxoReader);
+        if (newReaderTaxoToken.createTime == taxoReaderToken.createTime) {
+          newPair = getNewPair(newTaxoReader, newIndexReader, newReaderTaxoToken, taxoReaderToken);
+        }
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(newIndexReader, newTaxoReader);
+      }
+    }
+    
+    if (newPair == null) {
+      if (logger.isLoggable(Level.FINEST)) {
+        logger.finest("Search and Taxonomy indexes times do not match: searchIndex=["
+                + newReaderTaxoToken + "]; taxoIndex=[" + taxoReaderToken + "]");
+      }
+      // the taxonomy and search indexes do not match, close what was opened 
+      IOUtils.close(newTaxoReader, newIndexReader);
+    }
+    
+    return newPair;
+  }
+
+}

Property changes on: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/SearcherTaxoManager.java
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
Added: svn:eol-style
## -0,0 +1 ##
+native
Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java
===================================================================
--- lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java	(revision 0)
+++ lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java	(working copy)
@@ -0,0 +1,56 @@
+package org.apache.lucene.facet.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.store.Directory;
+
+/**
+ * Factory class used by {@link SearcherTaxoManager} to create new
+ * TaxonomyReaders. {@link DirectoryTaxonomyReaderFactory} returns
+ * {@link DirectoryTaxonomyReader}s on a given {@link Directory}.
+ * 
+ * @lucene.experimental
+ */
+public interface TaxonomyReaderFactory {
+
+  /**
+   * A {@link TaxonomyReaderFactory} which returns
+   * {@link DirectoryTaxonomyReader}s on the given {@link Directory}.
+   */
+  public static class DirectoryTaxonomyReaderFactory implements TaxonomyReaderFactory {
+
+    private final Directory taxoDir;
+    
+    public DirectoryTaxonomyReaderFactory(Directory taxoDir) {
+      this.taxoDir = taxoDir;
+    }
+    
+    public TaxonomyReader newTaxonomyReader() throws IOException {
+      return new DirectoryTaxonomyReader(taxoDir);
+    }
+    
+  }
+  
+  /** Returns a new TaxonomyReader. */
+  public TaxonomyReader newTaxonomyReader() throws IOException;
+  
+}

Property changes on: lucene/contrib/facet/src/java/org/apache/lucene/facet/search/TaxonomyReaderFactory.java
___________________________________________________________________
Added: svn:executable
## -0,0 +1 ##
+*
Added: svn:eol-style
## -0,0 +1 ##
+native
Index: lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
===================================================================
--- lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java	(revision 1326275)
+++ lucene/contrib/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java	(working copy)
@@ -83,14 +83,21 @@
 public class DirectoryTaxonomyWriter implements TaxonomyWriter {
 
   /**
-   * Property name of user commit data that contains the creation time of a
-   * taxonomy index.
+   * Property name of user commit data that contains the creation time (in nano
+   * seconds) of a taxonomy index.
    * <p>
    * Applications should not use this property in their commit data because it
    * will be overridden by this taxonomy writer.
    */
   public static final String INDEX_CREATE_TIME = "index.create.time";
 
+  /**
+   * Property name of user commit data that contains the update time (in nano
+   * seconds) of a taxonomy index. Set to the time that {@link #commit} is
+   * called.
+   */
+  public static final String INDEX_UPDATE_TIME = "index.update.time";
+
   private IndexWriter indexWriter;
   private int nextID;
   private char delimiter = Consts.DEFAULT_DELIMITER;
@@ -649,6 +656,9 @@
     if (userData != null) {
       m.putAll(userData);
     }
+    // application is not allowed to override these properties, therefore set
+    // them last.
+    m.put(INDEX_UPDATE_TIME, Long.toString(System.nanoTime()));
     if (createTime != null) {
       m.put(INDEX_CREATE_TIME, createTime);
     }