docs/attachments/LUCENE-2345/LUCENE-2345_3.0.plugins.patch

Index: src/java/org/apache/lucene/index/SegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentReader.java	(revision 927380)
+++ src/java/org/apache/lucene/index/SegmentReader.java	(working copy)
@@ -58,6 +58,9 @@
   private boolean normsDirty = false;
   private int pendingDeleteCount;
 
+  // The registered plugins
+  private Map<String, SegmentPlugin> plugins = new HashMap<String, SegmentPlugin>();
+
   private boolean rollbackHasChanges = false;
   private boolean rollbackDeletedDocsDirty = false;
   private boolean rollbackNormsDirty = false;
@@ -290,6 +293,13 @@
   }
 
   /**
+   * Get a plugin by name.
+   */
+  public SegmentPlugin getPlugin(String name) {
+    return plugins.get(name);
+  }
+
+  /**
    * Sets the initial value 
    */
   private class FieldsReaderLocal extends CloseableThreadLocal<FieldsReader> {
@@ -558,6 +568,14 @@
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
+  public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor, Map<String, SegmentPluginFactory> plugins) throws CorruptIndexException, IOException {
+    return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor, plugins);
+  }
+
+  /**
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
   public static SegmentReader get(boolean readOnly,
                                   Directory dir,
                                   SegmentInfo si,
@@ -565,6 +583,22 @@
                                   boolean doOpenStores,
                                   int termInfosIndexDivisor)
     throws CorruptIndexException, IOException {
+    return get(readOnly, dir, si, readBufferSize, doOpenStores, termInfosIndexDivisor, null);
+  }
+
+
+  /**
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static SegmentReader get(boolean readOnly,
+                                  Directory dir,
+                                  SegmentInfo si,
+                                  int readBufferSize,
+                                  boolean doOpenStores,
+                                  int termInfosIndexDivisor,
+                                  Map<String, SegmentPluginFactory> plugins)
+    throws CorruptIndexException, IOException {
     SegmentReader instance = readOnly ? new ReadOnlySegmentReader() : new SegmentReader();
     instance.readOnly = readOnly;
     instance.si = si;
@@ -579,6 +613,17 @@
       }
       instance.loadDeletedDocs();
       instance.openNorms(instance.core.cfsDir, readBufferSize);
+
+      // Initialize plugins
+      if (plugins != null) {
+        for (Map.Entry<String, SegmentPluginFactory> entry : plugins.entrySet()) {
+          final SegmentPlugin plugin = entry.getValue().open(instance);
+          if (plugin != null) {
+            instance.plugins.put(entry.getKey(), plugin);
+          }
+        }
+      }
+
       success = true;
     } finally {
 
@@ -730,6 +775,14 @@
       // any norms that have changed:
       clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), readBufferSize);
 
+      // Reopen plugins
+      for (Map.Entry<String, SegmentPlugin> entry : plugins.entrySet()) {
+        final SegmentPlugin plugin = entry.getValue().reopen(clone);
+        if (plugin != null) {
+          clone.plugins.put(entry.getKey(), entry.getValue().reopen(clone));
+        }
+      }
+
       success = true;
     } finally {
       if (!success) {
@@ -795,6 +848,11 @@
     if (core != null) {
       core.decRef();
     }
+
+    // Close the plugins
+    for (SegmentPlugin plugin : plugins.values()) {
+      plugin.close();
+    }
   }
 
   static boolean hasDeletions(SegmentInfo si) throws IOException {
Index: src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
===================================================================
--- src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java	(revision 927380)
+++ src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java	(working copy)
@@ -23,13 +23,13 @@
 import java.util.Map;
 
 class ReadOnlyDirectoryReader extends DirectoryReader {
-  ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor) throws IOException {
-    super(directory, sis, deletionPolicy, true, termInfosIndexDivisor);
+  ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws IOException {
+    super(directory, sis, deletionPolicy, true, termInfosIndexDivisor, plugins);
   }
 
   ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,  Map<String,byte[]> oldNormsCache, boolean doClone,
-                          int termInfosIndexDivisor) throws IOException {
-    super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor);
+                          int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws IOException {
+    super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor, plugins);
   }
   
   ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
Index: src/java/org/apache/lucene/index/SegmentPluginFactory.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentPluginFactory.java	(revision 0)
+++ src/java/org/apache/lucene/index/SegmentPluginFactory.java	(revision 0)
@@ -0,0 +1,15 @@
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+/**
+ * Factory for creating a plugin for SegmentReader.
+ */
+public abstract class SegmentPluginFactory {
+
+  /**
+   * Initialize a segment plugin for <code>reader</code>.
+   */
+  public abstract SegmentPlugin open(SegmentReader reader) throws IOException;
+
+}
\ No newline at end of file

Property changes on: src/java/org/apache/lucene/index/SegmentPluginFactory.java
___________________________________________________________________
Added: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/DirectoryReader.java
===================================================================
--- src/java/org/apache/lucene/index/DirectoryReader.java	(revision 927380)
+++ src/java/org/apache/lucene/index/DirectoryReader.java	(working copy)
@@ -63,25 +63,32 @@
   private int maxDoc = 0;
   private int numDocs = -1;
   private boolean hasDeletions = false;
+  private Map<String, SegmentPluginFactory> plugins; // factory for initializing SegmentReaders
 
   static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
                           final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+    return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
+  }
+
+  static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
+                          final int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws CorruptIndexException, IOException {
     return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
       @Override
       protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
         SegmentInfos infos = new SegmentInfos();
         infos.read(directory, segmentFileName);
         if (readOnly)
-          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, plugins);
         else
-          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, plugins);
       }
     }.run(commit);
   }
 
   /** Construct reading the named set of readers. */
-  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws IOException {
     this.directory = directory;
+    this.plugins = plugins;
     this.readOnly = readOnly;
     this.segmentInfos = sis;
     this.deletionPolicy = deletionPolicy;
@@ -102,7 +109,7 @@
     for (int i = sis.size()-1; i >= 0; i--) {
       boolean success = false;
       try {
-        readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor);
+        readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor, plugins);
         success = true;
       } finally {
         if (!success) {
@@ -125,6 +132,7 @@
   DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
     this.directory = writer.getDirectory();
     this.readOnly = true;
+    this.plugins = writer.getSegmentPlugins();
     this.segmentInfos = infos;
     segmentInfosStart = (SegmentInfos) infos.clone();
     this.termInfosIndexDivisor = termInfosIndexDivisor;
@@ -178,9 +186,10 @@
 
   /** This constructor is only used for {@link #reopen()} */
   DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
-                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException {
+                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws IOException {
     this.directory = directory;
     this.readOnly = readOnly;
+    this.plugins = plugins;
     this.segmentInfos = infos;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
     if (!readOnly) {
@@ -226,7 +235,7 @@
           assert !doClone;
 
           // this is a new reader; in case we hit an exception we can close it safely
-          newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor);
+          newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor, plugins);
         } else {
           newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly);
         }
@@ -438,9 +447,9 @@
   private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
     DirectoryReader reader;
     if (openReadOnly) {
-      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, plugins);
     } else {
-      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, plugins);
     }
     return reader;
   }
Index: src/java/org/apache/lucene/index/SegmentPlugin.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentPlugin.java	(revision 0)
+++ src/java/org/apache/lucene/index/SegmentPlugin.java	(revision 0)
@@ -0,0 +1,24 @@
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+/**
+ * Base class for plugins for SegmentReader.
+ */
+public abstract class SegmentPlugin {
+
+
+  /**
+   * Reopen this SegmentPlugin for <code>newReader</code>.
+   */
+  public abstract SegmentPlugin reopen(SegmentReader newReader) throws IOException;
+
+
+  /**
+   * Close this SegmentPlugin.
+   */
+  public abstract void close() throws IOException;
+
+
+}
+

Property changes on: src/java/org/apache/lucene/index/SegmentPlugin.java
___________________________________________________________________
Added: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/IndexReader.java	(revision 927380)
+++ src/java/org/apache/lucene/index/IndexReader.java	(working copy)
@@ -263,6 +263,36 @@
   }
 
   /** Expert: returns an IndexReader reading the index in
+   *  the given Directory, with a custom {@link
+   *  IndexDeletionPolicy}.  You should pass readOnly=true,
+   *  since it gives much better concurrent performance,
+   *  unless you intend to do write operations (delete
+   *  documents or change norms) with the reader.
+   * @param directory the index directory
+   * @param deletionPolicy a custom deletion policy (only used
+   *  if you use this reader to perform deletes or to set
+   *  norms); see {@link IndexWriter} for details.
+   * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
+   * @param termInfosIndexDivisor Subsamples which indexed
+   *  terms are loaded into RAM. This has the same effect as {@link
+   *  IndexWriter#setTermIndexInterval} except that setting
+   *  must be done at indexing time while this setting can be
+   *  set per reader.  When set to N, then one in every
+   *  N*termIndexInterval terms in the index is loaded into
+   *  memory.  By setting this to a value > 1 you can reduce
+   *  memory usage, at the expense of higher latency when
+   *  loading a TermInfo.  The default value is 1.  Set this
+   *  to -1 to skip loading the terms index entirely.
+   * @param plugins map of plugins to initialize for SegmentReaders
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws CorruptIndexException, IOException {
+    return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, plugins);
+  }
+
+
+  /** Expert: returns an IndexReader reading the index in
    *  the given Directory, using a specific commit and with
    *  a custom {@link IndexDeletionPolicy}.  You should pass
    *  readOnly=true, since it gives much better concurrent
@@ -312,10 +342,46 @@
     return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor);
   }
 
+  /** Expert: returns an IndexReader reading the index in
+   *  the given Directory, using a specific commit and with
+   *  a custom {@link IndexDeletionPolicy}.  You should pass
+   *  readOnly=true, since it gives much better concurrent
+   *  performance, unless you intend to do write operations
+   *  (delete documents or change norms) with the reader.
+   * @param commit the specific {@link IndexCommit} to open;
+   * see {@link IndexReader#listCommits} to list all commits
+   * in a directory
+   * @param deletionPolicy a custom deletion policy (only used
+   *  if you use this reader to perform deletes or to set
+   *  norms); see {@link IndexWriter} for details.
+   * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
+   * @param termInfosIndexDivisor Subsamples which indexed
+   *  terms are loaded into RAM. This has the same effect as {@link
+   *  IndexWriter#setTermIndexInterval} except that setting
+   *  must be done at indexing time while this setting can be
+   *  set per reader.  When set to N, then one in every
+   *  N*termIndexInterval terms in the index is loaded into
+   *  memory.  By setting this to a value > 1 you can reduce
+   *  memory usage, at the expense of higher latency when
+   *  loading a TermInfo.  The default value is 1.  Set this
+   *  to -1 to skip loading the terms index entirely.
+   * @param plugins map of plugins to initialize for segments
+   * @throws CorruptIndexException if the index is corrupt
+   * @throws IOException if there is a low-level IO error
+   */
+  public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws CorruptIndexException, IOException {
+    return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, plugins);
+  }
+
+
   private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
-    return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor);
+    return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
   }
 
+  private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, final Map<String, SegmentPluginFactory> plugins) throws CorruptIndexException, IOException {
+    return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, plugins);
+  }
+
   /**
    * Refreshes an IndexReader if the index has changed since this instance 
    * was (re)opened. 
Index: src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexWriter.java	(revision 927380)
+++ src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -265,6 +265,7 @@
 
   private DocumentsWriter docWriter;
   private IndexFileDeleter deleter;
+  private Map<String, SegmentPluginFactory> segmentPlugins = java.util.Collections.emptyMap();
 
   private Set<SegmentInfo> segmentsToOptimize = new HashSet<SegmentInfo>();           // used by optimize to note those needing optimization
 
@@ -606,7 +607,7 @@
         // TODO: we may want to avoid doing this while
         // synchronized
         // Returns a ref, which we xfer to readerMap:
-        sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor);
+        sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor, segmentPlugins);
         readerMap.put(info, sr);
       } else {
         if (doOpenStores) {
@@ -4850,6 +4851,21 @@
     return mergedSegmentWarmer;
   }
 
+  /**
+   * Set the plugins to initialize for a newly opened segment.
+   */
+  public void setSegmentPlugins(Map<String, SegmentPluginFactory> plugins) {
+    segmentPlugins = plugins;
+  }
+
+  /**
+   * Get the plugins for newly opened segments
+   */
+  public Map<String, SegmentPluginFactory> getSegmentPlugins() {
+    return segmentPlugins;
+  }
+
+
   private void handleOOM(OutOfMemoryError oom, String location) {
     if (infoStream != null) {
       message("hit OutOfMemoryError inside " + location);