docs/attachments/LUCENE-3397/LUCENE-3397-more.patch

Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
===================================================================
--- lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java	(revision 950008)
+++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java	(revision )
@@ -311,6 +311,12 @@
           tokens[p++].copyTo(reusableToken);
           return true;
         }
+
+        @Override
+        public void reset() throws IOException {
+          super.reset();
+          this.p = 0;
+        }
       };
       return ts;
     }
Index: modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
===================================================================
--- modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java	(revision 1040463)
+++ modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java	(revision )
@@ -28,6 +28,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.StopAnalyzer;
 import org.apache.lucene.analysis.core.StopFilter;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -220,10 +221,6 @@
    */
   @Override
   public TokenStream tokenStream(String fieldName, Reader reader) {
-    if (reader instanceof FastStringReader) { // fast path
-      return tokenStream(fieldName, ((FastStringReader)reader).getString());
-    }
-    
     try {
       String text = toString(reader);
       return tokenStream(fieldName, text);
@@ -290,6 +287,10 @@
    * @throws IOException if an I/O error occurs while reading the stream
    */
   private static String toString(Reader input) throws IOException {
+    if (input instanceof FastStringReader) { // fast path
+      return ((FastStringReader) input).getString();
+    }
+
     try {
       int len = 256;
       char[] buffer = new char[len];
@@ -324,9 +325,9 @@
    * The work horse; performance isn't fantastic, but it's not nearly as bad
    * as one might think - kudos to the Sun regex developers.
    */
-  private static final class PatternTokenizer extends TokenStream {
+  private static final class PatternTokenizer extends Tokenizer {
     
-    private final String str;
+    private String str;
     private final boolean toLowerCase;
     private Matcher matcher;
     private int pos = 0;
@@ -372,10 +373,22 @@
       // set final offset
       final int finalOffset = str.length();
     	this.offsetAtt.setOffset(finalOffset, finalOffset);
-    }    
+    }
+
+    @Override
+    public void reset(Reader input) throws IOException {
+      super.reset(input);
+      this.str = PatternAnalyzer.toString(input);
-  } 
-  
+    }
+
+    @Override
+    public void reset() throws IOException {
+      super.reset();
+      this.pos = 0;
+    }
+  }
   
+  
   ///////////////////////////////////////////////////////////////////////////////
   // Nested classes:
   ///////////////////////////////////////////////////////////////////////////////
@@ -383,9 +396,9 @@
    * Special-case class for best performance in common cases; this class is
    * otherwise unnecessary.
    */
-  private static final class FastStringTokenizer extends TokenStream {
+  private static final class FastStringTokenizer extends Tokenizer {
     
-    private final String str;
+    private String str;
     private int pos;
     private final boolean isLetter;
     private final boolean toLowerCase;
@@ -464,10 +477,20 @@
     private boolean isStopWord(String text) {
       return stopWords != null && stopWords.contains(text);
     }
-    
+
+    @Override
+    public void reset(Reader input) throws IOException {
+      this.str = PatternAnalyzer.toString(input);
-  }
+    }
 
+    @Override
+    public void reset() throws IOException {
+      super.reset();
+      this.pos = 0;
+    }
+  }
-  
+
+  
   ///////////////////////////////////////////////////////////////////////////////
   // Nested classes:
   ///////////////////////////////////////////////////////////////////////////////
Index: lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java	(revision 1085374)
+++ lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java	(revision )
@@ -492,7 +492,6 @@
 
   final class PayloadFilter extends TokenFilter {
     String fieldName;
-    int numSeen = 0;
     Set<String> entities = new HashSet<String>();
     Set<String> nopayload = new HashSet<String>();
     int pos;
@@ -530,7 +529,13 @@
       }
       return false;
     }
+
+    @Override
+    public void reset() throws IOException {
+      super.reset();
+      this.pos = 0;
-  }
+    }
+  }
   
   public final class TestPayloadAnalyzer extends Analyzer {
 
Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
===================================================================
--- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java	(revision 1144196)
+++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java	(revision )
@@ -447,6 +447,12 @@
             }
             return input.incrementToken();
           }
+
+          @Override
+          public void reset() throws IOException {
+            super.reset();
+            this.count = 0;
+          }
         };
       }