diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
index d8fbd15..6978b77 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
@@ -100,7 +100,14 @@
     }
   }
 
-  public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], Integer finalOffset) throws IOException {
+  // offsetsAreCorrect also validates:
+  //   - graph offsets are correct (all tokens leaving from
+  //     pos X have the same startOffset; all tokens
+  //     arriving to pos Y have the same endOffset)
+  //   - offsets only move forwards (startOffset >=
+  //     lastStartOffset)
+  public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], Integer finalOffset,
+                                               boolean offsetsAreCorrect) throws IOException {
     assertNotNull(output);
     CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class);
     
@@ -137,6 +144,7 @@
 
     ts.reset();
     int pos = -1;
+    int lastStartOffset = 0;
     for (int i = 0; i < output.length; i++) {
       // extra safety to enforce, that the state is not preserved and also assign bogus values
       ts.clearAttributes();
@@ -176,7 +184,12 @@
                      endOffset <= finalOffset.intValue());
         }
 
-        if (posLengthAtt != null && posIncrAtt != null) {
+        if (offsetsAreCorrect) {
+          assertTrue("offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset, offsetAtt.startOffset() >= lastStartOffset);
+          lastStartOffset = offsetAtt.startOffset();
+        }
+
+        if (offsetsAreCorrect && posLengthAtt != null && posIncrAtt != null) {
           // Validate offset consistency in the graph, ie
           // all tokens leaving from a certain pos have the
           // same startOffset, and all tokens arriving to a
@@ -194,7 +207,7 @@
             // We've seen a token leaving from this position
             // before; verify the startOffset is the same:
             //System.out.println("  + vs " + pos + " -> " + startOffset);
-            assertEquals(posToStartOffset.get(pos).intValue(), startOffset);
+            assertEquals("pos=" + pos + " posLen=" + posLength + " token=" + termAtt, posToStartOffset.get(pos).intValue(), startOffset);
           }
 
           final int endPos = pos + posLength;
@@ -207,7 +220,7 @@
             // We've seen a token arriving to this position
             // before; verify the endOffset is the same:
             //System.out.println("  + ve " + endPos + " -> " + endOffset);
-            assertEquals(posToEndOffset.get(endPos).intValue(), endOffset);
+            assertEquals("pos=" + pos + " posLen=" + posLength + " token=" + termAtt, posToEndOffset.get(endPos).intValue(), endOffset);
           }
         }
       }
@@ -222,7 +235,7 @@
         assertTrue("posLength must be >= 1", posLengthAtt.getPositionLength() >= 1);
       }
     }
-    assertFalse("TokenStream has more tokens than expected", ts.incrementToken());
+    assertFalse("TokenStream has more tokens than expected (expected count=" + output.length + ")", ts.incrementToken());
     ts.end();
     if (finalOffset != null) {
       assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset());
@@ -233,6 +246,10 @@
     ts.close();
   }
   
+  public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], Integer finalOffset) throws IOException {
+    assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, posLengths, finalOffset, true);
+  }
+
   public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], Integer finalOffset) throws IOException {
     assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, null, finalOffset);
   }
@@ -280,6 +297,10 @@
   public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[]) throws IOException {
     assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, posLengths, input.length());
   }
+
+  public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], int posLengths[], boolean offsetsAreCorrect) throws IOException {
+    assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, posLengths, input.length(), offsetsAreCorrect);
+  }
   
   public static void assertAnalyzesTo(Analyzer a, String input, String[] output) throws IOException {
     assertAnalyzesTo(a, input, output, null, null, null, null, null);
@@ -342,12 +363,12 @@
   
   /** utility method for blasting tokenstreams with data to make sure they don't do anything crazy */
   public static void checkRandomData(Random random, Analyzer a, int iterations) throws IOException {
-    checkRandomData(random, a, iterations, 20, false);
+    checkRandomData(random, a, iterations, 20, false, true);
   }
-  
+
   /** utility method for blasting tokenstreams with data to make sure they don't do anything crazy */
   public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength) throws IOException {
-    checkRandomData(random, a, iterations, maxWordLength, false);
+    checkRandomData(random, a, iterations, maxWordLength, false, true);
   }
   
   /** 
@@ -355,43 +376,63 @@
    * @param simple true if only ascii strings will be used (try to avoid)
    */
   public static void checkRandomData(Random random, Analyzer a, int iterations, boolean simple) throws IOException {
-    checkRandomData(random, a, iterations, 20, simple);
+    checkRandomData(random, a, iterations, 20, simple, true);
   }
   
   static class AnalysisThread extends Thread {
     final int iterations;
     final int maxWordLength;
-    final Random random;
+    final long seed;
     final Analyzer a;
+    final boolean useCharFilter;
     final boolean simple;
+    final boolean offsetsAreCorrect;
+
+    // NOTE: not volatile because we don't want the tests to
+    // add memory barriers (ie alter how threads
+    // interact)... so this is just "best effort":
+    public boolean failed;
     
-    AnalysisThread(Random random, Analyzer a, int iterations, int maxWordLength, boolean simple) {
-      this.random = random;
+    AnalysisThread(long seed, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect) {
+      this.seed = seed;
       this.a = a;
       this.iterations = iterations;
       this.maxWordLength = maxWordLength;
+      this.useCharFilter = useCharFilter;
       this.simple = simple;
+      this.offsetsAreCorrect = offsetsAreCorrect;
     }
     
     @Override
     public void run() {
+      boolean success = false;
       try {
         // see the part in checkRandomData where it replays the same text again
         // to verify reproducability/reuse: hopefully this would catch thread hazards.
-        checkRandomData(random, a, iterations, maxWordLength, random.nextBoolean(), simple);
+        checkRandomData(new Random(seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect);
+        success = true;
       } catch (IOException e) {
         Rethrow.rethrow(e);
+      } finally {
+        failed = !success;
       }
     }
   };
   
   public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean simple) throws IOException {
-    checkRandomData(random, a, iterations, maxWordLength, random.nextBoolean(), simple);
-    // now test with multiple threads
+    checkRandomData(random, a, iterations, maxWordLength, simple, true);
+  }
+
+  public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean simple, boolean offsetsAreCorrect) throws IOException {
+    long seed = random.nextLong();
+    boolean useCharFilter = random.nextBoolean();
+    checkRandomData(new Random(seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect);
+    // now test with multiple threads: note we do the EXACT same thing we did before in each thread,
+    // so this should only really fail from another thread if its an actual thread problem
     int numThreads = _TestUtil.nextInt(random, 4, 8);
-    Thread threads[] = new Thread[numThreads];
+    AnalysisThread threads[] = new AnalysisThread[numThreads];
     for (int i = 0; i < threads.length; i++) {
-      threads[i] = new AnalysisThread(new Random(random.nextLong()), a, iterations, maxWordLength, simple);
+      threads[i] = new AnalysisThread(seed, a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect);
     }
     for (int i = 0; i < threads.length; i++) {
       threads[i].start();
@@ -403,9 +444,14 @@
         throw new RuntimeException(e);
       }
     }
+    for (int i = 0; i < threads.length; i++) {
+      if (threads[i].failed) {
+        throw new RuntimeException("some thread(s) failed");
+      }
+    }
   }
 
-  private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple) throws IOException {
+  private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect) throws IOException {
 
     final LineFileDocs docs = new LineFileDocs(random);
 
@@ -437,7 +483,7 @@
       }
 
       try {
-        checkAnalysisConsistency(random, a, useCharFilter, text);
+        checkAnalysisConsistency(random, a, useCharFilter, text, offsetsAreCorrect);
       } catch (Throwable t) {
         // TODO: really we should pass a random seed to
         // checkAnalysisConsistency then print it here too:
@@ -477,6 +523,10 @@
   }
 
   public static void checkAnalysisConsistency(Random random, Analyzer a, boolean useCharFilter, String text) throws IOException {
+    checkAnalysisConsistency(random, a, useCharFilter, text, true);
+  }
+
+  public static void checkAnalysisConsistency(Random random, Analyzer a, boolean useCharFilter, String text, boolean offsetsAreCorrect) throws IOException {
 
     if (VERBOSE) {
       System.out.println(Thread.currentThread().getName() + ": NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text);
@@ -616,7 +666,8 @@
                                 types.toArray(new String[types.size()]),
                                 toIntArray(positions),
                                 toIntArray(positionLengths),
-                                text.length());
+                                text.length(),
+                                offsetsAreCorrect);
     } else if (typeAtt != null && posIncAtt != null && offsetAtt != null) {
       // offset + pos + type
       assertTokenStreamContents(ts, 
@@ -626,7 +677,8 @@
                                 types.toArray(new String[types.size()]),
                                 toIntArray(positions),
                                 null,
-                                text.length());
+                                text.length(),
+                                offsetsAreCorrect);
     } else if (posIncAtt != null && posLengthAtt != null && offsetAtt != null) {
       // offset + pos + posLength
       assertTokenStreamContents(ts, 
@@ -636,7 +688,8 @@
                                 null,
                                 toIntArray(positions),
                                 toIntArray(positionLengths),
-                                text.length());
+                                text.length(),
+                                offsetsAreCorrect);
     } else if (posIncAtt != null && offsetAtt != null) {
       // offset + pos
       assertTokenStreamContents(ts, 
@@ -646,7 +699,8 @@
                                 null,
                                 toIntArray(positions),
                                 null,
-                                text.length());
+                                text.length(),
+                                offsetsAreCorrect);
     } else if (offsetAtt != null) {
       // offset
       assertTokenStreamContents(ts, 
@@ -656,7 +710,8 @@
                                 null,
                                 null,
                                 null,
-                                text.length());
+                                text.length(),
+                                offsetsAreCorrect);
     } else {
       // terms only
       assertTokenStreamContents(ts, 
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java
index 298ab96..9515ae9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/LookaheadTokenFilter.java
@@ -151,7 +151,7 @@
         startPosData.startOffset = startOffset;
       } else {
         // Make sure our input isn't messing up offsets:
-        assert startPosData.startOffset == startOffset;
+        assert startPosData.startOffset == startOffset: "prev startOffset=" + startPosData.startOffset + " vs new startOffset=" + startOffset + " inputPos=" + inputPos;
       }
 
       final int endOffset = offsetAtt.endOffset();
@@ -159,7 +159,7 @@
         endPosData.endOffset = endOffset;
       } else {
         // Make sure our input isn't messing up offsets:
-        assert endPosData.endOffset == endOffset;
+        assert endPosData.endOffset == endOffset: "prev endOffset=" + endPosData.endOffset + " vs new endOffset=" + endOffset + " inputPos=" + inputPos;
       }
 
       tokenPending = true;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
index 642b28f..b1ab259 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
@@ -76,7 +76,7 @@
    * MockAnalyzer(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false}).
    */
   public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
-    this(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false);
+    this(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, true);
   }
 
   /** 
@@ -93,7 +93,8 @@
   public TokenStreamComponents createComponents(String fieldName, Reader reader) {
     MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase, maxTokenLength);
     tokenizer.setEnableChecks(enableChecks);
-    TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
+    MockTokenFilter filt = new MockTokenFilter(tokenizer, filter);
+    filt.setEnablePositionIncrements(enablePositionIncrements);
     return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName));
   }
   
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
index a488c4b..5a11b97 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
@@ -34,7 +34,9 @@
     // TODO: instead of fixed remainder... maybe a fixed
     // random seed?
     this.remainder = remainder;
-    assert remainder >= 0 && remainder < 10 : "invalid parameter";
+    if (remainder < 0 || remainder >= 10) {
+      throw new IllegalArgumentException("invalid remainder parameter (must be 0..10): " + remainder);
+    }
   }
   
   // for testing only, uses a remainder of 0
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java
index 74e2339..bbe2f37 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java
@@ -34,6 +34,9 @@
 
   public MockFixedLengthPayloadFilter(Random random, TokenStream in, int length) {
     super(in);
+    if (length < 0) {
+      throw new IllegalArgumentException("length must be >= 0");
+    }
     this.random = random;
     this.bytes = new byte[length];
     this.payload = new Payload(bytes);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java
index e47551b..44215e7 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockRandomLookaheadTokenFilter.java
@@ -31,10 +31,12 @@
 
   private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final Random random;
+  private final long seed;
 
   public MockRandomLookaheadTokenFilter(Random random, TokenStream in) {
     super(in);
-    this.random = random;
+    this.seed = random.nextLong();
+    this.random = new Random(seed);
   }
 
   @Override
@@ -57,9 +59,6 @@
 
     if (!end) {
       while (true) {
-        // We can use un-re-seeded random, because how far
-        // ahead we peek should never alter the resulting
-        // tokens as seen by the consumer:
         if (random.nextInt(3) == 1) {
           if (!peekToken()) {
             if (DEBUG) {
@@ -91,4 +90,10 @@
     }
     return result;
   }
+
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    random.setSeed(seed);
+  }
 }
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java
index 97863a4..efc7633 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java
@@ -55,7 +55,7 @@
       makeString("with"))));
   
   private final CharacterRunAutomaton filter;
-  private boolean enablePositionIncrements = false;
+  private boolean enablePositionIncrements = true;
 
   private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
@@ -67,14 +67,16 @@
    * @param filter DFA representing the terms that should be removed.
    * @param enablePositionIncrements true if the removal should accumulate position increments.
    */
-  public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+  public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter) {
     super(input);
     this.filter = filter;
-    this.enablePositionIncrements = enablePositionIncrements;
   }
   
   @Override
   public boolean incrementToken() throws IOException {
+    // TODO: fix me when posInc=false, to work like FilteringTokenFilter in that case and not return
+    // initial token with posInc=0 ever
+    
     // return the first non-stop word found
     int skippedPositions = 0;
     while (input.incrementToken()) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java
new file mode 100644
index 0000000..f213545
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java
@@ -0,0 +1,170 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.util.Attribute;
+
+// TODO: rename to OffsetsXXXTF?  ie we only validate
+// offsets (now anyway...)
+
+// TODO: also make a DebuggingTokenFilter, that just prints
+// all att values that come through it...
+
+// TODO: BTSTC should just append this to the chain
+// instead of checking itself:
+
+/** A TokenFilter that checks consistency of the tokens (eg
+ *  offsets are consistent with one another). */
+public final class ValidatingTokenFilter extends TokenFilter {
+
+  private int pos;
+  private int lastStartOffset;
+
+  // Maps position to the start/end offset:
+  private final Map<Integer,Integer> posToStartOffset = new HashMap<Integer,Integer>();
+  private final Map<Integer,Integer> posToEndOffset = new HashMap<Integer,Integer>();
+
+  private final PositionIncrementAttribute posIncAtt = getAttrIfExists(PositionIncrementAttribute.class);
+  private final PositionLengthAttribute posLenAtt = getAttrIfExists(PositionLengthAttribute.class);
+  private final OffsetAttribute offsetAtt = getAttrIfExists(OffsetAttribute.class);
+  private final CharTermAttribute termAtt = getAttrIfExists(CharTermAttribute.class);
+  private final boolean offsetsAreCorrect;
+
+  private final String name;
+
+  // Returns null if the attr wasn't already added
+  private <A extends Attribute> A getAttrIfExists(Class<A> att) {
+    if (hasAttribute(att)) {
+      return getAttribute(att);
+    } else {
+      return null;
+    }
+  }
+
+  /** The name arg is used to identify this stage when
+   *  throwing exceptions (useful if you have more than one
+   *  instance in your chain). */
+  public ValidatingTokenFilter(TokenStream in, String name, boolean offsetsAreCorrect) {
+    super(in);
+    this.name = name;
+    this.offsetsAreCorrect = offsetsAreCorrect;
+  }
+
+  @Override
+  public boolean incrementToken() throws IOException {
+    if (!input.incrementToken()) {
+      return false;
+    }
+
+    int startOffset = 0;
+    int endOffset = 0;
+    int posLen = 0;
+    
+    if (posIncAtt != null) {
+      pos += posIncAtt.getPositionIncrement();
+      if (pos == -1) {
+        throw new IllegalStateException("first posInc must be > 0");
+      }
+    }
+
+    // System.out.println("  got token=" + termAtt + " pos=" + pos);
+    
+    if (offsetAtt != null) {
+      startOffset = offsetAtt.startOffset();
+      endOffset = offsetAtt.endOffset();
+
+      if (startOffset < 0) {
+        throw new IllegalStateException(name + ": startOffset=" + startOffset + " is < 0");
+      }
+      if (endOffset < 0) {
+        throw new IllegalStateException(name + ": endOffset=" + endOffset + " is < 0");
+      }
+      if (endOffset < startOffset) {
+        throw new IllegalStateException(name + ": startOffset=" + startOffset + " is > endOffset=" + endOffset + " pos=" + pos + "; token=" + termAtt);
+      }
+      if (offsetsAreCorrect && offsetAtt.startOffset() < lastStartOffset) {
+        throw new IllegalStateException(name + ": offsets must not go backwards startOffset=" + startOffset + " is < lastStartOffset=" + lastStartOffset);
+      }
+      lastStartOffset = offsetAtt.startOffset();
+    }
+    
+    posLen = posLenAtt == null ? 1 : posLenAtt.getPositionLength();
+    
+    if (offsetAtt != null && posIncAtt != null && offsetsAreCorrect) {
+
+      if (!posToStartOffset.containsKey(pos)) {
+        // First time we've seen a token leaving from this position:
+        posToStartOffset.put(pos, startOffset);
+        //System.out.println("  + s " + pos + " -> " + startOffset);
+      } else {
+        // We've seen a token leaving from this position
+        // before; verify the startOffset is the same:
+        //System.out.println("  + vs " + pos + " -> " + startOffset);
+        final int oldStartOffset = posToStartOffset.get(pos);
+        if (oldStartOffset != startOffset) {
+          throw new IllegalStateException(name + ": inconsistent startOffset at pos=" + pos + ": " + oldStartOffset + " vs " + startOffset + "; token=" + termAtt);
+        }
+      }
+
+      final int endPos = pos + posLen;
+
+      if (!posToEndOffset.containsKey(endPos)) {
+        // First time we've seen a token arriving to this position:
+        posToEndOffset.put(endPos, endOffset);
+        //System.out.println("  + e " + endPos + " -> " + endOffset);
+      } else {
+        // We've seen a token arriving to this position
+        // before; verify the endOffset is the same:
+        //System.out.println("  + ve " + endPos + " -> " + endOffset);
+        final int oldEndOffset = posToEndOffset.get(endPos);
+        if (oldEndOffset != endOffset) {
+          throw new IllegalStateException(name + ": inconsistent endOffset at pos=" + endPos + ": " + oldEndOffset + " vs " + endOffset + "; token=" + termAtt);
+        }
+      }
+    }
+
+    return true;
+  }
+
+  @Override
+  public void end() throws IOException {
+    super.end();
+
+    // TODO: what else to validate
+
+    // TODO: check that endOffset is >= max(endOffset)
+    // we've seen
+  }
+
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    pos = -1;
+    posToStartOffset.clear();
+    posToEndOffset.clear();
+    lastStartOffset = 0;
+  }
+}
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
index 8232b88..9798464 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/commongrams/CommonGramsFilter.java
@@ -16,6 +16,7 @@
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
@@ -54,6 +55,7 @@
   private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
   private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
   private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
+  private final PositionLengthAttribute posLenAttribute = addAttribute(PositionLengthAttribute.class);
 
   private int lastStartOffset;
   private boolean lastWasCommon;
@@ -166,6 +168,7 @@
     buffer.getChars(0, length, termText, 0);
     termAttribute.setLength(length);
     posIncAttribute.setPositionIncrement(0);
+    posLenAttribute.setPositionLength(2); // bigram
     offsetAttribute.setOffset(lastStartOffset, endOffset);
     typeAttribute.setType(GRAM_TYPE);
     buffer.setLength(0);
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
index 3b3fae9..909ef5e 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
@@ -82,8 +82,17 @@
     super(input);
     
     this.tokens=new LinkedList<CompoundToken>();
+    if (minWordSize < 0) {
+      throw new IllegalArgumentException("minWordSize cannot be negative");
+    }
     this.minWordSize=minWordSize;
+    if (minSubwordSize < 0) {
+      throw new IllegalArgumentException("minSubwordSize cannot be negative");
+    }
     this.minSubwordSize=minSubwordSize;
+    if (maxSubwordSize < 0) {
+      throw new IllegalArgumentException("maxSubwordSize cannot be negative");
+    }
     this.maxSubwordSize=maxSubwordSize;
     this.onlyLongestMatch=onlyLongestMatch;
     this.dictionary = dictionary;
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
index 935c607..71d317b 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
@@ -191,6 +191,8 @@
         // we only put subwords to the token stream
         // that are longer than minPartSize
         if (partLength < this.minSubwordSize) {
+          // BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the 
+          // calculation above, and we rely upon minSubwordSize being >=0 to filter them out...
           continue;
         }
 
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
index c9d73ef..44ee084 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
@@ -43,16 +43,25 @@
 
   public KeywordTokenizer(Reader input, int bufferSize) {
     super(input);
+    if (bufferSize <= 0) {
+      throw new IllegalArgumentException("bufferSize must be > 0");
+    }
     termAtt.resizeBuffer(bufferSize);
   }
 
   public KeywordTokenizer(AttributeSource source, Reader input, int bufferSize) {
     super(source, input);
+    if (bufferSize <= 0) {
+      throw new IllegalArgumentException("bufferSize must be > 0");
+    }
     termAtt.resizeBuffer(bufferSize);
   }
 
   public KeywordTokenizer(AttributeFactory factory, Reader input, int bufferSize) {
     super(factory, input);
+    if (bufferSize <= 0) {
+      throw new IllegalArgumentException("bufferSize must be > 0");
+    }
     termAtt.resizeBuffer(bufferSize);
   }
   
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
index 26b5b1d..3755775 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
@@ -65,6 +65,12 @@
 
   public PathHierarchyTokenizer(Reader input, int bufferSize, char delimiter, char replacement, int skip) {
     super(input);
+    if (bufferSize < 0) {
+      throw new IllegalArgumentException("bufferSize cannot be negative");
+    }
+    if (skip < 0) {
+      throw new IllegalArgumentException("skip cannot be negative");
+    }
     termAtt.resizeBuffer(bufferSize);
 
     this.delimiter = delimiter;
@@ -85,10 +91,11 @@
   private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
   private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class);
   private int startPosition = 0;
-  private int finalOffset = 0;
   private int skipped = 0;
   private boolean endDelimiter = false;
   private StringBuilder resultToken;
+  
+  private int charsRead = 0;
 
 
   @Override
@@ -112,12 +119,13 @@
 
     while (true) {
       int c = input.read();
-      if( c < 0 ){
+      if (c >= 0) {
+        charsRead++;
+      } else {
         if( skipped > skip ) {
           length += resultToken.length();
           termAtt.setLength(length);
-          finalOffset = correctOffset(startPosition + length);
-          offsetAtt.setOffset(correctOffset(startPosition), finalOffset);
+           offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition + length));
           if( added ){
             resultToken.setLength(0);
             resultToken.append(termAtt.buffer(), 0, length);
@@ -125,7 +133,6 @@
           return added;
         }
         else{
-          finalOffset = correctOffset(startPosition + length);
           return false;
         }
       }
@@ -168,8 +175,7 @@
     }
     length += resultToken.length();
     termAtt.setLength(length);
-    finalOffset = correctOffset(startPosition + length);
-    offsetAtt.setOffset(correctOffset(startPosition), finalOffset);
+    offsetAtt.setOffset(correctOffset(startPosition), correctOffset(startPosition+length));
     resultToken.setLength(0);
     resultToken.append(termAtt.buffer(), 0, length);
     return true;
@@ -178,15 +184,17 @@
   @Override
   public final void end() {
     // set final offset
+    int finalOffset = correctOffset(charsRead);
     offsetAtt.setOffset(finalOffset, finalOffset);
   }
 
   @Override
-  public void reset(Reader input) throws IOException {
-    super.reset(input);
+  public void reset() throws IOException {
+    super.reset();
     resultToken.setLength(0);
-    finalOffset = 0;
+    charsRead = 0;
     endDelimiter = false;
     skipped = 0;
+    startPosition = 0;
   }
 }
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java
index fc8a683..97593c6 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java
@@ -77,6 +77,12 @@
 
   public ReversePathHierarchyTokenizer(Reader input, int bufferSize, char delimiter, char replacement, int skip) {
     super(input);
+    if (bufferSize < 0) {
+      throw new IllegalArgumentException("bufferSize cannot be negative");
+    }
+    if (skip < 0) {
+      throw new IllegalArgumentException("skip cannot be negative");
+    }
     termAtt.resizeBuffer(bufferSize);
     this.delimiter = delimiter;
     this.replacement = replacement;
@@ -137,7 +143,11 @@
       }
       resultToken.getChars(0, resultToken.length(), resultTokenBuffer, 0);
       resultToken.setLength(0);
-      endPosition = delimiterPositions.get(delimitersCount-1 - skip);
+      int idx = delimitersCount-1 - skip;
+      if (idx >= 0) {
+        // otherwise its ok, because we will skip and return false
+        endPosition = delimiterPositions.get(idx);
+      }
       finalOffset = correctOffset(length);
       posAtt.setPositionIncrement(1);
     }
@@ -163,10 +173,11 @@
   }
 
   @Override
-  public void reset(Reader input) throws IOException {
-    super.reset(input);
+  public void reset() throws IOException {
+    super.reset();
     resultToken.setLength(0);
     finalOffset = 0;
+    endPosition = 0;
     skipped = 0;
     delimitersCount = -1;
     delimiterPositions.clear();
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
index 3d43d17..bc80391 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
@@ -69,8 +69,17 @@
     super(input);
     this.pattern = pattern;
     this.group = group;
+
+    // Use "" instead of str so don't consume chars
+    // (fillBuffer) from the input on throwing IAE below:
+    matcher = pattern.matcher("");
+
+    // confusingly group count depends ENTIRELY on the pattern but is only accessible via matcher
+    if (group >= 0 && group > matcher.groupCount()) {
+      throw new IllegalArgumentException("invalid group specified: pattern only has: " + matcher.groupCount() + " capturing groups");
+    }
     fillBuffer(str, input);
-    matcher = pattern.matcher(str);
+    matcher.reset(str);
     index = 0;
   }
 
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
index 97f5fef..04737ed 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
@@ -57,6 +57,9 @@
    */
   public PositionFilter(final TokenStream input, final int positionIncrement) {
     super(input);
+    if (positionIncrement < 0) {
+      throw new IllegalArgumentException("positionIncrement may not be negative");
+    }
     this.positionIncrement = positionIncrement;
   }
 
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
index 464bde0..d0b8e05 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
@@ -23,9 +23,10 @@
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.AttributeSource;
 
@@ -150,6 +151,7 @@
   private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
   private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+  private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
   private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
 
 
@@ -319,6 +321,7 @@
           noShingleOutput = false;
         }
         offsetAtt.setOffset(offsetAtt.startOffset(), nextToken.offsetAtt.endOffset());
+        posLenAtt.setPositionLength(builtGramSize);
         isOutputHere = true;
         gramSize.advance();
         tokenAvailable = true;
@@ -436,6 +439,8 @@
     super.reset();
     gramSize.reset();
     inputWindow.clear();
+    nextInputStreamToken = null;
+    isNextInputStreamToken = false;
     numFillerTokensToInsert = 0;
     isOutputHere = false;
     noShingleOutput = true;
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
index c69d470..7a2639e 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
@@ -67,7 +67,7 @@
         Class.forName("org.tartarus.snowball.ext." + name + "Stemmer").asSubclass(SnowballProgram.class);
       stemmer = stemClass.newInstance();
     } catch (Exception e) {
-      throw new RuntimeException(e.toString());
+      throw new IllegalArgumentException("Invalid stemmer class specified: " + name, e);
     }
   }
 
diff --git a/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java b/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
index c495bdd..c5ba3a0 100644
--- a/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
+++ b/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
@@ -177,6 +177,12 @@
   }
   
   private void init(int tokenOutput, Set<String> untokenizedTypes) {
+    // TODO: cutover to enum
+    if (tokenOutput != TOKENS_ONLY &&
+        tokenOutput != UNTOKENIZED_ONLY &&
+        tokenOutput != BOTH) {
+      throw new IllegalArgumentException("tokenOutput must be TOKENS_ONLY, UNTOKENIZED_ONLY or BOTH");
+    }
     this.tokenOutput = tokenOutput;
     this.untokenizedTypes = untokenizedTypes;    
   }
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java
index 9740baf..fa77b40 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java
@@ -19,6 +19,8 @@
 
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -27,6 +29,8 @@
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util._TestUtil;
+import org.junit.Ignore;
 
 public class TestMappingCharFilter extends BaseTokenStreamTestCase {
 
@@ -190,4 +194,67 @@
     int numRounds = RANDOM_MULTIPLIER * 10000;
     checkRandomData(random, analyzer, numRounds);
   }
+  
+  @Ignore("wrong finalOffset: https://issues.apache.org/jira/browse/LUCENE-3971")
+  public void testFinalOffsetSpecialCase() throws Exception {  
+    final NormalizeCharMap map = new NormalizeCharMap();
+    map.add("t", "");
+    // even though this below rule has no effect, the test passes if you remove it!!
+    map.add("tmakdbl", "c");
+    
+    Analyzer analyzer = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+        return new TokenStreamComponents(tokenizer, tokenizer);
+      }
+
+      @Override
+      protected Reader initReader(Reader reader) {
+        return new MappingCharFilter(map, CharReader.get(reader));
+      }
+    };
+    
+    String text = "gzw f quaxot";
+    checkAnalysisConsistency(random, analyzer, false, text);
+  }
+  
+  @Ignore("wrong finalOffset: https://issues.apache.org/jira/browse/LUCENE-3971")
+  public void testRandomMaps() throws Exception {
+    for (int i = 0; i < 100; i++) {
+      final NormalizeCharMap map = randomMap();
+      Analyzer analyzer = new Analyzer() {
+        @Override
+        protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+          Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+          return new TokenStreamComponents(tokenizer, tokenizer);
+        }
+
+        @Override
+        protected Reader initReader(Reader reader) {
+          return new MappingCharFilter(map, CharReader.get(reader));
+        }
+      };
+      int numRounds = RANDOM_MULTIPLIER * 100;
+      checkRandomData(random, analyzer, numRounds);
+    }
+  }
+  
+  private NormalizeCharMap randomMap() {
+    NormalizeCharMap map = new NormalizeCharMap();
+    // we can't add duplicate keys, or NormalizeCharMap gets angry
+    Set<String> keys = new HashSet<String>();
+    int num = random.nextInt(5);
+    //System.out.println("NormalizeCharMap=");
+    for (int i = 0; i < num; i++) {
+      String key = _TestUtil.randomSimpleString(random);
+      if (!keys.contains(key)) {
+        String value = _TestUtil.randomSimpleString(random);
+        map.add(key, value);
+        keys.add(key);
+        //System.out.println("mapping: '" + key + "' => '" + value + "'");
+      }
+    }
+    return map;
+  }
 }
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
index b40022a..016b107 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
@@ -18,17 +18,28 @@
  */
 
 import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Reader;
 import java.io.StringReader;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Modifier;
 import java.net.URL;
+import java.nio.CharBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
+import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -36,67 +47,174 @@
 import org.apache.lucene.analysis.CharReader;
 import org.apache.lucene.analysis.CharStream;
 import org.apache.lucene.analysis.EmptyTokenizer;
+import org.apache.lucene.analysis.MockGraphTokenFilter;
+import org.apache.lucene.analysis.MockRandomLookaheadTokenFilter;
+import org.apache.lucene.analysis.MockTokenFilter;
+import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
+import org.apache.lucene.analysis.ValidatingTokenFilter;
+import org.apache.lucene.analysis.charfilter.CharFilter;
+import org.apache.lucene.analysis.charfilter.MappingCharFilter;
+import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
+import org.apache.lucene.analysis.cjk.CJKBigramFilter;
+import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
+import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
+import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
+import org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter;
+import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
+import org.apache.lucene.analysis.hunspell.HunspellDictionary;
+import org.apache.lucene.analysis.hunspell.HunspellDictionaryTest;
+import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter;
+import org.apache.lucene.analysis.miscellaneous.TrimFilter;
+import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
 import org.apache.lucene.analysis.ngram.NGramTokenFilter;
 import org.apache.lucene.analysis.ngram.NGramTokenizer;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizer;
+import org.apache.lucene.analysis.path.ReversePathHierarchyTokenizer;
+import org.apache.lucene.analysis.payloads.IdentityEncoder;
+import org.apache.lucene.analysis.payloads.PayloadEncoder;
+import org.apache.lucene.analysis.position.PositionFilter;
+import org.apache.lucene.analysis.snowball.TestSnowball;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.synonym.SynonymMap;
+import org.apache.lucene.analysis.th.ThaiWordFilter;
+import org.apache.lucene.analysis.util.CharArrayMap;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.Rethrow;
 import org.apache.lucene.util.Version;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.tartarus.snowball.SnowballProgram;
+import org.xml.sax.InputSource;
 
 /** tests random analysis chains */
 public class TestRandomChains extends BaseTokenStreamTestCase {
-  static List<Class<? extends Tokenizer>> tokenizers;
-  static List<Class<? extends TokenFilter>> tokenfilters;
-  static List<Class<? extends CharStream>> charfilters;
+
+  static List<Constructor<? extends Tokenizer>> tokenizers;
+  static List<Constructor<? extends TokenFilter>> tokenfilters;
+  static List<Constructor<? extends CharStream>> charfilters;
+
+  // TODO: fix those and remove
+  private static final Set<Class<?>> brokenComponents = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+  static {
+    // TODO: can we promote some of these to be only
+    // offsets offenders?
+    Collections.<Class<?>>addAll(brokenComponents,
+      // TODO: fix basetokenstreamtestcase not to trip because this one has no CharTermAtt
+      EmptyTokenizer.class,
+      // doesn't actual reset itself!
+      CachingTokenFilter.class,
+      // doesn't consume whole stream!
+      LimitTokenCountFilter.class,
+      // Not broken: we forcefully add this, so we shouldn't
+      // also randomly pick it:
+      ValidatingTokenFilter.class,
+      // NOTE: these by themselves won't cause any 'basic assertions' to fail.
+      // but see https://issues.apache.org/jira/browse/LUCENE-3920, if any 
+      // tokenfilter that combines words (e.g. shingles) comes after them,
+      // this will create bogus offsets because their 'offsets go backwards',
+      // causing shingle or whatever to make a single token with a 
+      // startOffset thats > its endOffset
+      // (see LUCENE-3738 for a list of other offenders here)
+      // broken!
+      NGramTokenizer.class,
+      // broken!
+      NGramTokenFilter.class,
+      // broken!
+      EdgeNGramTokenizer.class,
+      // broken!
+      EdgeNGramTokenFilter.class,
+      // broken!
+      WordDelimiterFilter.class,
+      // broken!
+      TrimFilter.class,
+      // TODO: remove this class after we fix its finalOffset bug
+      MappingCharFilter.class
+    );
+  }
+
+  // TODO: also fix these and remove (maybe):
+  // Classes that don't produce consistent graph offsets:
+  private static final Set<Class<?>> brokenOffsetsComponents = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+  static {
+    Collections.<Class<?>>addAll(brokenOffsetsComponents,
+      ReversePathHierarchyTokenizer.class,
+      PathHierarchyTokenizer.class,
+      HyphenationCompoundWordTokenFilter.class,
+      DictionaryCompoundWordTokenFilter.class,
+      // TODO: corrumpts graphs (offset consistency check):
+      PositionFilter.class,
+      // TODO: it seems to mess up offsets!?
+      WikipediaTokenizer.class,
+      // TODO: doesn't handle graph inputs
+      ThaiWordFilter.class,
+      // TODO: doesn't handle graph inputs
+      CJKBigramFilter.class
+    );
+  }
   
   @BeforeClass
   public static void beforeClass() throws Exception {
     List<Class<?>> analysisClasses = new ArrayList<Class<?>>();
     getClassesForPackage("org.apache.lucene.analysis", analysisClasses);
-    tokenizers = new ArrayList<Class<? extends Tokenizer>>();
-    tokenfilters = new ArrayList<Class<? extends TokenFilter>>();
-    charfilters = new ArrayList<Class<? extends CharStream>>();
-    for (Class<?> c : analysisClasses) {
-      // don't waste time with abstract classes or deprecated known-buggy ones
+    tokenizers = new ArrayList<Constructor<? extends Tokenizer>>();
+    tokenfilters = new ArrayList<Constructor<? extends TokenFilter>>();
+    charfilters = new ArrayList<Constructor<? extends CharStream>>();
+    for (final Class<?> c : analysisClasses) {
       final int modifiers = c.getModifiers();
-      if (Modifier.isAbstract(modifiers) || !Modifier.isPublic(modifiers)
-          || c.getAnnotation(Deprecated.class) != null
-          || c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface()
-          // TODO: fix basetokenstreamtestcase not to trip because this one has no CharTermAtt
-          || c.equals(EmptyTokenizer.class)
-          // doesn't actual reset itself!
-          || c.equals(CachingTokenFilter.class)
-          // broken!
-          || c.equals(NGramTokenizer.class)
-          // broken!
-          || c.equals(NGramTokenFilter.class)
-          // broken!
-          || c.equals(EdgeNGramTokenizer.class)
-          // broken!
-          || c.equals(EdgeNGramTokenFilter.class)) {
+      if (
+        // don't waste time with abstract classes or deprecated known-buggy ones
+        Modifier.isAbstract(modifiers) || !Modifier.isPublic(modifiers)
+        || c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface()
+        || brokenComponents.contains(c)
+        || c.isAnnotationPresent(Deprecated.class)
+        || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharStream.class.isAssignableFrom(c))
+      ) {
         continue;
       }
-      if (Tokenizer.class.isAssignableFrom(c)) {
-        tokenizers.add(c.asSubclass(Tokenizer.class));
-      } else if (TokenFilter.class.isAssignableFrom(c)) {
-        tokenfilters.add(c.asSubclass(TokenFilter.class));
-      } else if (CharStream.class.isAssignableFrom(c)) {
-        charfilters.add(c.asSubclass(CharStream.class));
+      
+      for (final Constructor<?> ctor : c.getConstructors()) {
+        // don't test synthetic or deprecated ctors, they likely have known bugs:
+        if (ctor.isSynthetic() || ctor.isAnnotationPresent(Deprecated.class)) {
+          continue;
+        }
+        if (Tokenizer.class.isAssignableFrom(c)) {
+          assertTrue(ctor.toGenericString() + " has unsupported parameter types",
+            allowedTokenizerArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
+          tokenizers.add(castConstructor(Tokenizer.class, ctor));
+        } else if (TokenFilter.class.isAssignableFrom(c)) {
+          assertTrue(ctor.toGenericString() + " has unsupported parameter types",
+            allowedTokenFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
+          tokenfilters.add(castConstructor(TokenFilter.class, ctor));
+        } else if (CharStream.class.isAssignableFrom(c)) {
+          assertTrue(ctor.toGenericString() + " has unsupported parameter types",
+            allowedCharFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
+          charfilters.add(castConstructor(CharStream.class, ctor));
+        } else {
+          fail("Cannot get here");
+        }
       }
     }
-    final Comparator<Class<?>> classComp = new Comparator<Class<?>>() {
+    
+    final Comparator<Constructor<?>> ctorComp = new Comparator<Constructor<?>>() {
       @Override
-      public int compare(Class<?> arg0, Class<?> arg1) {
-        return arg0.getName().compareTo(arg1.getName());
+      public int compare(Constructor<?> arg0, Constructor<?> arg1) {
+        return arg0.toGenericString().compareTo(arg1.toGenericString());
       }
     };
-    Collections.sort(tokenizers, classComp);
-    Collections.sort(tokenfilters, classComp);
-    Collections.sort(charfilters, classComp);
+    Collections.sort(tokenizers, ctorComp);
+    Collections.sort(tokenfilters, ctorComp);
+    Collections.sort(charfilters, ctorComp);
     if (VERBOSE) {
       System.out.println("tokenizers = " + tokenizers);
       System.out.println("tokenfilters = " + tokenfilters);
@@ -111,170 +229,12 @@
     charfilters = null;
   }
   
-  static class MockRandomAnalyzer extends Analyzer {
-    final long seed;
-    
-    MockRandomAnalyzer(long seed) {
-      this.seed = seed;
-    }
-    
-    @Override
-    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
-      Random random = new Random(seed);
-      TokenizerSpec tokenizerspec = newTokenizer(random, reader);
-      TokenFilterSpec filterspec = newFilterChain(random, tokenizerspec.tokenizer);
-      return new TokenStreamComponents(tokenizerspec.tokenizer, filterspec.stream);
-    }
-
-    @Override
-    protected Reader initReader(Reader reader) {
-      Random random = new Random(seed);
-      CharFilterSpec charfilterspec = newCharFilterChain(random, reader);
-      return charfilterspec.reader;
-    }
-
-    @Override
-    public String toString() {
-      Random random = new Random(seed);
-      StringBuilder sb = new StringBuilder();
-      CharFilterSpec charfilterSpec = newCharFilterChain(random, new StringReader(""));
-      sb.append("\ncharfilters=");
-      sb.append(charfilterSpec.toString);
-      // intentional: initReader gets its own separate random
-      random = new Random(seed);
-      TokenizerSpec tokenizerSpec = newTokenizer(random, charfilterSpec.reader);
-      sb.append("\n");
-      sb.append("tokenizer=");
-      sb.append(tokenizerSpec.toString);
-      TokenFilterSpec tokenfilterSpec = newFilterChain(random, tokenizerSpec.tokenizer);
-      sb.append("\n");
-      sb.append("filters=");
-      sb.append(tokenfilterSpec.toString);
-      return sb.toString();
-    }
-    
-    // create a new random tokenizer from classpath
-    private TokenizerSpec newTokenizer(Random random, Reader reader) {
-      TokenizerSpec spec = new TokenizerSpec();
-      boolean success = false;
-      while (!success) {
-        try {
-          // TODO: check Reader+Version,Version+Reader too
-          // also look for other variants and handle them special
-          int idx = random.nextInt(tokenizers.size());
-          try {
-            Constructor<? extends Tokenizer> c = tokenizers.get(idx).getConstructor(Version.class, Reader.class);
-            spec.tokenizer = c.newInstance(TEST_VERSION_CURRENT, reader);
-          } catch (NoSuchMethodException e) {
-            Constructor<? extends Tokenizer> c = tokenizers.get(idx).getConstructor(Reader.class);
-            spec.tokenizer = c.newInstance(reader);
-          }
-          spec.toString = tokenizers.get(idx).toString();
-          success = true;
-        } catch (Exception e) {
-          // ignore
-        }
-      }
-      return spec;
-    }
-    
-    private CharFilterSpec newCharFilterChain(Random random, Reader reader) {
-      CharFilterSpec spec = new CharFilterSpec();
-      spec.reader = reader;
-      StringBuilder descr = new StringBuilder();
-      int numFilters = random.nextInt(3);
-      for (int i = 0; i < numFilters; i++) {
-        boolean success = false;
-        while (!success) {
-          try {
-            // TODO: also look for other variants and handle them special
-            int idx = random.nextInt(charfilters.size());
-            try {
-              Constructor<? extends CharStream> c = charfilters.get(idx).getConstructor(Reader.class);
-              spec.reader = c.newInstance(spec.reader);
-            } catch (NoSuchMethodException e) {
-              Constructor<? extends CharStream> c = charfilters.get(idx).getConstructor(CharStream.class);
-              spec.reader = c.newInstance(CharReader.get(spec.reader));
-            }
-
-            if (descr.length() > 0) {
-              descr.append(",");
-            }
-            descr.append(charfilters.get(idx).toString());
-            success = true;
-          } catch (Exception e) {
-            // ignore
-          }
-        }
-      }
-      spec.toString = descr.toString();
-      return spec;
-    }
-    
-    private TokenFilterSpec newFilterChain(Random random, Tokenizer tokenizer) {
-      TokenFilterSpec spec = new TokenFilterSpec();
-      spec.stream = tokenizer;
-      StringBuilder descr = new StringBuilder();
-      int numFilters = random.nextInt(5);
-      for (int i = 0; i < numFilters; i++) {
-        boolean success = false;
-        while (!success) {
-          try {
-            // TODO: also look for other variants and handle them special
-            int idx = random.nextInt(tokenfilters.size());
-            try {
-              Constructor<? extends TokenFilter> c = tokenfilters.get(idx).getConstructor(Version.class, TokenStream.class);
-              spec.stream = c.newInstance(TEST_VERSION_CURRENT, spec.stream);
-            } catch (NoSuchMethodException e) {
-              Constructor<? extends TokenFilter> c = tokenfilters.get(idx).getConstructor(TokenStream.class);
-              spec.stream = c.newInstance(spec.stream);
-            }
-            if (descr.length() > 0) {
-              descr.append(",");
-            }
-            descr.append(tokenfilters.get(idx).toString());
-            success = true;
-          } catch (Exception e) {
-            // ignore
-          }
-        }
-      }
-      spec.toString = descr.toString();
-      return spec;
-    }
+  /** Hack to work around the stupidness of Oracle's strict Java backwards compatibility.
+   * {@code Class<T>#getConstructors()} should return unmodifiable {@code List<Constructor<T>>} not array! */
+  @SuppressWarnings("unchecked") 
+  private static <T> Constructor<T> castConstructor(Class<T> instanceClazz, Constructor<?> ctor) {
+    return (Constructor<T>) ctor;
   }
-  
-  static class TokenizerSpec {
-    Tokenizer tokenizer;
-    String toString;
-  }
-  
-  static class TokenFilterSpec {
-    TokenStream stream;
-    String toString;
-  }
-  
-  static class CharFilterSpec {
-    Reader reader;
-    String toString;
-  }
-  
-  public void testRandomChains() throws Throwable {
-    int numIterations = atLeast(20);
-    for (int i = 0; i < numIterations; i++) {
-      MockRandomAnalyzer a = new MockRandomAnalyzer(random.nextLong());
-      if (VERBOSE) {
-        System.out.println("Creating random analyzer:" + a);
-      }
-      try {
-        checkRandomData(random, a, 1000);
-      } catch (Throwable e) {
-        System.err.println("Exception from random analyzer: " + a);
-        throw e;
-      }
-    }
-  }
-  
   private static void getClassesForPackage(String pckgname, List<Class<?>> classes) throws Exception {
     final ClassLoader cld = TestRandomChains.class.getClassLoader();
     final String path = pckgname.replace('.', '/');
@@ -303,4 +263,568 @@
       }
     }
   }
+  
+  private static interface ArgProducer {
+    Object create(Random random);
+  }
+  
+  private static final Map<Class<?>,ArgProducer> argProducers = new IdentityHashMap<Class<?>,ArgProducer>() {{
+    put(int.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: could cause huge ram usage to use full int range for some filters
+        // (e.g. allocate enormous arrays)
+        // return Integer.valueOf(random.nextInt());
+        return Integer.valueOf(_TestUtil.nextInt(random, -100, 100));
+      }
+    });
+    put(char.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: fix any filters that care to throw IAE instead.
+        // also add a unicode validating filter to validate termAtt?
+        // return Character.valueOf((char)random.nextInt(65536));
+        while(true) {
+          char c = (char)random.nextInt(65536);
+          if (c < '\uD800' || c > '\uDFFF') {
+            return Character.valueOf(c);
+          }
+        }
+      }
+    });
+    put(float.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        return Float.valueOf(random.nextFloat());
+      }
+    });
+    put(boolean.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        return Boolean.valueOf(random.nextBoolean());
+      }
+    });
+    put(byte.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // this wraps to negative when casting to byte
+        return Byte.valueOf((byte) random.nextInt(256));
+      }
+    });
+    put(byte[].class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        byte bytes[] = new byte[random.nextInt(256)];
+        random.nextBytes(bytes);
+        return bytes;
+      }
+    });
+    put(Random.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        return new Random(random.nextLong());
+      }
+    });
+    put(Version.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // we expect bugs in emulating old versions
+        return TEST_VERSION_CURRENT;
+      }
+    });
+    put(Set.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TypeTokenFilter
+        Set<String> set = new HashSet<String>();
+        int num = random.nextInt(5);
+        for (int i = 0; i < num; i++) {
+          set.add(StandardTokenizer.TOKEN_TYPES[random.nextInt(StandardTokenizer.TOKEN_TYPES.length)]);
+        }
+        return set;
+      }
+    });
+    put(Collection.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // CapitalizationFilter
+        Collection<char[]> col = new ArrayList<char[]>();
+        int num = random.nextInt(5);
+        for (int i = 0; i < num; i++) {
+          col.add(_TestUtil.randomSimpleString(random).toCharArray());
+        }
+        return col;
+      }
+    });
+    put(CharArraySet.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        int num = random.nextInt(10);
+        CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, num, random.nextBoolean());
+        for (int i = 0; i < num; i++) {
+          // TODO: make nastier
+          set.add(_TestUtil.randomSimpleString(random));
+        }
+        return set;
+      }
+    });
+    put(Pattern.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: don't want to make the exponentially slow ones Dawid documents
+        // in TestPatternReplaceFilter, so dont use truly random patterns (for now)
+        return Pattern.compile("a");
+      }
+    });
+    put(PayloadEncoder.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        return new IdentityEncoder(); // the other encoders will throw exceptions if tokens arent numbers?
+      }
+    });
+    put(HunspellDictionary.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: make nastier
+        InputStream affixStream = HunspellDictionaryTest.class.getResourceAsStream("test.aff");
+        InputStream dictStream = HunspellDictionaryTest.class.getResourceAsStream("test.dic");
+        try {
+         return new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT);
+        } catch (Exception ex) {
+          Rethrow.rethrow(ex);
+          return null; // unreachable code
+        }
+      }
+    });
+    put(EdgeNGramTokenizer.Side.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        return random.nextBoolean() 
+            ? EdgeNGramTokenizer.Side.FRONT 
+            : EdgeNGramTokenizer.Side.BACK;
+      }
+    });
+    put(EdgeNGramTokenFilter.Side.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        return random.nextBoolean() 
+            ? EdgeNGramTokenFilter.Side.FRONT 
+            : EdgeNGramTokenFilter.Side.BACK;
+      }
+    });
+    put(HyphenationTree.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: make nastier
+        try {
+          InputSource is = new InputSource(TestCompoundWordTokenFilter.class.getResource("da_UTF8.xml").toExternalForm());
+          HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);
+          return hyphenator;
+        } catch (Exception ex) {
+          Rethrow.rethrow(ex);
+          return null; // unreachable code
+        }
+      }
+    });
+    put(SnowballProgram.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        try {
+          String lang = TestSnowball.SNOWBALL_LANGS[random.nextInt(TestSnowball.SNOWBALL_LANGS.length)];
+          Class<? extends SnowballProgram> clazz = Class.forName("org.tartarus.snowball.ext." + lang + "Stemmer").asSubclass(SnowballProgram.class);
+          return clazz.newInstance();
+        } catch (Exception ex) {
+          Rethrow.rethrow(ex);
+          return null; // unreachable code
+        }
+      }
+    });
+    put(String.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: make nastier
+        if (random.nextBoolean()) {
+          // a token type
+          return StandardTokenizer.TOKEN_TYPES[random.nextInt(StandardTokenizer.TOKEN_TYPES.length)];
+        } else {
+          return _TestUtil.randomSimpleString(random);
+        }
+      }
+    });
+    put(NormalizeCharMap.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        NormalizeCharMap map = new NormalizeCharMap();
+        // we can't add duplicate keys, or NormalizeCharMap gets angry
+        Set<String> keys = new HashSet<String>();
+        int num = random.nextInt(5);
+        //System.out.println("NormalizeCharMap=");
+        for (int i = 0; i < num; i++) {
+          String key = _TestUtil.randomSimpleString(random);
+          if (!keys.contains(key)) {
+            String value = _TestUtil.randomSimpleString(random);
+            map.add(key, value);
+            keys.add(key);
+            //System.out.println("mapping: '" + key + "' => '" + value + "'");
+          }
+        }
+        return map;
+      }
+    });
+    put(CharacterRunAutomaton.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        // TODO: could probably use a purely random automaton
+        switch(random.nextInt(5)) {
+          case 0: return MockTokenizer.KEYWORD;
+          case 1: return MockTokenizer.SIMPLE;
+          case 2: return MockTokenizer.WHITESPACE;
+          case 3: return MockTokenFilter.EMPTY_STOPSET;
+          default: return MockTokenFilter.ENGLISH_STOPSET;
+        }
+      }
+    });
+    put(CharArrayMap.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        int num = random.nextInt(10);
+        CharArrayMap<String> map = new CharArrayMap<String>(TEST_VERSION_CURRENT, num, random.nextBoolean());
+        for (int i = 0; i < num; i++) {
+          // TODO: make nastier
+          map.put(_TestUtil.randomSimpleString(random), _TestUtil.randomSimpleString(random));
+        }
+        return map;
+      }
+    });
+    put(SynonymMap.class, new ArgProducer() {
+      @Override public Object create(Random random) {
+        SynonymMap.Builder b = new SynonymMap.Builder(random.nextBoolean());
+        final int numEntries = atLeast(10);
+        for (int j = 0; j < numEntries; j++) {
+          addSyn(b, randomNonEmptyString(random), randomNonEmptyString(random), random.nextBoolean());
+        }
+        try {
+          return b.build();
+        } catch (Exception ex) {
+          Rethrow.rethrow(ex);
+          return null; // unreachable code
+        }
+      }
+      
+      private void addSyn(SynonymMap.Builder b, String input, String output, boolean keepOrig) {
+        b.add(new CharsRef(input.replaceAll(" +", "\u0000")),
+              new CharsRef(output.replaceAll(" +", "\u0000")),
+              keepOrig);
+      }
+      
+      private String randomNonEmptyString(Random random) {
+        while(true) {
+          final String s = _TestUtil.randomUnicodeString(random).trim();
+          if (s.length() != 0 && s.indexOf('\u0000') == -1) {
+            return s;
+          }
+        }
+      }    
+    });
+  }};
+  
+  static final Set<Class<?>> allowedTokenizerArgs, allowedTokenFilterArgs, allowedCharFilterArgs;
+  static {
+    allowedTokenizerArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+    allowedTokenizerArgs.addAll(argProducers.keySet());
+    allowedTokenizerArgs.add(Reader.class);
+    allowedTokenizerArgs.add(AttributeFactory.class);
+    allowedTokenizerArgs.add(AttributeSource.class);
+    
+    allowedTokenFilterArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+    allowedTokenFilterArgs.addAll(argProducers.keySet());
+    allowedTokenFilterArgs.add(TokenStream.class);
+    // TODO: fix this one, thats broken:
+    allowedTokenFilterArgs.add(CommonGramsFilter.class);
+    
+    allowedCharFilterArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
+    allowedCharFilterArgs.addAll(argProducers.keySet());
+    allowedCharFilterArgs.add(Reader.class);
+    allowedCharFilterArgs.add(CharStream.class);
+  }
+  
+  @SuppressWarnings("unchecked")
+  static <T> T newRandomArg(Random random, Class<T> paramType) {
+    final ArgProducer producer = argProducers.get(paramType);
+    assertNotNull("No producer for arguments of type " + paramType.getName() + " found", producer);
+    return (T) producer.create(random);
+  }
+  
+  static Object[] newTokenizerArgs(Random random, Reader reader, Class<?>[] paramTypes) {
+    Object[] args = new Object[paramTypes.length];
+    for (int i = 0; i < args.length; i++) {
+      Class<?> paramType = paramTypes[i];
+      if (paramType == Reader.class) {
+        args[i] = reader;
+      } else if (paramType == AttributeFactory.class) {
+        // TODO: maybe the collator one...???
+        args[i] = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
+      } else if (paramType == AttributeSource.class) {
+        // TODO: args[i] = new AttributeSource();
+        // this is currently too scary to deal with!
+        args[i] = null; // force IAE
+      } else {
+        args[i] = newRandomArg(random, paramType);
+      }
+    }
+    return args;
+  }
+  
+  static Object[] newCharFilterArgs(Random random, Reader reader, Class<?>[] paramTypes) {
+    Object[] args = new Object[paramTypes.length];
+    for (int i = 0; i < args.length; i++) {
+      Class<?> paramType = paramTypes[i];
+      if (paramType == Reader.class) {
+        args[i] = reader;
+      } else if (paramType == CharStream.class) {
+        args[i] = CharReader.get(reader);
+      } else {
+        args[i] = newRandomArg(random, paramType);
+      }
+    }
+    return args;
+  }
+  
+  static Object[] newFilterArgs(Random random, TokenStream stream, Class<?>[] paramTypes) {
+    Object[] args = new Object[paramTypes.length];
+    for (int i = 0; i < args.length; i++) {
+      Class<?> paramType = paramTypes[i];
+      if (paramType == TokenStream.class) {
+        args[i] = stream;
+      } else if (paramType == CommonGramsFilter.class) {
+        // TODO: fix this one, thats broken: CommonGramsQueryFilter takes this one explicitly
+        args[i] = new CommonGramsFilter(TEST_VERSION_CURRENT, stream, newRandomArg(random, CharArraySet.class));
+      } else {
+        args[i] = newRandomArg(random, paramType);
+      }
+    }
+    return args;
+  }
+
+  static class MockRandomAnalyzer extends Analyzer {
+    final long seed;
+    
+    MockRandomAnalyzer(long seed) {
+      this.seed = seed;
+    }
+
+    public boolean offsetsAreCorrect() {
+      // TODO: can we not do the full chain here!?
+      Random random = new Random(seed);
+      TokenizerSpec tokenizerSpec = newTokenizer(random, new StringReader(""));
+      TokenFilterSpec filterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
+      return filterSpec.offsetsAreCorrect;
+    }
+    
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+      Random random = new Random(seed);
+      TokenizerSpec tokenizerSpec = newTokenizer(random, reader);
+      TokenFilterSpec filterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
+      return new TokenStreamComponents(tokenizerSpec.tokenizer, filterSpec.stream);
+    }
+
+    @Override
+    protected Reader initReader(Reader reader) {
+      Random random = new Random(seed);
+      CharFilterSpec charfilterspec = newCharFilterChain(random, reader);
+      return charfilterspec.reader;
+    }
+
+    @Override
+    public String toString() {
+      Random random = new Random(seed);
+      StringBuilder sb = new StringBuilder();
+      CharFilterSpec charFilterSpec = newCharFilterChain(random, new StringReader(""));
+      sb.append("\ncharfilters=");
+      sb.append(charFilterSpec.toString);
+      // intentional: initReader gets its own separate random
+      random = new Random(seed);
+      TokenizerSpec tokenizerSpec = newTokenizer(random, charFilterSpec.reader);
+      sb.append("\n");
+      sb.append("tokenizer=");
+      sb.append(tokenizerSpec.toString);
+      TokenFilterSpec tokenFilterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
+      sb.append("\n");
+      sb.append("filters=");
+      sb.append(tokenFilterSpec.toString);
+      sb.append("\n");
+      sb.append("offsetsAreCorrect=" + tokenFilterSpec.offsetsAreCorrect);
+      return sb.toString();
+    }
+    
+    private <T> T createComponent(Constructor<T> ctor, Object[] args, StringBuilder descr) {
+      try {
+        final T instance = ctor.newInstance(args);
+        /*
+        if (descr.length() > 0) {
+          descr.append(",");
+        }
+        */
+        descr.append("\n  ");
+        descr.append(ctor.getDeclaringClass().getName());
+        String params = Arrays.toString(args);
+        params = params.substring(1, params.length()-1);
+        descr.append("(").append(params).append(")");
+        return instance;
+      } catch (InvocationTargetException ite) {
+        final Throwable cause = ite.getCause();
+        if (cause instanceof IllegalArgumentException ||
+            cause instanceof UnsupportedOperationException) {
+          // thats ok, ignore
+          if (VERBOSE) {
+            System.err.println("Ignoring IAE/UOE from ctor:");
+            cause.printStackTrace(System.err);
+          }
+        } else {
+          Rethrow.rethrow(cause);
+        }
+      } catch (IllegalAccessException iae) {
+        Rethrow.rethrow(iae);
+      } catch (InstantiationException ie) {
+        Rethrow.rethrow(ie);
+      }
+      return null; // no success
+    }
+    
+    // create a new random tokenizer from classpath
+    private TokenizerSpec newTokenizer(Random random, Reader reader) {
+      TokenizerSpec spec = new TokenizerSpec();
+      while (spec.tokenizer == null) {
+        final Constructor<? extends Tokenizer> ctor = tokenizers.get(random.nextInt(tokenizers.size()));
+        final StringBuilder descr = new StringBuilder();
+        final CheckThatYouDidntReadAnythingReaderWrapper wrapper = new CheckThatYouDidntReadAnythingReaderWrapper(reader);
+        final Object args[] = newTokenizerArgs(random, wrapper, ctor.getParameterTypes());
+        spec.tokenizer = createComponent(ctor, args, descr);
+        if (brokenOffsetsComponents.contains(ctor.getDeclaringClass())) {
+          spec.offsetsAreCorrect = false;
+        }
+        if (spec.tokenizer == null) {
+          assertFalse(ctor.getDeclaringClass().getName() + " has read something in ctor but failed with UOE/IAE", wrapper.readSomething);
+        }
+        spec.toString = descr.toString();
+      }
+      return spec;
+    }
+    
+    private CharFilterSpec newCharFilterChain(Random random, Reader reader) {
+      CharFilterSpec spec = new CharFilterSpec();
+      spec.reader = reader;
+      StringBuilder descr = new StringBuilder();
+      int numFilters = random.nextInt(3);
+      for (int i = 0; i < numFilters; i++) {
+        while (true) {
+          final Constructor<? extends CharStream> ctor = charfilters.get(random.nextInt(charfilters.size()));
+          final Object args[] = newCharFilterArgs(random, spec.reader, ctor.getParameterTypes());
+          reader = createComponent(ctor, args, descr);
+          if (reader != null) {
+            spec.reader = reader;
+            break;
+          }
+        }
+      }
+      spec.toString = descr.toString();
+      return spec;
+    }
+    
+    private TokenFilterSpec newFilterChain(Random random, Tokenizer tokenizer, boolean offsetsAreCorrect) {
+      TokenFilterSpec spec = new TokenFilterSpec();
+      spec.offsetsAreCorrect = offsetsAreCorrect;
+      spec.stream = tokenizer;
+      StringBuilder descr = new StringBuilder();
+      int numFilters = random.nextInt(5);
+      for (int i = 0; i < numFilters; i++) {
+
+        // Insert ValidatingTF after each stage so we can
+        // catch problems right after the TF that "caused"
+        // them:
+        spec.stream = new ValidatingTokenFilter(spec.stream, "stage " + i, spec.offsetsAreCorrect);
+
+        while (true) {
+          final Constructor<? extends TokenFilter> ctor = tokenfilters.get(random.nextInt(tokenfilters.size()));
+          
+          // hack: MockGraph/MockLookahead has assertions that will trip if they follow
+          // an offsets violator. so we cant use them after e.g. wikipediatokenizer
+          if (!spec.offsetsAreCorrect &&
+              (ctor.getDeclaringClass().equals(MockGraphTokenFilter.class)
+               || ctor.getDeclaringClass().equals(MockRandomLookaheadTokenFilter.class))) {
+            continue;
+          }
+          
+          final Object args[] = newFilterArgs(random, spec.stream, ctor.getParameterTypes());
+          final TokenFilter flt = createComponent(ctor, args, descr);
+          if (flt != null) {
+            if (brokenOffsetsComponents.contains(ctor.getDeclaringClass())) {
+              spec.offsetsAreCorrect = false;
+            }
+            spec.stream = flt;
+            break;
+          }
+        }
+      }
+
+      // Insert ValidatingTF after each stage so we can
+      // catch problems right after the TF that "caused"
+      // them:
+      spec.stream = new ValidatingTokenFilter(spec.stream, "last stage", spec.offsetsAreCorrect);
+
+      spec.toString = descr.toString();
+      return spec;
+    }
+  }
+  
+  static final class CheckThatYouDidntReadAnythingReaderWrapper extends CharFilter {
+    boolean readSomething = false;
+    
+    CheckThatYouDidntReadAnythingReaderWrapper(Reader in) {
+      super(CharReader.get(in));
+    }
+
+    @Override
+    public int read(char[] cbuf, int off, int len) throws IOException {
+      readSomething = true;
+      return super.read(cbuf, off, len);
+    }
+
+    @Override
+    public int read() throws IOException {
+      readSomething = true;
+      return super.read();
+    }
+
+    @Override
+    public int read(CharBuffer target) throws IOException {
+      readSomething = true;
+      return super.read(target);
+    }
+
+    @Override
+    public int read(char[] cbuf) throws IOException {
+      readSomething = true;
+      return super.read(cbuf);
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+      readSomething = true;
+      return super.skip(n);
+    }
+  }
+  
+  static class TokenizerSpec {
+    Tokenizer tokenizer;
+    String toString;
+    boolean offsetsAreCorrect = true;
+  }
+  
+  static class TokenFilterSpec {
+    TokenStream stream;
+    String toString;
+    boolean offsetsAreCorrect = true;
+  }
+  
+  static class CharFilterSpec {
+    Reader reader;
+    String toString;
+  }
+  
+  public void testRandomChains() throws Throwable {
+    int numIterations = atLeast(20);
+    for (int i = 0; i < numIterations; i++) {
+      MockRandomAnalyzer a = new MockRandomAnalyzer(random.nextLong());
+      if (VERBOSE) {
+        System.out.println("Creating random analyzer:" + a);
+      }
+      try {
+        checkRandomData(random, a, 1000, 20, false,
+                        false /* We already validate our own offsets... */);
+      } catch (Throwable e) {
+        System.err.println("Exception from random analyzer: " + a);
+        throw e;
+      }
+    }
+  }
 }
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
index 0179b94..e3e8813 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
@@ -65,7 +65,11 @@
         new String[] { "a", "b", "c", "" },
         new int[] { 1, 0, 1, 3 },
         new int[] { 2, 1, 2, 3 },
-        new int[] { 1, 1, 1, 1 });
+        null,
+        new int[] { 1, 1, 1, 1 },
+        null,
+        null,
+        false);
   }
   
   /**
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
index 754116c..54e68ab 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
@@ -72,14 +72,16 @@
     assertTokenStreamContents(wdf, 
         new String[] { "foo", "bar", "foobar" },
         new int[] { 5, 9, 5 }, 
-        new int[] { 8, 12, 12 });
+        new int[] { 8, 12, 12 },
+        null, null, null, null, false);
 
     wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("foo-bar", 5, 6)), DEFAULT_WORD_DELIM_TABLE, flags, null);
     
     assertTokenStreamContents(wdf,
         new String[] { "foo", "bar", "foobar" },
         new int[] { 5, 5, 5 },
-        new int[] { 6, 6, 6 });
+        new int[] { 6, 6, 6 },
+        null, null, null, null, false);
   }
   
   @Test
@@ -123,7 +125,8 @@
     assertTokenStreamContents(wdf,
         new String[] { "foo", "bar", "foobar"},
         new int[] { 8, 12, 8 },
-        new int[] { 11, 15, 15 });
+        new int[] { 11, 15, 15 },
+        null, null, null, null, false);
   }
 
   public void doSplit(final String input, String... output) throws Exception {
@@ -230,18 +233,27 @@
     assertAnalyzesTo(a, "LUCENE / SOLR", new String[] { "LUCENE", "SOLR" },
         new int[] { 0, 9 },
         new int[] { 6, 13 },
-        new int[] { 1, 1 });
+        null,
+        new int[] { 1, 1 },
+        null,
+        false);
     
     /* only in this case, posInc of 2 ?! */
     assertAnalyzesTo(a, "LUCENE / solR", new String[] { "LUCENE", "sol", "R", "solR" },
         new int[] { 0, 9, 12, 9 },
         new int[] { 6, 12, 13, 13 },
-        new int[] { 1, 1, 1, 0 });
+        null,
+        new int[] { 1, 1, 1, 0 },
+        null,
+        false);
     
     assertAnalyzesTo(a, "LUCENE / NUTCH SOLR", new String[] { "LUCENE", "NUTCH", "SOLR" },
         new int[] { 0, 9, 15 },
         new int[] { 6, 14, 19 },
-        new int[] { 1, 1, 1 });
+        null,
+        new int[] { 1, 1, 1 },
+        null,
+        false);
     
     /* analyzer that will consume tokens with large position increments */
     Analyzer a2 = new Analyzer() {
@@ -258,24 +270,36 @@
     assertAnalyzesTo(a2, "LUCENE largegap SOLR", new String[] { "LUCENE", "largegap", "SOLR" },
         new int[] { 0, 7, 16 },
         new int[] { 6, 15, 20 },
-        new int[] { 1, 10, 1 });
+        null,
+        new int[] { 1, 10, 1 },
+        null,
+        false);
     
     /* the "/" had a position increment of 10, where did it go?!?!! */
     assertAnalyzesTo(a2, "LUCENE / SOLR", new String[] { "LUCENE", "SOLR" },
         new int[] { 0, 9 },
         new int[] { 6, 13 },
-        new int[] { 1, 11 });
+        null,
+        new int[] { 1, 11 },
+        null,
+        false);
     
     /* in this case, the increment of 10 from the "/" is carried over */
     assertAnalyzesTo(a2, "LUCENE / solR", new String[] { "LUCENE", "sol", "R", "solR" },
         new int[] { 0, 9, 12, 9 },
         new int[] { 6, 12, 13, 13 },
-        new int[] { 1, 11, 1, 0 });
+        null,
+        new int[] { 1, 11, 1, 0 },
+        null,
+        false);
     
     assertAnalyzesTo(a2, "LUCENE / NUTCH SOLR", new String[] { "LUCENE", "NUTCH", "SOLR" },
         new int[] { 0, 9, 15 },
         new int[] { 6, 14, 19 },
-        new int[] { 1, 11, 1 });
+        null,
+        new int[] { 1, 11, 1 },
+        null,
+        false);
 
     Analyzer a3 = new Analyzer() {
       @Override
@@ -292,14 +316,20 @@
         new String[] { "lucene", "solr", "lucenesolr" },
         new int[] { 0, 7, 0 },
         new int[] { 6, 11, 11 },
-        new int[] { 1, 1, 0 });
+        null,
+        new int[] { 1, 1, 0 },
+        null,
+        false);
 
     /* the stopword should add a gap here */
     assertAnalyzesTo(a3, "the lucene.solr", 
         new String[] { "lucene", "solr", "lucenesolr" }, 
         new int[] { 4, 11, 4 }, 
         new int[] { 10, 15, 15 },
-        new int[] { 2, 1, 0 });
+        null,
+        new int[] { 2, 1, 0 },
+        null,
+        false);
   }
   
   /** blast some random strings through the analyzer */
@@ -322,7 +352,7 @@
           return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(tokenizer, flags, protectedWords));
         }
       };
-      checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+      checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
     }
   }
   
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
index e8e7f6c..adb8870 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
@@ -94,7 +94,15 @@
 
   public void testBackRangeOfNgrams() throws Exception {
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 3);
-    assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5});
+    assertTokenStreamContents(tokenizer,
+                              new String[]{"e","de","cde"},
+                              new int[]{4,3,2},
+                              new int[]{5,5,5},
+                              null,
+                              null,
+                              null,
+                              null,
+                              false);
   }
   
   public void testSmallTokenInStream() throws Exception {
@@ -151,7 +159,7 @@
             new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 15));
       }    
     };
-    checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+    checkRandomData(random, b, 10000*RANDOM_MULTIPLIER, 20, false, false);
   }
   
   public void testEmptyTerm() throws Exception {
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
index 90611a1..158c603 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
@@ -90,7 +90,7 @@
 
   public void testBackRangeOfNgrams() throws Exception {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.BACK, 1, 3);
-    assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5}, 5 /* abcde */);
+    assertTokenStreamContents(tokenizer, new String[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5}, null, null, null, 5 /* abcde */, false);
   }
   
   public void testReset() throws Exception {
@@ -109,8 +109,8 @@
         return new TokenStreamComponents(tokenizer, tokenizer);
       }    
     };
-    checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
-    checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192);
+    checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
+    checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192, false, false);
     
     Analyzer b = new Analyzer() {
       @Override
@@ -119,7 +119,7 @@
         return new TokenStreamComponents(tokenizer, tokenizer);
       }    
     };
-    checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
-    checkRandomData(random, b, 200*RANDOM_MULTIPLIER, 8192);
+    checkRandomData(random, b, 10000*RANDOM_MULTIPLIER, 20, false, false);
+    checkRandomData(random, b, 200*RANDOM_MULTIPLIER, 8192, false, false);
   }
 }
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
index 3375c02..f5f3071 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
@@ -77,7 +77,8 @@
     assertTokenStreamContents(filter,
         new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"}, 
         new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
-        new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5}
+        new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5},
+        null, null, null, null, false
         );
   }
   
@@ -130,7 +131,7 @@
             new NGramTokenFilter(tokenizer, 2, 15));
       }    
     };
-    checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+    checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
   }
   
   public void testEmptyTerm() throws Exception {
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
index 9dd3c65..86a9782 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
@@ -73,7 +73,11 @@
         new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"}, 
         new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2},
         new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5},
-        5 /* abcde */
+        null,
+        null,
+        null,
+        5 /* abcde */,
+        false
         );
   }
   
@@ -98,7 +102,7 @@
         return new TokenStreamComponents(tokenizer, tokenizer);
       }    
     };
-    checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
-    checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192);
+    checkRandomData(random, a, 10000*RANDOM_MULTIPLIER, 20, false, false);
+    checkRandomData(random, a, 200*RANDOM_MULTIPLIER, 8192, false, false);
   }
 }
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
index 36bc262..7791fb4 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
@@ -142,14 +142,16 @@
     }
   }
   
+  /** for testing purposes ONLY */
+  public static String SNOWBALL_LANGS[] = {
+    "Armenian", "Basque", "Catalan", "Danish", "Dutch", "English",
+    "Finnish", "French", "German2", "German", "Hungarian", "Irish",
+    "Italian", "Kp", "Lovins", "Norwegian", "Porter", "Portuguese",
+    "Romanian", "Russian", "Spanish", "Swedish", "Turkish"
+  };
+  
   public void testEmptyTerm() throws IOException {
-    String langs[] = { 
-        "Armenian", "Basque", "Catalan", "Danish", "Dutch", "English",
-        "Finnish", "French", "German2", "German", "Hungarian", "Irish",
-        "Italian", "Kp", "Lovins", "Norwegian", "Porter", "Portuguese",
-        "Romanian", "Russian", "Spanish", "Swedish", "Turkish"
-    };
-    for (final String lang : langs) {
+    for (final String lang : SNOWBALL_LANGS) {
       Analyzer a = new Analyzer() {
         @Override
         protected TokenStreamComponents createComponents(String fieldName, Reader reader) {