lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.lucene.analysis;


 import java.io.Closeable;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;

 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.AttributeFactory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.Version;

 /**
  * An Analyzer builds TokenStreams, which analyze text.  It thus represents a
  * policy for extracting index terms from text.
  * <p>
  * In order to define what analysis is done, subclasses must define their
  * {@link TokenStreamComponents TokenStreamComponents} in {@link #createComponents(String)}.
  * The components are then reused in each call to {@link #tokenStream(String, Reader)}.
  * <p>
  * Simple example:
  * <pre class="prettyprint">
  * Analyzer analyzer = new Analyzer() {
  *  {@literal @Override}
  *   protected TokenStreamComponents createComponents(String fieldName) {
  *     Tokenizer source = new FooTokenizer(reader);
  *     TokenStream filter = new FooFilter(source);
  *     filter = new BarFilter(filter);
  *     return new TokenStreamComponents(source, filter);
  *   }
  *   {@literal @Override}
  *   protected TokenStream normalize(TokenStream in) {
  *     // Assuming FooFilter is about normalization and BarFilter is about
  *     // stemming, only FooFilter should be applied
  *     return new FooFilter(in);
  *   }
  * };
  * </pre>
  * For more examples, see the {@link org.apache.lucene.analysis Analysis package documentation}.
  * <p>
  * For some concrete implementations bundled with Lucene, look in the analysis modules:
  * <ul>
  *   <li><a href="{@docRoot}/../analyzers-common/overview-summary.html">Common</a>:
  *       Analyzers for indexing content in different languages and domains.
  *   <li><a href="{@docRoot}/../analyzers-icu/overview-summary.html">ICU</a>:
  *       Exposes functionality from ICU to Apache Lucene.
  *   <li><a href="{@docRoot}/../analyzers-kuromoji/overview-summary.html">Kuromoji</a>:
  *       Morphological analyzer for Japanese text.
  *   <li><a href="{@docRoot}/../analyzers-morfologik/overview-summary.html">Morfologik</a>:
  *       Dictionary-driven lemmatization for the Polish language.
  *   <li><a href="{@docRoot}/../analyzers-phonetic/overview-summary.html">Phonetic</a>:
  *       Analysis for indexing phonetic signatures (for sounds-alike search).
  *   <li><a href="{@docRoot}/../analyzers-smartcn/overview-summary.html">Smart Chinese</a>:
  *       Analyzer for Simplified Chinese, which indexes words.
  *   <li><a href="{@docRoot}/../analyzers-stempel/overview-summary.html">Stempel</a>:
  *       Algorithmic Stemmer for the Polish Language.
  * </ul>
  *
  * @since 3.1
  */
 public abstract class Analyzer implements Closeable {

   private final ReuseStrategy reuseStrategy;
   private Version version = Version.LATEST;

   // non final as it gets nulled if closed; pkg private for access by ReuseStrategy's final helper methods:
   CloseableThreadLocal<Object> storedValue = new CloseableThreadLocal<>();

   /**
    * Create a new Analyzer, reusing the same set of components per-thread
    * across calls to {@link #tokenStream(String, Reader)}.
    */
   public Analyzer() {
     this(GLOBAL_REUSE_STRATEGY);
   }

   /**
    * Expert: create a new Analyzer with a custom {@link ReuseStrategy}.
    * <p>
    * NOTE: if you just want to reuse on a per-field basis, it's easier to
    * use a subclass of {@link AnalyzerWrapper} such as
    * <a href="{@docRoot}/../analyzers-common/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.html">
    * PerFieldAnalyerWrapper</a> instead.
    */
   public Analyzer(ReuseStrategy reuseStrategy) {
     this.reuseStrategy = reuseStrategy;
   }

   /**
    * Creates a new {@link TokenStreamComponents} instance for this analyzer.
    *
    * @param fieldName
    *          the name of the fields content passed to the
    *          {@link TokenStreamComponents} sink as a reader

    * @return the {@link TokenStreamComponents} for this analyzer.
    */
   protected abstract TokenStreamComponents createComponents(String fieldName);

   /**
    * Wrap the given {@link TokenStream} in order to apply normalization filters.
    * The default implementation returns the {@link TokenStream} as-is. This is
    * used by {@link #normalize(String, String)}.
    */
   protected TokenStream normalize(String fieldName, TokenStream in) {
     return in;
   }

   /**
    * Returns a TokenStream suitable for <code>fieldName</code>, tokenizing
    * the contents of <code>reader</code>.
    * <p>
    * This method uses {@link #createComponents(String)} to obtain an
    * instance of {@link TokenStreamComponents}. It returns the sink of the
    * components and stores the components internally. Subsequent calls to this
    * method will reuse the previously stored components after resetting them
    * through {@link TokenStreamComponents#setReader(Reader)}.
    * <p>
    * <b>NOTE:</b> After calling this method, the consumer must follow the
    * workflow described in {@link TokenStream} to properly consume its contents.
    * See the {@link org.apache.lucene.analysis Analysis package documentation} for
    * some examples demonstrating this.
    *
    * <b>NOTE:</b> If your data is available as a {@code String}, use
    * {@link #tokenStream(String, String)} which reuses a {@code StringReader}-like
    * instance internally.
    *
    * @param fieldName the name of the field the created TokenStream is used for
    * @param reader the reader the streams source reads from
    * @return TokenStream for iterating the analyzed content of <code>reader</code>
    * @throws AlreadyClosedException if the Analyzer is closed.
    * @see #tokenStream(String, String)
    */
   public final TokenStream tokenStream(final String fieldName,
                                        final Reader reader) {
     TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
     final Reader r = initReader(fieldName, reader);
     if (components == null) {
       components = createComponents(fieldName);
       reuseStrategy.setReusableComponents(this, fieldName, components);
     }
     components.setReader(r);
     return components.getTokenStream();
   }

   /**
    * Returns a TokenStream suitable for <code>fieldName</code>, tokenizing
    * the contents of <code>text</code>.
    * <p>
    * This method uses {@link #createComponents(String)} to obtain an
    * instance of {@link TokenStreamComponents}. It returns the sink of the
    * components and stores the components internally. Subsequent calls to this
    * method will reuse the previously stored components after resetting them
    * through {@link TokenStreamComponents#setReader(Reader)}.
    * <p>
    * <b>NOTE:</b> After calling this method, the consumer must follow the
    * workflow described in {@link TokenStream} to properly consume its contents.
    * See the {@link org.apache.lucene.analysis Analysis package documentation} for
    * some examples demonstrating this.
    *
    * @param fieldName the name of the field the created TokenStream is used for
    * @param text the String the streams source reads from
    * @return TokenStream for iterating the analyzed content of <code>reader</code>
    * @throws AlreadyClosedException if the Analyzer is closed.
    * @see #tokenStream(String, Reader)
    */
   public final TokenStream tokenStream(final String fieldName, final String text) {
     TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
     @SuppressWarnings("resource") final ReusableStringReader strReader =
         (components == null || components.reusableStringReader == null) ?
         new ReusableStringReader() : components.reusableStringReader;
     strReader.setValue(text);
     final Reader r = initReader(fieldName, strReader);
     if (components == null) {
       components = createComponents(fieldName);
       reuseStrategy.setReusableComponents(this, fieldName, components);
     }

     components.setReader(r);
     components.reusableStringReader = strReader;
     return components.getTokenStream();
   }

   /**
    * Normalize a string down to the representation that it would have in the
    * index.
    * <p>
    * This is typically used by query parsers in order to generate a query on
    * a given term, without tokenizing or stemming, which are undesirable if
    * the string to analyze is a partial word (eg. in case of a wildcard or
    * fuzzy query).
    * <p>
    * This method uses {@link #initReaderForNormalization(String, Reader)} in
    * order to apply necessary character-level normalization and then
    * {@link #normalize(String, TokenStream)} in order to apply the normalizing
    * token filters.
    */
   public final BytesRef normalize(final String fieldName, final String text) {
     try {
       // apply char filters
       final String filteredText;
       try (Reader reader = new StringReader(text)) {
         Reader filterReader = initReaderForNormalization(fieldName, reader);
         char[] buffer = new char[64];
         StringBuilder builder = new StringBuilder();
         for (;;) {
           final int read = filterReader.read(buffer, 0, buffer.length);
           if (read == -1) {
             break;
           }
           builder.append(buffer, 0, read);
         }
         filteredText = builder.toString();
       } catch (IOException e) {
         throw new IllegalStateException("Normalization threw an unexpected exception", e);
       }

       final AttributeFactory attributeFactory = attributeFactory(fieldName);
       try (TokenStream ts = normalize(fieldName,
           new StringTokenStream(attributeFactory, filteredText, text.length()))) {
         final TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
         ts.reset();
         if (ts.incrementToken() == false) {
           throw new IllegalStateException("The normalization token stream is "
               + "expected to produce exactly 1 token, but got 0 for analyzer "
               + this + " and input \"" + text + "\"");
         }
         final BytesRef term = BytesRef.deepCopyOf(termAtt.getBytesRef());
         if (ts.incrementToken()) {
           throw new IllegalStateException("The normalization token stream is "
               + "expected to produce exactly 1 token, but got 2+ for analyzer "
               + this + " and input \"" + text + "\"");
         }
         ts.end();
         return term;
       }
     } catch (IOException e) {
       throw new IllegalStateException("Normalization threw an unexpected exception", e);
     }
   }

   /**
    * Override this if you want to add a CharFilter chain.
    * <p>
    * The default implementation returns <code>reader</code>
    * unchanged.
    *
    * @param fieldName IndexableField name being indexed
    * @param reader original Reader
    * @return reader, optionally decorated with CharFilter(s)
    */
   protected Reader initReader(String fieldName, Reader reader) {
     return reader;
   }

   /** Wrap the given {@link Reader} with {@link CharFilter}s that make sense
    *  for normalization. This is typically a subset of the {@link CharFilter}s
    *  that are applied in {@link #initReader(String, Reader)}. This is used by
    *  {@link #normalize(String, String)}. */
   protected Reader initReaderForNormalization(String fieldName, Reader reader) {
     return reader;
   }

   /** Return the {@link AttributeFactory} to be used for
    *  {@link #tokenStream analysis} and
    *  {@link #normalize(String, String) normalization} on the given
    *  {@code FieldName}. The default implementation returns
    *  {@link TokenStream#DEFAULT_TOKEN_ATTRIBUTE_FACTORY}. */
   protected AttributeFactory attributeFactory(String fieldName) {
     return TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY;
   }

   /**
    * Invoked before indexing a IndexableField instance if
    * terms have already been added to that field.  This allows custom
    * analyzers to place an automatic position increment gap between
    * IndexbleField instances using the same field name.  The default value
    * position increment gap is 0.  With a 0 position increment gap and
    * the typical default token position increment of 1, all terms in a field,
    * including across IndexableField instances, are in successive positions, allowing
    * exact PhraseQuery matches, for instance, across IndexableField instance boundaries.
    *
    * @param fieldName IndexableField name being indexed.
    * @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}.
    *         This value must be {@code >= 0}.
    */
   public int getPositionIncrementGap(String fieldName) {
     return 0;
   }

   /**
    * Just like {@link #getPositionIncrementGap}, except for
    * Token offsets instead.  By default this returns 1.
    * This method is only called if the field
    * produced at least one token for indexing.
    *
    * @param fieldName the field just indexed
    * @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}.
    *         This value must be {@code >= 0}.
    */
   public int getOffsetGap(String fieldName) {
     return 1;
   }

   /**
    * Returns the used {@link ReuseStrategy}.
    */
   public final ReuseStrategy getReuseStrategy() {
     return reuseStrategy;
   }

   /**
    * Set the version of Lucene this analyzer should mimic the behavior for for analysis.
    */
   public void setVersion(Version v) {
     version = v; // TODO: make write once?
   }

   /**
    * Return the version of Lucene this analyzer will mimic the behavior of for analysis.
    */
   public Version getVersion() {
     return version;
   }

   /** Frees persistent resources used by this Analyzer */
   @Override
   public void close() {
     if (storedValue != null) {
       storedValue.close();
       storedValue = null;
     }
   }

   /**
    * This class encapsulates the outer components of a token stream. It provides
    * access to the source (a {@link Reader} {@link Consumer} and the outer end (sink), an
    * instance of {@link TokenFilter} which also serves as the
    * {@link TokenStream} returned by
    * {@link Analyzer#tokenStream(String, Reader)}.
    */
   public static final class TokenStreamComponents {
     /**
      * Original source of the tokens.
      */
     protected final Consumer<Reader> source;
     /**
      * Sink tokenstream, such as the outer tokenfilter decorating
      * the chain. This can be the source if there are no filters.
      */
     protected final TokenStream sink;

     /** Internal cache only used by {@link Analyzer#tokenStream(String, String)}. */
     transient ReusableStringReader reusableStringReader;

     /**
      * Creates a new {@link TokenStreamComponents} instance.
      *
      * @param source
      *          the source to set the reader on
      * @param result
      *          the analyzer's resulting token stream
      */
     public TokenStreamComponents(final Consumer<Reader> source,
         final TokenStream result) {
       this.source = source;
       this.sink = result;
     }

     /**
      * Creates a new {@link TokenStreamComponents} instance
      * @param tokenizer the analyzer's Tokenizer
      * @param result    the analyzer's resulting token stream
      */
     public TokenStreamComponents(final Tokenizer tokenizer, final TokenStream result) {
       this(tokenizer::setReader, result);
     }

     /**
      * Creates a new {@link TokenStreamComponents} from a Tokenizer
      */
     public TokenStreamComponents(final Tokenizer tokenizer) {
       this(tokenizer::setReader, tokenizer);
     }

     /**
      * Resets the encapsulated components with the given reader. If the components
      * cannot be reset, an Exception should be thrown.
      *
      * @param reader
      *          a reader to reset the source component
      */
     private void setReader(final Reader reader) {
       source.accept(reader);
     }

     /**
      * Returns the sink {@link TokenStream}
      *
      * @return the sink {@link TokenStream}
      */
     public TokenStream getTokenStream() {
       return sink;
     }

     /**
      * Returns the component's source
      */
     public Consumer<Reader> getSource() {
       return source;
     }
   }

   /**
    * Strategy defining how TokenStreamComponents are reused per call to
    * {@link Analyzer#tokenStream(String, java.io.Reader)}.
    */
   public static abstract class ReuseStrategy {

     /** Sole constructor. (For invocation by subclass constructors, typically implicit.) */
     public ReuseStrategy() {}

     /**
      * Gets the reusable TokenStreamComponents for the field with the given name.
      *
      * @param analyzer Analyzer from which to get the reused components. Use
      *        {@link #getStoredValue(Analyzer)} and {@link #setStoredValue(Analyzer, Object)}
      *        to access the data on the Analyzer.
      * @param fieldName Name of the field whose reusable TokenStreamComponents
      *        are to be retrieved
      * @return Reusable TokenStreamComponents for the field, or {@code null}
      *         if there was no previous components for the field
      */
     public abstract TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName);

     /**
      * Stores the given TokenStreamComponents as the reusable components for the
      * field with the give name.
      *
      * @param fieldName Name of the field whose TokenStreamComponents are being set
      * @param components TokenStreamComponents which are to be reused for the field
      */
     public abstract void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components);

     /**
      * Returns the currently stored value.
      *
      * @return Currently stored value or {@code null} if no value is stored
      * @throws AlreadyClosedException if the Analyzer is closed.
      */
     protected final Object getStoredValue(Analyzer analyzer) {
       if (analyzer.storedValue == null) {
         throw new AlreadyClosedException("this Analyzer is closed");
       }
       return analyzer.storedValue.get();
     }

     /**
      * Sets the stored value.
      *
      * @param storedValue Value to store
      * @throws AlreadyClosedException if the Analyzer is closed.
      */
     protected final void setStoredValue(Analyzer analyzer, Object storedValue) {
       if (analyzer.storedValue == null) {
         throw new AlreadyClosedException("this Analyzer is closed");
       }
       analyzer.storedValue.set(storedValue);
     }

   }

   /**
    * A predefined {@link ReuseStrategy}  that reuses the same components for
    * every field.
    */
   public static final ReuseStrategy GLOBAL_REUSE_STRATEGY = new ReuseStrategy() {

     @Override
     public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
       return (TokenStreamComponents) getStoredValue(analyzer);
     }

     @Override
     public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
       setStoredValue(analyzer, components);
     }
   };

   /**
    * A predefined {@link ReuseStrategy} that reuses components per-field by
    * maintaining a Map of TokenStreamComponent per field name.
    */
   public static final ReuseStrategy PER_FIELD_REUSE_STRATEGY = new ReuseStrategy() {

     @SuppressWarnings("unchecked")
     @Override
     public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
       Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue(analyzer);
       return componentsPerField != null ? componentsPerField.get(fieldName) : null;
     }

     @SuppressWarnings("unchecked")
     @Override
     public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
       Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue(analyzer);
       if (componentsPerField == null) {
         componentsPerField = new HashMap<>();
         setStoredValue(analyzer, componentsPerField);
       }
       componentsPerField.put(fieldName, components);
     }
   };

   private static final class StringTokenStream extends TokenStream {

     private final String value;
     private final int length;
     private boolean used = true;
     private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
     private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);

     StringTokenStream(AttributeFactory attributeFactory, String value, int length) {
       super(attributeFactory);
       this.value = value;
       this.length = length;
     }

     @Override
     public void reset() {
       used = false;
     }

     @Override
     public boolean incrementToken() {
       if (used) {
         return false;
       }
       clearAttributes();
       termAttribute.append(value);
       offsetAttribute.setOffset(0, length);
       used = true;
       return true;
     }

     @Override
     public void end() throws IOException {
       super.end();
       offsetAttribute.setOffset(length, length);
     }
   }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.lucene.analysis;


	import java.io.Closeable;
	import java.io.IOException;
	import java.io.Reader;
	import java.io.StringReader;
	import java.util.HashMap;
	import java.util.Map;
	import java.util.function.Consumer;

	import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
	import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
	import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
	import org.apache.lucene.store.AlreadyClosedException;
	import org.apache.lucene.util.AttributeFactory;
	import org.apache.lucene.util.BytesRef;
	import org.apache.lucene.util.CloseableThreadLocal;
	import org.apache.lucene.util.Version;

	/**
	* An Analyzer builds TokenStreams, which analyze text. It thus represents a
	* policy for extracting index terms from text.
	* <p>
	* In order to define what analysis is done, subclasses must define their
	* {@link TokenStreamComponents TokenStreamComponents} in {@link #createComponents(String)}.
	* The components are then reused in each call to {@link #tokenStream(String, Reader)}.
	* <p>
	* Simple example:
	* <pre class="prettyprint">
	* Analyzer analyzer = new Analyzer() {
	* {@literal @Override}
	* protected TokenStreamComponents createComponents(String fieldName) {
	* Tokenizer source = new FooTokenizer(reader);
	* TokenStream filter = new FooFilter(source);
	* filter = new BarFilter(filter);
	* return new TokenStreamComponents(source, filter);
	* }
	* {@literal @Override}
	* protected TokenStream normalize(TokenStream in) {
	* // Assuming FooFilter is about normalization and BarFilter is about
	* // stemming, only FooFilter should be applied
	* return new FooFilter(in);
	* }
	* };
	* </pre>
	* For more examples, see the {@link org.apache.lucene.analysis Analysis package documentation}.
	* <p>
	* For some concrete implementations bundled with Lucene, look in the analysis modules:
	* <ul>
	* <li><a href="{@docRoot}/../analyzers-common/overview-summary.html">Common</a>:
	* Analyzers for indexing content in different languages and domains.
	* <li><a href="{@docRoot}/../analyzers-icu/overview-summary.html">ICU</a>:
	* Exposes functionality from ICU to Apache Lucene.
	* <li><a href="{@docRoot}/../analyzers-kuromoji/overview-summary.html">Kuromoji</a>:
	* Morphological analyzer for Japanese text.
	* <li><a href="{@docRoot}/../analyzers-morfologik/overview-summary.html">Morfologik</a>:
	* Dictionary-driven lemmatization for the Polish language.
	* <li><a href="{@docRoot}/../analyzers-phonetic/overview-summary.html">Phonetic</a>:
	* Analysis for indexing phonetic signatures (for sounds-alike search).
	* <li><a href="{@docRoot}/../analyzers-smartcn/overview-summary.html">Smart Chinese</a>:
	* Analyzer for Simplified Chinese, which indexes words.
	* <li><a href="{@docRoot}/../analyzers-stempel/overview-summary.html">Stempel</a>:
	* Algorithmic Stemmer for the Polish Language.
	* </ul>
	*
	* @since 3.1
	*/
	public abstract class Analyzer implements Closeable {

	private final ReuseStrategy reuseStrategy;
	private Version version = Version.LATEST;

	// non final as it gets nulled if closed; pkg private for access by ReuseStrategy's final helper methods:
	CloseableThreadLocal<Object> storedValue = new CloseableThreadLocal<>();

	/**
	* Create a new Analyzer, reusing the same set of components per-thread
	* across calls to {@link #tokenStream(String, Reader)}.
	*/
	public Analyzer() {
	this(GLOBAL_REUSE_STRATEGY);
	}

	/**
	* Expert: create a new Analyzer with a custom {@link ReuseStrategy}.
	* <p>
	* NOTE: if you just want to reuse on a per-field basis, it's easier to
	* use a subclass of {@link AnalyzerWrapper} such as
	* <a href="{@docRoot}/../analyzers-common/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.html">
	* PerFieldAnalyerWrapper</a> instead.
	*/
	public Analyzer(ReuseStrategy reuseStrategy) {
	this.reuseStrategy = reuseStrategy;
	}

	/**
	* Creates a new {@link TokenStreamComponents} instance for this analyzer.
	*
	* @param fieldName
	* the name of the fields content passed to the
	* {@link TokenStreamComponents} sink as a reader

	* @return the {@link TokenStreamComponents} for this analyzer.
	*/
	protected abstract TokenStreamComponents createComponents(String fieldName);

	/**
	* Wrap the given {@link TokenStream} in order to apply normalization filters.
	* The default implementation returns the {@link TokenStream} as-is. This is
	* used by {@link #normalize(String, String)}.
	*/
	protected TokenStream normalize(String fieldName, TokenStream in) {
	return in;
	}

	/**
	* Returns a TokenStream suitable for <code>fieldName</code>, tokenizing
	* the contents of <code>reader</code>.
	* <p>
	* This method uses {@link #createComponents(String)} to obtain an
	* instance of {@link TokenStreamComponents}. It returns the sink of the
	* components and stores the components internally. Subsequent calls to this
	* method will reuse the previously stored components after resetting them
	* through {@link TokenStreamComponents#setReader(Reader)}.
	* <p>
	* <b>NOTE:</b> After calling this method, the consumer must follow the
	* workflow described in {@link TokenStream} to properly consume its contents.
	* See the {@link org.apache.lucene.analysis Analysis package documentation} for
	* some examples demonstrating this.
	*
	* <b>NOTE:</b> If your data is available as a {@code String}, use
	* {@link #tokenStream(String, String)} which reuses a {@code StringReader}-like
	* instance internally.
	*
	* @param fieldName the name of the field the created TokenStream is used for
	* @param reader the reader the streams source reads from
	* @return TokenStream for iterating the analyzed content of <code>reader</code>
	* @throws AlreadyClosedException if the Analyzer is closed.
	* @see #tokenStream(String, String)
	*/
	public final TokenStream tokenStream(final String fieldName,
	final Reader reader) {
	TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
	final Reader r = initReader(fieldName, reader);
	if (components == null) {
	components = createComponents(fieldName);
	reuseStrategy.setReusableComponents(this, fieldName, components);
	}
	components.setReader(r);
	return components.getTokenStream();
	}

	/**
	* Returns a TokenStream suitable for <code>fieldName</code>, tokenizing
	* the contents of <code>text</code>.
	* <p>
	* This method uses {@link #createComponents(String)} to obtain an
	* instance of {@link TokenStreamComponents}. It returns the sink of the
	* components and stores the components internally. Subsequent calls to this
	* method will reuse the previously stored components after resetting them
	* through {@link TokenStreamComponents#setReader(Reader)}.
	* <p>
	* <b>NOTE:</b> After calling this method, the consumer must follow the
	* workflow described in {@link TokenStream} to properly consume its contents.
	* See the {@link org.apache.lucene.analysis Analysis package documentation} for
	* some examples demonstrating this.
	*
	* @param fieldName the name of the field the created TokenStream is used for
	* @param text the String the streams source reads from
	* @return TokenStream for iterating the analyzed content of <code>reader</code>
	* @throws AlreadyClosedException if the Analyzer is closed.
	* @see #tokenStream(String, Reader)
	*/
	public final TokenStream tokenStream(final String fieldName, final String text) {
	TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
	@SuppressWarnings("resource") final ReusableStringReader strReader =
	(components == null \|\| components.reusableStringReader == null) ?
	new ReusableStringReader() : components.reusableStringReader;
	strReader.setValue(text);
	final Reader r = initReader(fieldName, strReader);
	if (components == null) {
	components = createComponents(fieldName);
	reuseStrategy.setReusableComponents(this, fieldName, components);
	}

	components.setReader(r);
	components.reusableStringReader = strReader;
	return components.getTokenStream();
	}

	/**
	* Normalize a string down to the representation that it would have in the
	* index.
	* <p>
	* This is typically used by query parsers in order to generate a query on
	* a given term, without tokenizing or stemming, which are undesirable if
	* the string to analyze is a partial word (eg. in case of a wildcard or
	* fuzzy query).
	* <p>
	* This method uses {@link #initReaderForNormalization(String, Reader)} in
	* order to apply necessary character-level normalization and then
	* {@link #normalize(String, TokenStream)} in order to apply the normalizing
	* token filters.
	*/
	public final BytesRef normalize(final String fieldName, final String text) {
	try {
	// apply char filters
	final String filteredText;
	try (Reader reader = new StringReader(text)) {
	Reader filterReader = initReaderForNormalization(fieldName, reader);
	char[] buffer = new char[64];
	StringBuilder builder = new StringBuilder();
	for (;;) {
	final int read = filterReader.read(buffer, 0, buffer.length);
	if (read == -1) {
	break;
	}
	builder.append(buffer, 0, read);
	}
	filteredText = builder.toString();
	} catch (IOException e) {
	throw new IllegalStateException("Normalization threw an unexpected exception", e);
	}

	final AttributeFactory attributeFactory = attributeFactory(fieldName);
	try (TokenStream ts = normalize(fieldName,
	new StringTokenStream(attributeFactory, filteredText, text.length()))) {
	final TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
	ts.reset();
	if (ts.incrementToken() == false) {
	throw new IllegalStateException("The normalization token stream is "
	+ "expected to produce exactly 1 token, but got 0 for analyzer "
	+ this + " and input \"" + text + "\"");
	}
	final BytesRef term = BytesRef.deepCopyOf(termAtt.getBytesRef());
	if (ts.incrementToken()) {
	throw new IllegalStateException("The normalization token stream is "
	+ "expected to produce exactly 1 token, but got 2+ for analyzer "
	+ this + " and input \"" + text + "\"");
	}
	ts.end();
	return term;
	}
	} catch (IOException e) {
	throw new IllegalStateException("Normalization threw an unexpected exception", e);
	}
	}

	/**
	* Override this if you want to add a CharFilter chain.
	* <p>
	* The default implementation returns <code>reader</code>
	* unchanged.
	*
	* @param fieldName IndexableField name being indexed
	* @param reader original Reader
	* @return reader, optionally decorated with CharFilter(s)
	*/
	protected Reader initReader(String fieldName, Reader reader) {
	return reader;
	}

	/** Wrap the given {@link Reader} with {@link CharFilter}s that make sense
	* for normalization. This is typically a subset of the {@link CharFilter}s
	* that are applied in {@link #initReader(String, Reader)}. This is used by
	* {@link #normalize(String, String)}. */
	protected Reader initReaderForNormalization(String fieldName, Reader reader) {
	return reader;
	}

	/** Return the {@link AttributeFactory} to be used for
	* {@link #tokenStream analysis} and
	* {@link #normalize(String, String) normalization} on the given
	* {@code FieldName}. The default implementation returns
	* {@link TokenStream#DEFAULT_TOKEN_ATTRIBUTE_FACTORY}. */
	protected AttributeFactory attributeFactory(String fieldName) {
	return TokenStream.DEFAULT_TOKEN_ATTRIBUTE_FACTORY;
	}

	/**
	* Invoked before indexing a IndexableField instance if
	* terms have already been added to that field. This allows custom
	* analyzers to place an automatic position increment gap between
	* IndexbleField instances using the same field name. The default value
	* position increment gap is 0. With a 0 position increment gap and
	* the typical default token position increment of 1, all terms in a field,
	* including across IndexableField instances, are in successive positions, allowing
	* exact PhraseQuery matches, for instance, across IndexableField instance boundaries.
	*
	* @param fieldName IndexableField name being indexed.
	* @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}.
	* This value must be {@code >= 0}.
	*/
	public int getPositionIncrementGap(String fieldName) {
	return 0;
	}

	/**
	* Just like {@link #getPositionIncrementGap}, except for
	* Token offsets instead. By default this returns 1.
	* This method is only called if the field
	* produced at least one token for indexing.
	*
	* @param fieldName the field just indexed
	* @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}.
	* This value must be {@code >= 0}.
	*/
	public int getOffsetGap(String fieldName) {
	return 1;
	}

	/**
	* Returns the used {@link ReuseStrategy}.
	*/
	public final ReuseStrategy getReuseStrategy() {
	return reuseStrategy;
	}

	/**
	* Set the version of Lucene this analyzer should mimic the behavior for for analysis.
	*/
	public void setVersion(Version v) {
	version = v; // TODO: make write once?
	}

	/**
	* Return the version of Lucene this analyzer will mimic the behavior of for analysis.
	*/
	public Version getVersion() {
	return version;
	}

	/** Frees persistent resources used by this Analyzer */
	@Override
	public void close() {
	if (storedValue != null) {
	storedValue.close();
	storedValue = null;
	}
	}

	/**
	* This class encapsulates the outer components of a token stream. It provides
	* access to the source (a {@link Reader} {@link Consumer} and the outer end (sink), an
	* instance of {@link TokenFilter} which also serves as the
	* {@link TokenStream} returned by
	* {@link Analyzer#tokenStream(String, Reader)}.
	*/
	public static final class TokenStreamComponents {
	/**
	* Original source of the tokens.
	*/
	protected final Consumer<Reader> source;
	/**
	* Sink tokenstream, such as the outer tokenfilter decorating
	* the chain. This can be the source if there are no filters.
	*/
	protected final TokenStream sink;

	/** Internal cache only used by {@link Analyzer#tokenStream(String, String)}. */
	transient ReusableStringReader reusableStringReader;

	/**
	* Creates a new {@link TokenStreamComponents} instance.
	*
	* @param source
	* the source to set the reader on
	* @param result
	* the analyzer's resulting token stream
	*/
	public TokenStreamComponents(final Consumer<Reader> source,
	final TokenStream result) {
	this.source = source;
	this.sink = result;
	}

	/**
	* Creates a new {@link TokenStreamComponents} instance
	* @param tokenizer the analyzer's Tokenizer
	* @param result the analyzer's resulting token stream
	*/
	public TokenStreamComponents(final Tokenizer tokenizer, final TokenStream result) {
	this(tokenizer::setReader, result);
	}

	/**
	* Creates a new {@link TokenStreamComponents} from a Tokenizer
	*/
	public TokenStreamComponents(final Tokenizer tokenizer) {
	this(tokenizer::setReader, tokenizer);
	}

	/**
	* Resets the encapsulated components with the given reader. If the components
	* cannot be reset, an Exception should be thrown.
	*
	* @param reader
	* a reader to reset the source component
	*/
	private void setReader(final Reader reader) {
	source.accept(reader);
	}

	/**
	* Returns the sink {@link TokenStream}
	*
	* @return the sink {@link TokenStream}
	*/
	public TokenStream getTokenStream() {
	return sink;
	}

	/**
	* Returns the component's source
	*/
	public Consumer<Reader> getSource() {
	return source;
	}
	}

	/**
	* Strategy defining how TokenStreamComponents are reused per call to
	* {@link Analyzer#tokenStream(String, java.io.Reader)}.
	*/
	public static abstract class ReuseStrategy {

	/** Sole constructor. (For invocation by subclass constructors, typically implicit.) */
	public ReuseStrategy() {}

	/**
	* Gets the reusable TokenStreamComponents for the field with the given name.
	*
	* @param analyzer Analyzer from which to get the reused components. Use
	* {@link #getStoredValue(Analyzer)} and {@link #setStoredValue(Analyzer, Object)}
	* to access the data on the Analyzer.
	* @param fieldName Name of the field whose reusable TokenStreamComponents
	* are to be retrieved
	* @return Reusable TokenStreamComponents for the field, or {@code null}
	* if there was no previous components for the field
	*/
	public abstract TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName);

	/**
	* Stores the given TokenStreamComponents as the reusable components for the
	* field with the give name.
	*
	* @param fieldName Name of the field whose TokenStreamComponents are being set
	* @param components TokenStreamComponents which are to be reused for the field
	*/
	public abstract void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components);

	/**
	* Returns the currently stored value.
	*
	* @return Currently stored value or {@code null} if no value is stored
	* @throws AlreadyClosedException if the Analyzer is closed.
	*/
	protected final Object getStoredValue(Analyzer analyzer) {
	if (analyzer.storedValue == null) {
	throw new AlreadyClosedException("this Analyzer is closed");
	}
	return analyzer.storedValue.get();
	}

	/**
	* Sets the stored value.
	*
	* @param storedValue Value to store
	* @throws AlreadyClosedException if the Analyzer is closed.
	*/
	protected final void setStoredValue(Analyzer analyzer, Object storedValue) {
	if (analyzer.storedValue == null) {
	throw new AlreadyClosedException("this Analyzer is closed");
	}
	analyzer.storedValue.set(storedValue);
	}

	}

	/**
	* A predefined {@link ReuseStrategy} that reuses the same components for
	* every field.
	*/
	public static final ReuseStrategy GLOBAL_REUSE_STRATEGY = new ReuseStrategy() {

	@Override
	public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
	return (TokenStreamComponents) getStoredValue(analyzer);
	}

	@Override
	public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
	setStoredValue(analyzer, components);
	}
	};

	/**
	* A predefined {@link ReuseStrategy} that reuses components per-field by
	* maintaining a Map of TokenStreamComponent per field name.
	*/
	public static final ReuseStrategy PER_FIELD_REUSE_STRATEGY = new ReuseStrategy() {

	@SuppressWarnings("unchecked")
	@Override
	public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
	Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue(analyzer);
	return componentsPerField != null ? componentsPerField.get(fieldName) : null;
	}

	@SuppressWarnings("unchecked")
	@Override
	public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
	Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue(analyzer);
	if (componentsPerField == null) {
	componentsPerField = new HashMap<>();
	setStoredValue(analyzer, componentsPerField);
	}
	componentsPerField.put(fieldName, components);
	}
	};

	private static final class StringTokenStream extends TokenStream {

	private final String value;
	private final int length;
	private boolean used = true;
	private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
	private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);

	StringTokenStream(AttributeFactory attributeFactory, String value, int length) {
	super(attributeFactory);
	this.value = value;
	this.length = length;
	}

	@Override
	public void reset() {
	used = false;
	}

	@Override
	public boolean incrementToken() {
	if (used) {
	return false;
	}
	clearAttributes();
	termAttribute.append(value);
	offsetAttribute.setOffset(0, length);
	used = true;
	return true;
	}

	@Override
	public void end() throws IOException {
	super.end();
	offsetAttribute.setOffset(length, length);
	}
	}
	}