| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.3"/> |
| <title>Lucene.Net: core/Analysis/TokenStream.cs File Reference</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td id="projectlogo"><img alt="Logo" src="lucene-net-icon-128x128.png"/></td> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Lucene.Net |
|  <span id="projectnumber">3.0.3</span> |
| </div> |
| <div id="projectbrief">Lucene.Net is a .NET port of the Java Lucene Indexing Library</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.3 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="Index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Packages</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Properties</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_1efe588324fc547683bba4a3f1bac847.html">core</a></li><li class="navelem"><a class="el" href="dir_6336841c10663a412729e336c4986ba5.html">Analysis</a></li> </ul> |
| </div> |
| </div><!-- top --> |
| <div class="header"> |
| <div class="summary"> |
| <a href="#nested-classes">Classes</a> | |
| <a href="#namespaces">Namespaces</a> | |
| <a href="#typedef-members">Typedefs</a> </div> |
| <div class="headertitle"> |
| <div class="title">TokenStream.cs File Reference</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| |
| <p><a href="_token_stream_8cs_source.html">Go to the source code of this file.</a></p> |
| <table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="nested-classes"></a> |
| Classes</h2></td></tr> |
| <tr class="memitem:"><td class="memItemLeft" align="right" valign="top">class  </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html">Lucene.Net.Analysis.TokenStream</a></td></tr> |
| <tr class="memdesc:"><td class="mdescLeft"> </td><td class="mdescRight">A <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> enumerates the sequence of tokens, either from <a class="el" href="_token_stream_8cs.html#a6d7b175c6799cf1d18e9493727d23fc6">Field</a>s of a <a class="el" href="_searcher_8cs.html#a2ad6aef8641c7b7a6cf9277f7d54b2cd">Document</a> or from query text. This is an abstract class. Concrete subclasses are: </p> |
| <ul> |
| <li> |
| <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_tokenizer.html" title="A Tokenizer is a TokenStream whose input is a Reader. This is an abstract class; subclasses must over...">Tokenizer</a>, a <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> whose input is a Reader; and </li> |
| <li> |
| <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_filter.html" title="A TokenFilter is a TokenStream whose input is another TokenStream. This is an abstract class; subclas...">TokenFilter</a>, a <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> whose input is another <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code>. </li> |
| </ul> |
| <p>A new <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> API has been introduced with <a class="el" href="namespace_lucene.html">Lucene</a> 2.9. This API has moved from being <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a> based to IAttribute based. While <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a> still exists in 2.9 as a convenience class, the preferred way to store the information of a <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a> is to use <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute.html" title="Base class for Attributes that can be added to a Lucene.Net.Util.AttributeSource. Attributes are used...">Util.Attribute</a>s. <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> now extends <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html#af8654555ddde74d7e463266371f3508c" title="An AttributeSource using the default attribute factory AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY.">AttributeSource</a>, which provides access to all of the token IAttributes for the <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code>. Note that only one instance per <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute.html" title="Base class for Attributes that can be added to a Lucene.Net.Util.AttributeSource. Attributes are used...">Util.Attribute</a> is created and reused for every token. This approach reduces object creation and allows local caching of references to the <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute.html" title="Base class for Attributes that can be added to a Lucene.Net.Util.AttributeSource. Attributes are used...">Util.Attribute</a>s. See <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a358a544108511440a57b134e722f6027" title="Consumers (i.e., IndexWriter) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate Util.Attributes with the attributes of the next token.">IncrementToken()</a> for further details. <b>The workflow of the new <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> API is as follows:</b> </p> |
| <ul> |
| <li> |
| Instantiation of <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code>/<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_filter.html" title="A TokenFilter is a TokenStream whose input is another TokenStream. This is an abstract class; subclas...">TokenFilter</a>s which add/get attributes to/from the <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html#af8654555ddde74d7e463266371f3508c" title="An AttributeSource using the default attribute factory AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY.">AttributeSource</a>. </li> |
| <li> |
| The consumer calls <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#af7c4a3464dc78b1d0451492d9c520965" title="Resets this stream to the beginning. This is an optional operation, so subclasses may or may not impl...">TokenStream.Reset()</a>. </li> |
| <li> |
| The consumer retrieves attributes from the stream and stores local references to all attributes it wants to access </li> |
| <li> |
| The consumer calls <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a358a544108511440a57b134e722f6027" title="Consumers (i.e., IndexWriter) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate Util.Attributes with the attributes of the next token.">IncrementToken()</a> until it returns false and consumes the attributes after each call. </li> |
| <li> |
| The consumer calls <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a92b3c4eab12fbf83daa07b7684cfbcaf" title="This method is called by the consumer after the last token has been consumed, after IncrementToken re...">End()</a> so that any end-of-stream operations can be performed. </li> |
| <li> |
| The consumer calls <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a5f66ac4c37218611b4a13f81f98332cd" title="Releases resources associated with this stream. ">Close()</a> to release any resource when finished using the <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> </li> |
| </ul> |
| <p>To make sure that filters and consumers know which attributes are available, the attributes must be added during instantiation. Filters and consumers are not required to check for availability of attributes in <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a358a544108511440a57b134e722f6027" title="Consumers (i.e., IndexWriter) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate Util.Attributes with the attributes of the next token.">IncrementToken()</a>. You can find some example code for the new API in the analysis package level Javadoc. Sometimes it is desirable to capture a current state of a <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> , e. g. for buffering purposes (see <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_caching_token_filter.html" title="This class can be used if the token attributes of a TokenStream are intended to be consumed more than...">CachingTokenFilter</a>, <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_tee_sink_token_filter.html" title="This TokenFilter provides the ability to set aside attribute states that have already been analyzed...">TeeSinkTokenFilter</a>). For this usecase AttributeSource.CaptureState and AttributeSource.RestoreState can be used. <a href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#details">More...</a><br/></td></tr> |
| <tr class="separator:"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="namespaces"></a> |
| Namespaces</h2></td></tr> |
| <tr class="memitem:namespace_lucene_1_1_net_1_1_analysis"><td class="memItemLeft" align="right" valign="top">package  </td><td class="memItemRight" valign="bottom"><a class="el" href="namespace_lucene_1_1_net_1_1_analysis.html">Lucene.Net.Analysis</a></td></tr> |
| <tr class="separator:"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="typedef-members"></a> |
| Typedefs</h2></td></tr> |
| <tr class="memitem:a2ad6aef8641c7b7a6cf9277f7d54b2cd"><td class="memItemLeft" align="right" valign="top">using </td><td class="memItemRight" valign="bottom"><a class="el" href="_token_stream_8cs.html#a2ad6aef8641c7b7a6cf9277f7d54b2cd">Document</a> = <a class="el" href="class_lucene_1_1_net_1_1_documents_1_1_document.html">Lucene.Net.Documents.Document</a></td></tr> |
| <tr class="separator:a2ad6aef8641c7b7a6cf9277f7d54b2cd"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a6d7b175c6799cf1d18e9493727d23fc6"><td class="memItemLeft" align="right" valign="top">using </td><td class="memItemRight" valign="bottom"><a class="el" href="_token_stream_8cs.html#a6d7b175c6799cf1d18e9493727d23fc6">Field</a> = <a class="el" href="class_lucene_1_1_net_1_1_documents_1_1_field.html">Lucene.Net.Documents.Field</a></td></tr> |
| <tr class="separator:a6d7b175c6799cf1d18e9493727d23fc6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a7d90824eac5b2433e714cc9faeb34d1b"><td class="memItemLeft" align="right" valign="top">using </td><td class="memItemRight" valign="bottom"><a class="el" href="_token_stream_8cs.html#a7d90824eac5b2433e714cc9faeb34d1b">IndexWriter</a> = <a class="el" href="class_lucene_1_1_net_1_1_index_1_1_index_writer.html">Lucene.Net.Index.IndexWriter</a></td></tr> |
| <tr class="separator:a7d90824eac5b2433e714cc9faeb34d1b"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aed60c4e663aea1a7dec59cf4c3710058"><td class="memItemLeft" align="right" valign="top">using </td><td class="memItemRight" valign="bottom"><a class="el" href="_token_stream_8cs.html#aed60c4e663aea1a7dec59cf4c3710058">AttributeSource</a> = <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html">Lucene.Net.Util.AttributeSource</a></td></tr> |
| <tr class="separator:aed60c4e663aea1a7dec59cf4c3710058"><td class="memSeparator" colspan="2"> </td></tr> |
| </table> |
| <h2 class="groupheader">Typedef Documentation</h2> |
| <a class="anchor" id="aed60c4e663aea1a7dec59cf4c3710058"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">using <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html">AttributeSource</a> = <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html">Lucene.Net.Util.AttributeSource</a></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_token_stream_8cs_source.html#l00023">23</a> of file <a class="el" href="_token_stream_8cs_source.html">TokenStream.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a2ad6aef8641c7b7a6cf9277f7d54b2cd"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">using <a class="el" href="_more_like_this_8cs.html#a2ad6aef8641c7b7a6cf9277f7d54b2cd">Document</a> = <a class="el" href="class_lucene_1_1_net_1_1_documents_1_1_document.html">Lucene.Net.Documents.Document</a></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_token_stream_8cs_source.html#l00020">20</a> of file <a class="el" href="_token_stream_8cs_source.html">TokenStream.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a6d7b175c6799cf1d18e9493727d23fc6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">using <a class="el" href="_syns2_index_8cs.html#a6d7b175c6799cf1d18e9493727d23fc6">Field</a> = <a class="el" href="class_lucene_1_1_net_1_1_documents_1_1_field.html">Lucene.Net.Documents.Field</a></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_token_stream_8cs_source.html#l00021">21</a> of file <a class="el" href="_token_stream_8cs_source.html">TokenStream.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a7d90824eac5b2433e714cc9faeb34d1b"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">using <a class="el" href="_syns2_index_8cs.html#a7d90824eac5b2433e714cc9faeb34d1b">IndexWriter</a> = <a class="el" href="class_lucene_1_1_net_1_1_index_1_1_index_writer.html">Lucene.Net.Index.IndexWriter</a></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_token_stream_8cs_source.html#l00022">22</a> of file <a class="el" href="_token_stream_8cs_source.html">TokenStream.cs</a>.</p> |
| |
| </div> |
| </div> |
| </div><!-- contents --> |
| <!-- start footer part --> |
| <hr class="footer"/><address class="footer"><small> |
| Generated on Thu Jan 3 2013 02:12:47 for Lucene.Net by  <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/> |
| </a> 1.8.3 |
| </small></address> |
| </body> |
| </html> |
