| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.3"/> |
| <title>Lucene.Net: Lucene.Net.Analysis.Standard.StandardTokenizer Class Reference</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td id="projectlogo"><img alt="Logo" src="lucene-net-icon-128x128.png"/></td> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Lucene.Net |
|  <span id="projectnumber">3.0.3</span> |
| </div> |
| <div id="projectbrief">Lucene.Net is a .NET port of the Java Lucene Indexing Library</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.3 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="Index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Packages</span></a></li> |
| <li class="current"><a href="annotated.html"><span>Classes</span></a></li> |
| <li><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="annotated.html"><span>Class List</span></a></li> |
| <li><a href="classes.html"><span>Class Index</span></a></li> |
| <li><a href="hierarchy.html"><span>Class Hierarchy</span></a></li> |
| <li><a href="functions.html"><span>Class Members</span></a></li> |
| </ul> |
| </div> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Properties</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="namespace_lucene.html">Lucene</a></li><li class="navelem"><a class="el" href="namespace_lucene_1_1_net.html">Net</a></li><li class="navelem"><a class="el" href="namespace_lucene_1_1_net_1_1_analysis.html">Analysis</a></li><li class="navelem"><a class="el" href="namespace_lucene_1_1_net_1_1_analysis_1_1_standard.html">Standard</a></li><li class="navelem"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html">StandardTokenizer</a></li> </ul> |
| </div> |
| </div><!-- top --> |
| <div class="header"> |
| <div class="summary"> |
| <a href="#pub-methods">Public Member Functions</a> | |
| <a href="#pub-attribs">Public Attributes</a> | |
| <a href="#pub-static-attribs">Static Public Attributes</a> | |
| <a href="#properties">Properties</a> | |
| <a href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer-members.html">List of all members</a> </div> |
| <div class="headertitle"> |
| <div class="title">Lucene.Net.Analysis.Standard.StandardTokenizer Class Reference</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| |
| <p>A grammar-based tokenizer constructed with JFlex |
| <a href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#details">More...</a></p> |
| |
| <p>Inherits <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_tokenizer.html">Lucene.Net.Analysis.Tokenizer</a>.</p> |
| <table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-methods"></a> |
| Public Member Functions</h2></td></tr> |
| <tr class="memitem:a02048366b9999a69eb4657169a2e3332"><td class="memItemLeft" align="right" valign="top"> </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a02048366b9999a69eb4657169a2e3332">StandardTokenizer</a> (<a class="el" href="namespace_lucene_1_1_net_1_1_util.html#a92602ef4dd5a04278f4b62379d552c0b">Version</a> matchVersion, System.IO.TextReader input)</td></tr> |
| <tr class="memdesc:a02048366b9999a69eb4657169a2e3332"><td class="mdescLeft"> </td><td class="mdescRight">Creates a new instance of the <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html" title="A grammar-based tokenizer constructed with JFlex">Lucene.Net.Analysis.Standard.StandardTokenizer</a>. Attaches the <code>input</code> to the newly created JFlex scanner. <a href="#a02048366b9999a69eb4657169a2e3332"></a><br/></td></tr> |
| <tr class="separator:a02048366b9999a69eb4657169a2e3332"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a78141634352f7684c7b86260b963b8ef"><td class="memItemLeft" align="right" valign="top"> </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a78141634352f7684c7b86260b963b8ef">StandardTokenizer</a> (<a class="el" href="namespace_lucene_1_1_net_1_1_util.html#a92602ef4dd5a04278f4b62379d552c0b">Version</a> matchVersion, <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html">AttributeSource</a> source, System.IO.TextReader input)</td></tr> |
| <tr class="memdesc:a78141634352f7684c7b86260b963b8ef"><td class="mdescLeft"> </td><td class="mdescRight">Creates a new <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html" title="A grammar-based tokenizer constructed with JFlex">StandardTokenizer</a> with a given <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html#af8654555ddde74d7e463266371f3508c" title="An AttributeSource using the default attribute factory AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY.">AttributeSource</a>. <a href="#a78141634352f7684c7b86260b963b8ef"></a><br/></td></tr> |
| <tr class="separator:a78141634352f7684c7b86260b963b8ef"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab6157fecfab87f3cfed2c7e850a71bad"><td class="memItemLeft" align="right" valign="top"> </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#ab6157fecfab87f3cfed2c7e850a71bad">StandardTokenizer</a> (<a class="el" href="namespace_lucene_1_1_net_1_1_util.html#a92602ef4dd5a04278f4b62379d552c0b">Version</a> matchVersion, <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source_1_1_attribute_factory.html">AttributeFactory</a> factory, System.IO.TextReader input)</td></tr> |
| <tr class="memdesc:ab6157fecfab87f3cfed2c7e850a71bad"><td class="mdescLeft"> </td><td class="mdescRight">Creates a new <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html" title="A grammar-based tokenizer constructed with JFlex">StandardTokenizer</a> with a given <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source_1_1_attribute_factory.html" title="An AttributeFactory creates instances of Attributes.">Lucene.Net.Util.AttributeSource.AttributeFactory</a> <a href="#ab6157fecfab87f3cfed2c7e850a71bad"></a><br/></td></tr> |
| <tr class="separator:ab6157fecfab87f3cfed2c7e850a71bad"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aac8f7ccf35da37ea8b22e808f885ebb3"><td class="memItemLeft" align="right" valign="top">override bool </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aac8f7ccf35da37ea8b22e808f885ebb3">IncrementToken</a> ()</td></tr> |
| <tr class="memdesc:aac8f7ccf35da37ea8b22e808f885ebb3"><td class="mdescLeft"> </td><td class="mdescRight">Consumers (i.e., <a class="el" href="_f_s_directory_8cs.html#a7d90824eac5b2433e714cc9faeb34d1b">IndexWriter</a>) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute.html" title="Base class for Attributes that can be added to a Lucene.Net.Util.AttributeSource. Attributes are used...">Util.Attribute</a>s with the attributes of the next token. <a href="#aac8f7ccf35da37ea8b22e808f885ebb3"></a><br/></td></tr> |
| <tr class="separator:aac8f7ccf35da37ea8b22e808f885ebb3"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af80e0ac4865731d362aab4f2fc565b6b"><td class="memItemLeft" align="right" valign="top">override void </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#af80e0ac4865731d362aab4f2fc565b6b">End</a> ()</td></tr> |
| <tr class="memdesc:af80e0ac4865731d362aab4f2fc565b6b"><td class="mdescLeft"> </td><td class="mdescRight">This method is called by the consumer after the last token has been consumed, after <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aac8f7ccf35da37ea8b22e808f885ebb3" title="Consumers (i.e., IndexWriter) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate Util.Attributes with the attributes of the next token.">IncrementToken</a> returned <code>false</code> (using the new <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> API). Streams implementing the old API should upgrade to use this feature. This method can be used to perform any end-of-stream operations, such as setting the final offset of a stream. The final offset of a stream might differ from the offset of the last token eg in case one or more whitespaces followed after the last token, but a <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_whitespace_tokenizer.html" title="A WhitespaceTokenizer is a tokenizer that divides text at whitespace. Adjacent sequences of non-White...">WhitespaceTokenizer</a> was used. <a href="#af80e0ac4865731d362aab4f2fc565b6b"></a><br/></td></tr> |
| <tr class="separator:af80e0ac4865731d362aab4f2fc565b6b"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af4cf1aac308b0d820174b49565f9d299"><td class="memItemLeft" align="right" valign="top">override void </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#af4cf1aac308b0d820174b49565f9d299">Reset</a> (System.IO.TextReader reader)</td></tr> |
| <tr class="memdesc:af4cf1aac308b0d820174b49565f9d299"><td class="mdescLeft"> </td><td class="mdescRight">Expert: Reset the tokenizer to a new reader. Typically, an analyzer (in its reusableTokenStream method) will use this to re-use a previously created tokenizer. <a href="#af4cf1aac308b0d820174b49565f9d299"></a><br/></td></tr> |
| <tr class="separator:af4cf1aac308b0d820174b49565f9d299"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a9eaa10d6c355854aeadd054629fa61fa"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a9eaa10d6c355854aeadd054629fa61fa">SetReplaceInvalidAcronym</a> (bool replaceInvalidAcronym)</td></tr> |
| <tr class="memdesc:a9eaa10d6c355854aeadd054629fa61fa"><td class="mdescLeft"> </td><td class="mdescRight">Remove in 3.X and make true the only valid value See <a href="https://issues.apache.org/jira/browse/LUCENE-1068">https://issues.apache.org/jira/browse/LUCENE-1068</a> <a href="#a9eaa10d6c355854aeadd054629fa61fa"></a><br/></td></tr> |
| <tr class="separator:a9eaa10d6c355854aeadd054629fa61fa"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-attribs"></a> |
| Public Attributes</h2></td></tr> |
| <tr class="memitem:ab8b3f27cac49dd561a54e4c8926dc782"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#ab8b3f27cac49dd561a54e4c8926dc782">ALPHANUM</a> = 0</td></tr> |
| <tr class="separator:ab8b3f27cac49dd561a54e4c8926dc782"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa044af66183a0d126bd9e1ee064fe1f3"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aa044af66183a0d126bd9e1ee064fe1f3">APOSTROPHE</a> = 1</td></tr> |
| <tr class="separator:aa044af66183a0d126bd9e1ee064fe1f3"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ad498ad47968bfbe1cecde682fe2afee9"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#ad498ad47968bfbe1cecde682fe2afee9">ACRONYM</a> = 2</td></tr> |
| <tr class="separator:ad498ad47968bfbe1cecde682fe2afee9"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a73e39ba9fe34f6ded54480e551a1220b"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a73e39ba9fe34f6ded54480e551a1220b">COMPANY</a> = 3</td></tr> |
| <tr class="separator:a73e39ba9fe34f6ded54480e551a1220b"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a44c8d56fb651823ad549c37804ea2d97"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a44c8d56fb651823ad549c37804ea2d97">EMAIL</a> = 4</td></tr> |
| <tr class="separator:a44c8d56fb651823ad549c37804ea2d97"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:afa7e66acabf67431f41db927104bd4b9"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#afa7e66acabf67431f41db927104bd4b9">HOST</a> = 5</td></tr> |
| <tr class="separator:afa7e66acabf67431f41db927104bd4b9"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a1fdff72e89a7481da2ab149fb6524b11"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a1fdff72e89a7481da2ab149fb6524b11">NUM</a> = 6</td></tr> |
| <tr class="separator:a1fdff72e89a7481da2ab149fb6524b11"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a3709232be024e7efa024fb7e8146ad60"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a3709232be024e7efa024fb7e8146ad60">CJ</a> = 7</td></tr> |
| <tr class="separator:a3709232be024e7efa024fb7e8146ad60"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa0918667086d927bb194c62c43e4ffb5"><td class="memItemLeft" align="right" valign="top">const int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aa0918667086d927bb194c62c43e4ffb5">ACRONYM_DEP</a> = 8</td></tr> |
| <tr class="separator:aa0918667086d927bb194c62c43e4ffb5"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-static-attribs"></a> |
| Static Public Attributes</h2></td></tr> |
| <tr class="memitem:afc5b8a18e82fe68035f211340dda3611"><td class="memItemLeft" align="right" valign="top">static readonly System.String[] </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#afc5b8a18e82fe68035f211340dda3611">TOKEN_TYPES</a> = new System.String[]{"<ALPHANUM>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aa044af66183a0d126bd9e1ee064fe1f3">APOSTROPHE</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#ad498ad47968bfbe1cecde682fe2afee9">ACRONYM</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a73e39ba9fe34f6ded54480e551a1220b">COMPANY</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a44c8d56fb651823ad549c37804ea2d97">EMAIL</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#afa7e66acabf67431f41db927104bd4b9">HOST</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a1fdff72e89a7481da2ab149fb6524b11">NUM</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a3709232be024e7efa024fb7e8146ad60">CJ</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aa0918667086d927bb194c62c43e4ffb5">ACRONYM_DEP</a>>"}</td></tr> |
| <tr class="memdesc:afc5b8a18e82fe68035f211340dda3611"><td class="mdescLeft"> </td><td class="mdescRight">String token types that correspond to token type int constants <a href="#afc5b8a18e82fe68035f211340dda3611"></a><br/></td></tr> |
| <tr class="separator:afc5b8a18e82fe68035f211340dda3611"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="properties"></a> |
| Properties</h2></td></tr> |
| <tr class="memitem:a8fe89fc92d1e564d9826e0269c730422"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a8fe89fc92d1e564d9826e0269c730422">MaxTokenLength</a><code> [get, set]</code></td></tr> |
| <tr class="memdesc:a8fe89fc92d1e564d9826e0269c730422"><td class="mdescLeft"> </td><td class="mdescRight">Set the max allowed token length. Any token longer than this is skipped. <a href="#a8fe89fc92d1e564d9826e0269c730422"></a><br/></td></tr> |
| <tr class="separator:a8fe89fc92d1e564d9826e0269c730422"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="inherited"></a> |
| Additional Inherited Members</h2></td></tr> |
| <tr class="inherit_header pro_methods_class_lucene_1_1_net_1_1_analysis_1_1_tokenizer"><td colspan="2" onclick="javascript:toggleInherit('pro_methods_class_lucene_1_1_net_1_1_analysis_1_1_tokenizer')"><img src="closed.png" alt="-"/> Protected Member Functions inherited from <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_tokenizer.html">Lucene.Net.Analysis.Tokenizer</a></td></tr> |
| <tr class="memitem:ad3eaa17e4598d5c15aa93741f6088b02 inherit pro_methods_class_lucene_1_1_net_1_1_analysis_1_1_tokenizer"><td class="memItemLeft" align="right" valign="top">override void </td><td class="memItemRight" valign="bottom"><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_tokenizer.html#ad3eaa17e4598d5c15aa93741f6088b02">Dispose</a> (bool disposing)</td></tr> |
| <tr class="separator:ad3eaa17e4598d5c15aa93741f6088b02 inherit pro_methods_class_lucene_1_1_net_1_1_analysis_1_1_tokenizer"><td class="memSeparator" colspan="2"> </td></tr> |
| </table> |
| <a name="details" id="details"></a><h2 class="groupheader">Detailed Description</h2> |
| <div class="textblock"><p>A grammar-based tokenizer constructed with JFlex </p> |
| <p>This should be a good tokenizer for most European-language documents:</p> |
| <ul> |
| <li> |
| Splits words at punctuation characters, removing punctuation. However, a dot that's not followed by whitespace is considered part of a token. </li> |
| <li> |
| Splits words at hyphens, unless there's a number in the token, in which case the whole token is interpreted as a product number and is not split. </li> |
| <li> |
| Recognizes email addresses and internet hostnames as one token. </li> |
| </ul> |
| <p>Many applications have specific tokenizer needs. If this tokenizer does not suit your application, please consider copying this source code directory to your project and maintaining your own grammar-based tokenizer.</p> |
| <p><a class="anchor" id="version"></a> </p> |
| <p>You must specify the required <a class="el" href="_search_files_8cs.html#a63c914dcb7d19365ff0f7c6ea7ef77c4">Version</a> compatibility when creating <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_analyzer.html" title="Filters StandardTokenizer with StandardFilter, LowerCaseFilter and StopFilter, using a list of Englis...">StandardAnalyzer</a>: </p> |
| <ul> |
| <li> |
| As of 2.4, Tokens incorrectly identified as acronyms are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1608</a> </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00056">56</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| </div><h2 class="groupheader">Constructor & Destructor Documentation</h2> |
| <a class="anchor" id="a02048366b9999a69eb4657169a2e3332"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Lucene.Net.Analysis.Standard.StandardTokenizer.StandardTokenizer </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="namespace_lucene_1_1_net_1_1_util.html#a92602ef4dd5a04278f4b62379d552c0b">Version</a> </td> |
| <td class="paramname"><em>matchVersion</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">System.IO.TextReader </td> |
| <td class="paramname"><em>input</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Creates a new instance of the <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html" title="A grammar-based tokenizer constructed with JFlex">Lucene.Net.Analysis.Standard.StandardTokenizer</a>. Attaches the <code>input</code> to the newly created JFlex scanner. </p> |
| <dl class="params"><dt>Parameters</dt><dd> |
| <table class="params"> |
| <tr><td class="paramname">matchVersion</td><td></td></tr> |
| <tr><td class="paramname">input</td><td>The input reader</td></tr> |
| </table> |
| </dd> |
| </dl> |
| <p>See <a href="http://issues.apache.org/jira/browse/LUCENE-1068">http://issues.apache.org/jira/browse/LUCENE-1068</a> </p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00106">106</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a78141634352f7684c7b86260b963b8ef"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Lucene.Net.Analysis.Standard.StandardTokenizer.StandardTokenizer </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="namespace_lucene_1_1_net_1_1_util.html#a92602ef4dd5a04278f4b62379d552c0b">Version</a> </td> |
| <td class="paramname"><em>matchVersion</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html">AttributeSource</a> </td> |
| <td class="paramname"><em>source</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">System.IO.TextReader </td> |
| <td class="paramname"><em>input</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Creates a new <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html" title="A grammar-based tokenizer constructed with JFlex">StandardTokenizer</a> with a given <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source.html#af8654555ddde74d7e463266371f3508c" title="An AttributeSource using the default attribute factory AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY.">AttributeSource</a>.</p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00114">114</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab6157fecfab87f3cfed2c7e850a71bad"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Lucene.Net.Analysis.Standard.StandardTokenizer.StandardTokenizer </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="namespace_lucene_1_1_net_1_1_util.html#a92602ef4dd5a04278f4b62379d552c0b">Version</a> </td> |
| <td class="paramname"><em>matchVersion</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source_1_1_attribute_factory.html">AttributeFactory</a> </td> |
| <td class="paramname"><em>factory</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">System.IO.TextReader </td> |
| <td class="paramname"><em>input</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Creates a new <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html" title="A grammar-based tokenizer constructed with JFlex">StandardTokenizer</a> with a given <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute_source_1_1_attribute_factory.html" title="An AttributeFactory creates instances of Attributes.">Lucene.Net.Util.AttributeSource.AttributeFactory</a> </p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00124">124</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Member Function Documentation</h2> |
| <a class="anchor" id="af80e0ac4865731d362aab4f2fc565b6b"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">override void Lucene.Net.Analysis.Standard.StandardTokenizer.End </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>This method is called by the consumer after the last token has been consumed, after <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aac8f7ccf35da37ea8b22e808f885ebb3" title="Consumers (i.e., IndexWriter) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate Util.Attributes with the attributes of the next token.">IncrementToken</a> returned <code>false</code> (using the new <code><a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></code> API). Streams implementing the old API should upgrade to use this feature. This method can be used to perform any end-of-stream operations, such as setting the final offset of a stream. The final offset of a stream might differ from the offset of the last token eg in case one or more whitespaces followed after the last token, but a <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_whitespace_tokenizer.html" title="A WhitespaceTokenizer is a tokenizer that divides text at whitespace. Adjacent sequences of non-White...">WhitespaceTokenizer</a> was used. </p> |
| <p><throws> IOException </throws> </p> |
| |
| <p>Reimplemented from <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a92b3c4eab12fbf83daa07b7684cfbcaf">Lucene.Net.Analysis.TokenStream</a>.</p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00207">207</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aac8f7ccf35da37ea8b22e808f885ebb3"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">override bool Lucene.Net.Analysis.Standard.StandardTokenizer.IncrementToken </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Consumers (i.e., <a class="el" href="_f_s_directory_8cs.html#a7d90824eac5b2433e714cc9faeb34d1b">IndexWriter</a>) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute.html" title="Base class for Attributes that can be added to a Lucene.Net.Util.AttributeSource. Attributes are used...">Util.Attribute</a>s with the attributes of the next token. </p> |
| <p>The producer must make no assumptions about the attributes after the method has been returned: the caller may arbitrarily change it. If the producer needs to preserve the state for subsequent calls, it can use AttributeSource.CaptureState to create a copy of the current attribute state.</p> |
| <p>This method is called for every token of a document, so an efficient implementation is crucial for good performance. To avoid calls to AttributeSource.AddAttribute{T}() and AttributeSource.GetAttribute{T}(), references to all <a class="el" href="class_lucene_1_1_net_1_1_util_1_1_attribute.html" title="Base class for Attributes that can be added to a Lucene.Net.Util.AttributeSource. Attributes are used...">Util.Attribute</a>s that this stream uses should be retrieved during instantiation.</p> |
| <p>To ensure that filters and consumers know which attributes are available, the attributes must be added during instantiation. Filters and consumers are not required to check for availability of attributes in <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aac8f7ccf35da37ea8b22e808f885ebb3" title="Consumers (i.e., IndexWriter) use this method to advance the stream to the next token. Implementing classes must implement this method and update the appropriate Util.Attributes with the attributes of the next token.">IncrementToken()</a>.</p> |
| <dl class="section return"><dt>Returns</dt><dd>false for end of stream; true otherwise</dd></dl> |
| |
| <p>Implements <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html#a358a544108511440a57b134e722f6027">Lucene.Net.Analysis.TokenStream</a>.</p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00159">159</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af4cf1aac308b0d820174b49565f9d299"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">override void Lucene.Net.Analysis.Standard.StandardTokenizer.Reset </td> |
| <td>(</td> |
| <td class="paramtype">System.IO.TextReader </td> |
| <td class="paramname"><em>input</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Expert: Reset the tokenizer to a new reader. Typically, an analyzer (in its reusableTokenStream method) will use this to re-use a previously created tokenizer. </p> |
| |
| <p>Reimplemented from <a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_tokenizer.html#ab79d84adefa171ac7d1a804153839546">Lucene.Net.Analysis.Tokenizer</a>.</p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00214">214</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a9eaa10d6c355854aeadd054629fa61fa"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void Lucene.Net.Analysis.Standard.StandardTokenizer.SetReplaceInvalidAcronym </td> |
| <td>(</td> |
| <td class="paramtype">bool </td> |
| <td class="paramname"><em>replaceInvalidAcronym</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Remove in 3.X and make true the only valid value See <a href="https://issues.apache.org/jira/browse/LUCENE-1068">https://issues.apache.org/jira/browse/LUCENE-1068</a> </p> |
| <dl class="params"><dt>Parameters</dt><dd> |
| <table class="params"> |
| <tr><td class="paramname">replaceInvalidAcronym</td><td>Set to true to replace mischaracterized acronyms as HOST. </td></tr> |
| </table> |
| </dd> |
| </dl> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00227">227</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Member Data Documentation</h2> |
| <a class="anchor" id="ad498ad47968bfbe1cecde682fe2afee9"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.ACRONYM = 2</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00067">67</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa0918667086d927bb194c62c43e4ffb5"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.ACRONYM_DEP = 8</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p><deprecated> this solves a bug where HOSTs that end with '.' are identified as ACRONYMs. </deprecated> </p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00078">78</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab8b3f27cac49dd561a54e4c8926dc782"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.ALPHANUM = 0</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00065">65</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa044af66183a0d126bd9e1ee064fe1f3"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.APOSTROPHE = 1</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00066">66</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a3709232be024e7efa024fb7e8146ad60"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.CJ = 7</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00072">72</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a73e39ba9fe34f6ded54480e551a1220b"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.COMPANY = 3</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00068">68</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a44c8d56fb651823ad549c37804ea2d97"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.EMAIL = 4</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00069">69</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="afa7e66acabf67431f41db927104bd4b9"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.HOST = 5</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00070">70</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a1fdff72e89a7481da2ab149fb6524b11"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int Lucene.Net.Analysis.Standard.StandardTokenizer.NUM = 6</td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00071">71</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="afc5b8a18e82fe68035f211340dda3611"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">readonly System.String [] Lucene.Net.Analysis.Standard.StandardTokenizer.TOKEN_TYPES = new System.String[]{"<ALPHANUM>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aa044af66183a0d126bd9e1ee064fe1f3">APOSTROPHE</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#ad498ad47968bfbe1cecde682fe2afee9">ACRONYM</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a73e39ba9fe34f6ded54480e551a1220b">COMPANY</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a44c8d56fb651823ad549c37804ea2d97">EMAIL</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#afa7e66acabf67431f41db927104bd4b9">HOST</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a1fdff72e89a7481da2ab149fb6524b11">NUM</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#a3709232be024e7efa024fb7e8146ad60">CJ</a>>", "<<a class="el" href="class_lucene_1_1_net_1_1_analysis_1_1_standard_1_1_standard_tokenizer.html#aa0918667086d927bb194c62c43e4ffb5">ACRONYM_DEP</a>>"}</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>String token types that correspond to token type int constants </p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00081">81</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Property Documentation</h2> |
| <a class="anchor" id="a8fe89fc92d1e564d9826e0269c730422"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int Lucene.Net.Analysis.Standard.StandardTokenizer.MaxTokenLength</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">get</span><span class="mlabel">set</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Set the max allowed token length. Any token longer than this is skipped. </p> |
| |
| <p>Definition at line <a class="el" href="_standard_tokenizer_8cs_source.html#l00091">91</a> of file <a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a>.</p> |
| |
| </div> |
| </div> |
| <hr/>The documentation for this class was generated from the following file:<ul> |
| <li>core/Analysis/Standard/<a class="el" href="_standard_tokenizer_8cs_source.html">StandardTokenizer.cs</a></li> |
| </ul> |
| </div><!-- contents --> |
| <!-- start footer part --> |
| <hr class="footer"/><address class="footer"><small> |
| Generated on Thu Jan 3 2013 02:12:49 for Lucene.Net by  <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/> |
| </a> 1.8.3 |
| </small></address> |
| </body> |
| </html> |
