﻿<html xmlns:MSHelp="http://msdn.microsoft.com/mshelp" xmlns:mshelp="http://msdn.microsoft.com/mshelp" xmlns:ddue="http://ddue.schemas.microsoft.com/authoring/2003/5" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:msxsl="urn:schemas-microsoft-com:xslt"><head><META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8" /><META NAME="save" CONTENT="history" /><title>Tokenizer Class</title><meta name="Language" content="en-us" /><meta name="System.Keywords" content="Tokenizer class" /><meta name="System.Keywords" content="Lucene.Net.Analysis.Tokenizer class" /><meta name="System.Keywords" content="Tokenizer class, about Tokenizer class" /><meta name="Microsoft.Help.F1" content="Lucene.Net.Analysis.Tokenizer" /><meta name="Microsoft.Help.Id" content="T:Lucene.Net.Analysis.Tokenizer" /><meta name="Description" content="A Tokenizer is a TokenStream whose input is a Reader." /><meta name="Microsoft.Help.ContentType" content="Reference" /><link rel="stylesheet" type="text/css" href="../styles/Presentation.css" /><link rel="stylesheet" type="text/css" href="ms-help://Hx/HxRuntime/HxLink.css" /><script type="text/javascript" src="../scripts/EventUtilities.js"> </script><script type="text/javascript" src="../scripts/SplitScreen.js"> </script><script type="text/javascript" src="../scripts/Dropdown.js"> </script><script type="text/javascript" src="../scripts/script_manifold.js"> </script><script type="text/javascript" src="../scripts/script_feedBack.js"> </script><script type="text/javascript" src="../scripts/CheckboxMenu.js"> </script><script type="text/javascript" src="../scripts/CommonUtilities.js"> </script><meta name="container" content="Lucene.Net.Analysis" /><meta name="file" content="d66ee711-7195-6c99-b3e7-ed4f9b55eff6" /><meta name="guid" content="d66ee711-7195-6c99-b3e7-ed4f9b55eff6" /><xml><MSHelp:Attr Name="AssetID" Value="T:Lucene.Net.Analysis.Tokenizer" /><MSHelp:TOCTitle Title="Tokenizer Class" /><MSHelp:RLTitle Title="Tokenizer Class (Lucene.Net.Analysis)" /><MSHelp:Keyword Index="A" Term="T:Lucene.Net.Analysis.Tokenizer" /><MSHelp:Keyword Index="A" Term="frlrfLuceneNetAnalysisTokenizerClassTopic" /><MSHelp:Keyword Index="K" Term="Tokenizer class" /><MSHelp:Keyword Index="K" Term="Lucene.Net.Analysis.Tokenizer class" /><MSHelp:Keyword Index="K" Term="Tokenizer class, about Tokenizer class" /><MSHelp:Keyword Index="F" Term="Lucene.Net.Analysis.Tokenizer" /><MSHelp:Attr Name="HelpPriority" Value="2" /><MSHelp:Attr Name="APIType" Value="Managed" /><MSHelp:Attr Name="APILocation" Value="Lucene.Net.dll" /><MSHelp:Attr Name="APIName" Value="Lucene.Net.Analysis.Tokenizer" /><MSHelp:Attr Name="DevLang" Value="CSharp" /><MSHelp:Attr Name="DevLang" Value="VB" /><MSHelp:Attr Name="Locale" Value="en-us" /><MSHelp:Attr Name="TopicType" Value="kbSyntax" /><MSHelp:Attr Name="TopicType" Value="apiref" /><MSHelp:Attr Name="Abstract" Value="A Tokenizer is a TokenStream whose input is a Reader. This is an abstract class; subclasses must override {@link #IncrementToken()} NOTE: Subclasses overriding {@link #next(Token)} must call ..." /><MSHelp:Attr Name="AssemblyVersion" Value="2.9.4.1" /></xml></head><body><input type="hidden" id="userDataCache" class="userDataStyle" /><input type="hidden" id="hiddenScrollOffset" /><img id="collapseImage" style="display:none; height:0; width:0;" src="../icons/collapse_all.gif" alt="Collapse image" title="Collapse image" /><img id="expandImage" style="display:none; height:0; width:0;" src="../icons/expand_all.gif" alt="Expand Image" title="Expand Image" /><img id="collapseAllImage" style="display:none; height:0; width:0;" src="../icons/collapse_all.gif" /><img id="expandAllImage" style="display:none; height:0; width:0;" src="../icons/expand_all.gif" /><img id="dropDownImage" style="display:none; height:0; width:0;" src="../icons/dropdown.gif" /><img id="dropDownHoverImage" style="display:none; height:0; width:0;" src="../icons/dropdownHover.gif" /><img id="copyImage" style="display:none; height:0; width:0;" src="../icons/copycode.gif" alt="Copy image" title="Copy image" /><img id="copyHoverImage" style="display:none; height:0; width:0;" src="../icons/copycodeHighlight.gif" alt="CopyHover image" title="CopyHover image" /><div id="header"><table id="topTable" cellspacing="0" cellpadding="0"><tr><td><span onclick="ExpandCollapseAll(toggleAllImage)" style="cursor:default;" onkeypress="ExpandCollapseAll_CheckKey(toggleAllImage, event)" tabindex="0"><img ID="toggleAllImage" class="toggleAll" src="../icons/collapse_all.gif" /> <label id="collapseAllLabel" for="toggleAllImage" style="display: none;">Collapse All</label><label id="expandAllLabel" for="toggleAllImage" style="display: none;">Expand All</label> </span><span>    </span><span id="devlangsDropdown" class="filter" tabindex="0"><img id="devlangsDropdownImage" src="../icons/dropdown.gif" /> <label id="devlangsMenuAllLabel" for="devlangsDropdownImage" style="display: none;"><nobr>Code: All </nobr></label><label id="devlangsMenuMultipleLabel" for="devlangsDropdownImage" style="display: none;"><nobr>Code: Multiple </nobr></label><label id="devlangsMenuCSharpLabel" for="devlangsDropdownImage" style="display: none;"><nobr>Code: C# </nobr></label><label id="devlangsMenuVisualBasicLabel" for="devlangsDropdownImage" style="display: none;"><nobr>Code: Visual Basic </nobr></label><label id="devlangsMenuManagedCPlusPlusLabel" for="devlangsDropdownImage" style="display: none;"><nobr>Code: Visual C++ </nobr></label></span></td></tr></table><div id="devlangsMenu"><input id="CSharpCheckbox" type="checkbox" data="CSharp,cs,'persist'" value="on" onClick="SetLanguage(this)" /><label class="checkboxLabel" for="CSharpCheckbox">C#</label><br /><input id="VisualBasicCheckbox" type="checkbox" data="VisualBasic,vb,'persist'" value="on" onClick="SetLanguage(this)" /><label class="checkboxLabel" for="VisualBasicCheckbox">Visual Basic</label><br /><input id="ManagedCPlusPlusCheckbox" type="checkbox" data="ManagedCPlusPlus,cpp,'persist'" value="on" onClick="SetLanguage(this)" /><label class="checkboxLabel" for="ManagedCPlusPlusCheckbox">Visual C++</label><br /></div><table id="bottomTable" cellpadding="0" cellspacing="0"><tr id="headerTableRow1"><td align="left"><span id="runningHeaderText">Lucene.Net Class Libraries</span></td></tr><tr id="headerTableRow2"><td align="left"><span id="nsrTitle">Tokenizer Class</span></td></tr><tr id="headerTableRow3"><td align="left"><a href="16a3ab00-3e76-b66b-aebd-6845326cb5de.htm">Members</a> <a href="#seeAlsoToggle" onclick="OpenSection(seeAlsoToggle)">See Also</a> <span id="headfeedbackarea" class="feedbackhead"><a href="javascript:SubmitFeedback('','Lucene.Net Class Libraries','','','','%0\dYour%20feedback%20is%20used%20to%20improve%20the%20documentation%20and%20the%20product.%20Your%20e-mail%20address%20will%20not%20be%20used%20for%20any%20other%20purpose%20and%20is%20disposed%20of%20after%20the%20issue%20you%20report%20is%20resolved.%20%20While%20working%20to%20resolve%20the%20issue%20that%20you%20report,%20you%20may%20be%20contacted%20via%20e-mail%20to%20get%20further%20details%20or%20clarification%20on%20the%20feedback%20you%20sent.%20After%20the%20issue%20you%20report%20has%20been%20addressed,%20you%20may%20receive%20an%20e-mail%20to%20let%20you%20know%20that%20your%20feedback%20has%20been%20addressed.%0\A%0\d','Customer%20Feedback');">Send Feedback</a></span></td></tr></table><table id="gradientTable"><tr><td class="nsrBottom" background="../icons/gradient.gif" /></tr></table></div><div id="mainSection"><div id="mainBody"><div id="allHistory" class="saveHistory" onsave="saveAll()" onload="loadAll()" /><span style="color: DarkGray"></span><div class="summary"> A Tokenizer is a TokenStream whose input is a Reader.
            <p />
            This is an abstract class; subclasses must override {@link #IncrementToken()}
            <p />
            NOTE: Subclasses overriding {@link #next(Token)} must call
            {@link AttributeSource#ClearAttributes()} before setting attributes.
            Subclasses overriding {@link #IncrementToken()} must call
            {@link Token#Clear()} before setting Token attributes.
            </div><p /><b>Namespace:</b> <a href="b449a860-f0af-3961-ccd7-4e000f2f2811.htm">Lucene.Net.Analysis</a><br /><b>Assembly:</b> <span sdata="assembly">Lucene.Net</span> (in Lucene.Net.dll) Version: 2.9.4.1<h1 class="heading"><span onclick="ExpandCollapse(syntaxToggle)" style="cursor:default;" onkeypress="ExpandCollapse_CheckKey(syntaxToggle, event)" tabindex="0"><img id="syntaxToggle" class="toggle" name="toggleSwitch" src="../icons/collapse_all.gif" />Syntax</span></h1><div id="syntaxSection" class="section" name="collapseableSection" style=""><div id="syntaxCodeBlocks" class="code"><span codeLanguage="CSharp"><table><tr><th>C#</th></tr><tr><td><pre xml:space="preserve"><span class="keyword">public</span> <span class="keyword">abstract</span> <span class="keyword">class</span> <span class="identifier">Tokenizer</span> : <a href="e288b6ba-8c6a-2dca-c6ed-9564c5e106f1.htm">TokenStream</a></pre></td></tr></table></span><span codeLanguage="VisualBasicDeclaration"><table><tr><th>Visual Basic</th></tr><tr><td><pre xml:space="preserve"><span class="keyword">Public</span> <span class="keyword">MustInherit</span> <span class="keyword">Class</span> <span class="identifier">Tokenizer</span> _
	<span class="keyword">Inherits</span> <a href="e288b6ba-8c6a-2dca-c6ed-9564c5e106f1.htm">TokenStream</a></pre></td></tr></table></span><span codeLanguage="ManagedCPlusPlus"><table><tr><th>Visual C++</th></tr><tr><td><pre xml:space="preserve"><span class="keyword">public</span> <span class="keyword">ref class</span> <span class="identifier">Tokenizer</span> <span class="keyword">abstract</span> : <span class="keyword">public</span> <a href="e288b6ba-8c6a-2dca-c6ed-9564c5e106f1.htm">TokenStream</a></pre></td></tr></table></span></div></div><h1 class="heading"><span onclick="ExpandCollapse(familyToggle)" style="cursor:default;" onkeypress="ExpandCollapse_CheckKey(familyToggle, event)" tabindex="0"><img id="familyToggle" class="toggle" name="toggleSwitch" src="../icons/collapse_all.gif" />Inheritance Hierarchy</span></h1><div id="familySection" class="section" name="collapseableSection" style=""><a href="http://msdn2.microsoft.com/en-us/library/e5kfa45b" target="_blank">System<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>Object</a><br />  <a href="73348f88-e7e1-7f51-9263-e94c55c78581.htm">Lucene.Net.Util<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>AttributeSource</a><br />    <a href="e288b6ba-8c6a-2dca-c6ed-9564c5e106f1.htm">Lucene.Net.Analysis<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>TokenStream</a><br />      <span class="selflink">Lucene.Net.Analysis<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>Tokenizer</span><br />        <a href="a4a91373-9833-241e-a7dc-0cc162a299fe.htm">Lucene.Net.Analysis<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>CharTokenizer</a><br />        <a href="5068120f-65f8-da6f-41af-f951ac68b96b.htm">Lucene.Net.Analysis.CJK<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>CJKTokenizer</a><br />        <a href="40d35110-0485-610c-7bb8-e0d960b1c605.htm">Lucene.Net.Analysis.Cn<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>ChineseTokenizer</a><br />        <a href="61a5d10b-2a97-4665-d45a-2e6dd2e87b3e.htm">Lucene.Net.Analysis<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>KeywordTokenizer</a><br />        <a href="6735b19a-e54f-0088-1759-8ef4c8f0b607.htm">Lucene.Net.Analysis.NGram<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>EdgeNGramTokenizer</a><br />        <a href="3eb56adb-c8e1-5717-4ffa-14f312c13e65.htm">Lucene.Net.Analysis.NGram<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>NGramTokenizer</a><br />        <a href="a430b4a0-f981-bac5-07ce-84404e8fc8c7.htm">Lucene.Net.Analysis<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>SinkTokenizer</a><br />        <a href="1f263d8a-aa99-abba-12cf-ef501cf91337.htm">Lucene.Net.Analysis.Standard<span class="languageSpecificText"><span class="cs">.</span><span class="vb">.</span><span class="cpp">::</span><span class="nu">.</span><span class="fs">.</span></span>StandardTokenizer</a><br /></div><h1 class="heading"><span onclick="ExpandCollapse(seeAlsoToggle)" style="cursor:default;" onkeypress="ExpandCollapse_CheckKey(seeAlsoToggle, event)" tabindex="0"><img id="seeAlsoToggle" class="toggle" name="toggleSwitch" src="../icons/collapse_all.gif" />See Also</span></h1><div id="seeAlsoSection" class="section" name="collapseableSection" style=""><div class="seeAlsoStyle"><a href="16a3ab00-3e76-b66b-aebd-6845326cb5de.htm">Tokenizer Members</a></div><div class="seeAlsoStyle"><a href="b449a860-f0af-3961-ccd7-4e000f2f2811.htm">Lucene.Net.Analysis Namespace</a></div></div></div><div id="footer"><div class="footerLine"><img width="100%" height="3px" src="../icons/footer.gif" alt="Footer image" title="Footer image" /></div><A NAME="feedback"></A><span id="fb" class="feedbackcss"></span></div></div></body></html>