| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.3"/> |
| <title>Lucene.Net: contrib/Highlighter/TokenSources.cs Source File</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td id="projectlogo"><img alt="Logo" src="lucene-net-icon-128x128.png"/></td> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Lucene.Net |
|  <span id="projectnumber">3.0.3</span> |
| </div> |
| <div id="projectbrief">Lucene.Net is a .NET port of the Java Lucene Indexing Library</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.3 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="Index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Packages</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Properties</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_3a8d697be1e2feab9f01acc78e9570fb.html">contrib</a></li><li class="navelem"><a class="el" href="dir_65044b083e4dd72dcfe0705fe45ee371.html">Highlighter</a></li> </ul> |
| </div> |
| </div><!-- top --> |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">TokenSources.cs</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| <a href="_token_sources_8cs.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">/*</span></div> |
| <div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment"> * Licensed to the Apache Software Foundation (ASF) under one or more</span></div> |
| <div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment"> * contributor license agreements. See the NOTICE file distributed with</span></div> |
| <div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment"> * this work for additional information regarding copyright ownership.</span></div> |
| <div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment"> * The ASF licenses this file to You under the Apache License, Version 2.0</span></div> |
| <div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment"> * (the "License"); you may not use this file except in compliance with</span></div> |
| <div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment"> * the License. You may obtain a copy of the License at</span></div> |
| <div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment"> * </span></div> |
| <div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment"> * http://www.apache.org/licenses/LICENSE-2.0</span></div> |
| <div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment"> * </span></div> |
| <div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment"> * Unless required by applicable law or agreed to in writing, software</span></div> |
| <div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment"> * distributed under the License is distributed on an "AS IS" BASIS,</span></div> |
| <div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div> |
| <div class="line"><a name="l00014"></a><span class="lineno"> 14</span> <span class="comment"> * See the License for the specific language governing permissions and</span></div> |
| <div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="comment"> * limitations under the License.</span></div> |
| <div class="line"><a name="l00016"></a><span class="lineno"> 16</span> <span class="comment"> */</span></div> |
| <div class="line"><a name="l00017"></a><span class="lineno"> 17</span> </div> |
| <div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="comment">/*</span></div> |
| <div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="comment">* Created on 28-Oct-2004</span></div> |
| <div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="comment">*/</span></div> |
| <div class="line"><a name="l00021"></a><span class="lineno"> 21</span> </div> |
| <div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="keyword">using</span> System;</div> |
| <div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="keyword">using</span> System.Collections.Generic;</div> |
| <div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="keyword">using</span> System.IO;</div> |
| <div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="keyword">using</span> System.Linq;</div> |
| <div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="keyword">using</span> Lucene.Net.Analysis;</div> |
| <div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="keyword">using</span> Lucene.Net.Analysis.Tokenattributes;</div> |
| <div class="line"><a name="l00028"></a><span class="lineno"> 28</span> <span class="keyword">using</span> Lucene.Net.Documents;</div> |
| <div class="line"><a name="l00029"></a><span class="lineno"> 29</span> <span class="keyword">using</span> Lucene.Net.Index;</div> |
| <div class="line"><a name="l00030"></a><span class="lineno"> 30</span> </div> |
| <div class="line"><a name="l00031"></a><span class="lineno"> 31</span> <span class="keyword">namespace </span>Lucene.Net.Search.Highlight</div> |
| <div class="line"><a name="l00032"></a><span class="lineno"> 32</span> {</div> |
| <div class="line"><a name="l00033"></a><span class="lineno"> 33</span> </div> |
| <div class="line"><a name="l00038"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html"> 38</a></span>  <span class="keyword">public</span> <span class="keyword">class </span><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html" title="Hides implementation issues associated with obtaining a TokenStream for use with the higlighter - can...">TokenSources</a></div> |
| <div class="line"><a name="l00039"></a><span class="lineno"> 39</span>  {</div> |
| <div class="line"><a name="l00040"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources_1_1_stored_token_stream.html"> 40</a></span>  <span class="keyword">public</span> <span class="keyword">class </span><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources_1_1_stored_token_stream.html">StoredTokenStream</a> : <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a></div> |
| <div class="line"><a name="l00041"></a><span class="lineno"> 41</span>  {</div> |
| <div class="line"><a name="l00042"></a><span class="lineno"> 42</span>  <span class="keyword">protected</span> <span class="keyword">internal</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a>[] tokens;</div> |
| <div class="line"><a name="l00043"></a><span class="lineno"> 43</span>  <span class="keyword">protected</span> <span class="keyword">internal</span> <span class="keywordtype">int</span> currentToken = 0;</div> |
| <div class="line"><a name="l00044"></a><span class="lineno"> 44</span>  <span class="keyword">protected</span> <span class="keyword">internal</span> <a class="code" href="interface_lucene_1_1_net_1_1_analysis_1_1_tokenattributes_1_1_i_term_attribute.html" title="The term text of a Token.">ITermAttribute</a> termAtt;</div> |
| <div class="line"><a name="l00045"></a><span class="lineno"> 45</span>  <span class="keyword">protected</span> <span class="keyword">internal</span> <a class="code" href="interface_lucene_1_1_net_1_1_analysis_1_1_tokenattributes_1_1_i_offset_attribute.html" title="The start and end character offset of a Token. ">IOffsetAttribute</a> offsetAtt;</div> |
| <div class="line"><a name="l00046"></a><span class="lineno"> 46</span> </div> |
| <div class="line"><a name="l00047"></a><span class="lineno"> 47</span>  <span class="keyword">protected</span> <span class="keyword">internal</span> <a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources_1_1_stored_token_stream.html">StoredTokenStream</a>(<a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a>[] tokens)</div> |
| <div class="line"><a name="l00048"></a><span class="lineno"> 48</span>  {</div> |
| <div class="line"><a name="l00049"></a><span class="lineno"> 49</span>  this.tokens = tokens;</div> |
| <div class="line"><a name="l00050"></a><span class="lineno"> 50</span>  termAtt = AddAttribute<ITermAttribute>();</div> |
| <div class="line"><a name="l00051"></a><span class="lineno"> 51</span>  offsetAtt = AddAttribute<IOffsetAttribute>();</div> |
| <div class="line"><a name="l00052"></a><span class="lineno"> 52</span>  }</div> |
| <div class="line"><a name="l00053"></a><span class="lineno"> 53</span> </div> |
| <div class="line"><a name="l00054"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources_1_1_stored_token_stream.html#aba3299da723184852cac31c239634da2"> 54</a></span>  <span class="keyword">public</span> <span class="keyword">override</span> <span class="keywordtype">bool</span> IncrementToken()</div> |
| <div class="line"><a name="l00055"></a><span class="lineno"> 55</span>  {</div> |
| <div class="line"><a name="l00056"></a><span class="lineno"> 56</span>  <span class="keywordflow">if</span> (currentToken >= tokens.Length)</div> |
| <div class="line"><a name="l00057"></a><span class="lineno"> 57</span>  {</div> |
| <div class="line"><a name="l00058"></a><span class="lineno"> 58</span>  <span class="keywordflow">return</span> <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00059"></a><span class="lineno"> 59</span>  }</div> |
| <div class="line"><a name="l00060"></a><span class="lineno"> 60</span>  ClearAttributes();</div> |
| <div class="line"><a name="l00061"></a><span class="lineno"> 61</span>  <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a> token = tokens[currentToken++];</div> |
| <div class="line"><a name="l00062"></a><span class="lineno"> 62</span>  termAtt.<a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html#af33e3e56ddfaae9179d39da6ce0402dc" title="Copies the contents of buffer, starting at offset for length characters, into the termBuffer array...">SetTermBuffer</a>(token.<a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html#ab3603dc298a059de4b5cfa181d836cca" title="Returns the Token's term text.">Term</a>);</div> |
| <div class="line"><a name="l00063"></a><span class="lineno"> 63</span>  offsetAtt.SetOffset(token.<a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html#a421271c40ecfd6be5852f55a4a08ccb0" title="Gets or sets this Token's starting offset, the position of the first character corresponding to this ...">StartOffset</a>, token.<a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html#a8dac36e25c06d03003cf46748858026f" title="Gets or sets this Token's ending offset, one greater than the position of the last character correspo...">EndOffset</a>);</div> |
| <div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  <span class="keywordflow">return</span> <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00065"></a><span class="lineno"> 65</span>  }</div> |
| <div class="line"><a name="l00066"></a><span class="lineno"> 66</span> </div> |
| <div class="line"><a name="l00067"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources_1_1_stored_token_stream.html#a94cb4558ce5b59515ac5b27a100fa4c4"> 67</a></span>  <span class="keyword">protected</span> <span class="keyword">override</span> <span class="keywordtype">void</span> Dispose(<span class="keywordtype">bool</span> disposing)</div> |
| <div class="line"><a name="l00068"></a><span class="lineno"> 68</span>  {</div> |
| <div class="line"><a name="l00069"></a><span class="lineno"> 69</span>  <span class="comment">// do nothing</span></div> |
| <div class="line"><a name="l00070"></a><span class="lineno"> 70</span>  }</div> |
| <div class="line"><a name="l00071"></a><span class="lineno"> 71</span>  }</div> |
| <div class="line"><a name="l00072"></a><span class="lineno"> 72</span> </div> |
| <div class="line"><a name="l00085"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#a1ddd5210a9ba3c62890187a65532ab67"> 85</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetAnyTokenStream(<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html" title="IndexReader is an abstract class, providing an interface for accessing an index. Search of an index i...">IndexReader</a> reader, <span class="keywordtype">int</span> docId, String field, <a class="code" href="class_lucene_1_1_net_1_1_documents_1_1_document.html" title="Documents are the unit of indexing and search.">Document</a> doc,</div> |
| <div class="line"><a name="l00086"></a><span class="lineno"> 86</span>  <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_analyzer.html" title="An Analyzer builds TokenStreams, which analyze text. It thus represents a policy for extracting index...">Analyzer</a> analyzer)</div> |
| <div class="line"><a name="l00087"></a><span class="lineno"> 87</span>  {</div> |
| <div class="line"><a name="l00088"></a><span class="lineno"> 88</span>  <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> ts = null;</div> |
| <div class="line"><a name="l00089"></a><span class="lineno"> 89</span> </div> |
| <div class="line"><a name="l00090"></a><span class="lineno"> 90</span>  var tfv = reader.<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html#a93ef270cec13352696660ca154d4a1be" title="Return a term frequency vector for the specified document and field. The returned vector contains ter...">GetTermFreqVector</a>(docId, field);</div> |
| <div class="line"><a name="l00091"></a><span class="lineno"> 91</span>  <span class="keywordflow">if</span> (tfv != null)</div> |
| <div class="line"><a name="l00092"></a><span class="lineno"> 92</span>  {</div> |
| <div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  var termPositionVector = tfv as <a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html" title="Extends TermFreqVector to provide additional information about positions in which each of the terms i...">TermPositionVector</a>;</div> |
| <div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  <span class="keywordflow">if</span> (termPositionVector != null)</div> |
| <div class="line"><a name="l00095"></a><span class="lineno"> 95</span>  {</div> |
| <div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  ts = GetTokenStream(termPositionVector);</div> |
| <div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  }</div> |
| <div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  }</div> |
| <div class="line"><a name="l00099"></a><span class="lineno"> 99</span>  <span class="comment">//No token info stored so fall back to analyzing raw content</span></div> |
| <div class="line"><a name="l00100"></a><span class="lineno"> 100</span>  <span class="keywordflow">return</span> ts ?? GetTokenStream(doc, field, analyzer);</div> |
| <div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  }</div> |
| <div class="line"><a name="l00102"></a><span class="lineno"> 102</span> </div> |
| <div class="line"><a name="l00109"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#ad23a6d9a669e4fcf8a312fc805326d36"> 109</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetAnyTokenStream(<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html" title="IndexReader is an abstract class, providing an interface for accessing an index. Search of an index i...">IndexReader</a> reader, <span class="keywordtype">int</span> docId, String field, <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_analyzer.html" title="An Analyzer builds TokenStreams, which analyze text. It thus represents a policy for extracting index...">Analyzer</a> analyzer)</div> |
| <div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  {</div> |
| <div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> ts = null;</div> |
| <div class="line"><a name="l00112"></a><span class="lineno"> 112</span> </div> |
| <div class="line"><a name="l00113"></a><span class="lineno"> 113</span>  var tfv = reader.<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html#a93ef270cec13352696660ca154d4a1be" title="Return a term frequency vector for the specified document and field. The returned vector contains ter...">GetTermFreqVector</a>(docId, field);</div> |
| <div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  <span class="keywordflow">if</span> (tfv != null)</div> |
| <div class="line"><a name="l00115"></a><span class="lineno"> 115</span>  {</div> |
| <div class="line"><a name="l00116"></a><span class="lineno"> 116</span>  var termPositionVector = tfv as <a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html" title="Extends TermFreqVector to provide additional information about positions in which each of the terms i...">TermPositionVector</a>;</div> |
| <div class="line"><a name="l00117"></a><span class="lineno"> 117</span>  <span class="keywordflow">if</span> (termPositionVector != null)</div> |
| <div class="line"><a name="l00118"></a><span class="lineno"> 118</span>  {</div> |
| <div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  ts = GetTokenStream(termPositionVector);</div> |
| <div class="line"><a name="l00120"></a><span class="lineno"> 120</span>  }</div> |
| <div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  }</div> |
| <div class="line"><a name="l00122"></a><span class="lineno"> 122</span>  <span class="comment">//No token info stored so fall back to analyzing raw content</span></div> |
| <div class="line"><a name="l00123"></a><span class="lineno"> 123</span>  <span class="keywordflow">return</span> ts ?? GetTokenStream(reader, docId, field, analyzer);</div> |
| <div class="line"><a name="l00124"></a><span class="lineno"> 124</span>  }</div> |
| <div class="line"><a name="l00125"></a><span class="lineno"> 125</span> </div> |
| <div class="line"><a name="l00126"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#a2b49326ca7301a888db5b696b3bd9ff9"> 126</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetTokenStream(<a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html" title="Extends TermFreqVector to provide additional information about positions in which each of the terms i...">TermPositionVector</a> tpv)</div> |
| <div class="line"><a name="l00127"></a><span class="lineno"> 127</span>  {</div> |
| <div class="line"><a name="l00128"></a><span class="lineno"> 128</span>  <span class="comment">//assumes the worst and makes no assumptions about token position sequences.</span></div> |
| <div class="line"><a name="l00129"></a><span class="lineno"> 129</span>  <span class="keywordflow">return</span> GetTokenStream(tpv, <span class="keyword">false</span>);</div> |
| <div class="line"><a name="l00130"></a><span class="lineno"> 130</span>  }</div> |
| <div class="line"><a name="l00131"></a><span class="lineno"> 131</span> </div> |
| <div class="line"><a name="l00157"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#a77e1ffdfd6fbaa619f69a9a6a3eaecdc"> 157</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetTokenStream(<a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html" title="Extends TermFreqVector to provide additional information about positions in which each of the terms i...">TermPositionVector</a> tpv, <span class="keywordtype">bool</span> tokenPositionsGuaranteedContiguous)</div> |
| <div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  {</div> |
| <div class="line"><a name="l00159"></a><span class="lineno"> 159</span>  <span class="comment">//code to reconstruct the original sequence of Tokens</span></div> |
| <div class="line"><a name="l00160"></a><span class="lineno"> 160</span>  String[] terms = tpv.<a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_i_term_freq_vector.html#a9a0fd66926229e5509e05a7e7358e8f5">GetTerms</a>();</div> |
| <div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <span class="keywordtype">int</span>[] freq = tpv.<a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_i_term_freq_vector.html#ac369bbcf6b0c91d554f7b5c6ad037fa8" title="Array of term frequencies. Locations of the array correspond one to one to the terms in the array obt...">GetTermFrequencies</a>();</div> |
| <div class="line"><a name="l00162"></a><span class="lineno"> 162</span> </div> |
| <div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  <span class="keywordtype">int</span> totalTokens = freq.Sum();</div> |
| <div class="line"><a name="l00164"></a><span class="lineno"> 164</span> </div> |
| <div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  var tokensInOriginalOrder = <span class="keyword">new</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token.html" title="A Token is an occurrence of a term from the text of a field. It consists of a term's text...">Token</a>[totalTokens];</div> |
| <div class="line"><a name="l00166"></a><span class="lineno"> 166</span>  List<Token> unsortedTokens = null;</div> |
| <div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> t = 0; t < freq.Length; t++)</div> |
| <div class="line"><a name="l00168"></a><span class="lineno"> 168</span>  {</div> |
| <div class="line"><a name="l00169"></a><span class="lineno"> 169</span>  <a class="code" href="struct_lucene_1_1_net_1_1_index_1_1_term_vector_offset_info.html" title="The TermVectorOffsetInfo class holds information pertaining to a Term in a Lucene.Net.Index.TermPositionVector's offset information. This offset information is the character offset as set during the Analysis phase (and thus may not be the actual offset in the original content). ">TermVectorOffsetInfo</a>[] offsets = tpv.<a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html#a39e0116361f7e7578e0576aae85b588f" title="Returns an array of TermVectorOffsetInfo in which the term is found. May return null if offsets have ...">GetOffsets</a>(t);</div> |
| <div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  <span class="keywordflow">if</span> (offsets == null)</div> |
| <div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  {</div> |
| <div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  <span class="keywordflow">return</span> null;</div> |
| <div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  }</div> |
| <div class="line"><a name="l00174"></a><span class="lineno"> 174</span> </div> |
| <div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  <span class="keywordtype">int</span>[] pos = null;</div> |
| <div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  <span class="keywordflow">if</span> (tokenPositionsGuaranteedContiguous)</div> |
| <div class="line"><a name="l00177"></a><span class="lineno"> 177</span>  {</div> |
| <div class="line"><a name="l00178"></a><span class="lineno"> 178</span>  <span class="comment">//try get the token position info to speed up assembly of tokens into sorted sequence</span></div> |
| <div class="line"><a name="l00179"></a><span class="lineno"> 179</span>  pos = tpv.<a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html#a0fdf7e199cd02ab8bc6b767b75bf0ca2" title="Returns an array of positions in which the term is found. Terms are identified by the index at which ...">GetTermPositions</a>(t);</div> |
| <div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  }</div> |
| <div class="line"><a name="l00181"></a><span class="lineno"> 181</span>  <span class="keywordflow">if</span> (pos == null)</div> |
| <div class="line"><a name="l00182"></a><span class="lineno"> 182</span>  {</div> |
| <div class="line"><a name="l00183"></a><span class="lineno"> 183</span>  <span class="comment">//tokens NOT stored with positions or not guaranteed contiguous - must add to list and sort later</span></div> |
| <div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  <span class="keywordflow">if</span> (unsortedTokens == null)</div> |
| <div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  {</div> |
| <div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  unsortedTokens = <span class="keyword">new</span> List<Token>();</div> |
| <div class="line"><a name="l00187"></a><span class="lineno"> 187</span>  }</div> |
| <div class="line"><a name="l00188"></a><span class="lineno"> 188</span> </div> |
| <div class="line"><a name="l00189"></a><span class="lineno"> 189</span>  <span class="keywordflow">foreach</span> (<a class="code" href="struct_lucene_1_1_net_1_1_index_1_1_term_vector_offset_info.html" title="The TermVectorOffsetInfo class holds information pertaining to a Term in a Lucene.Net.Index.TermPositionVector's offset information. This offset information is the character offset as set during the Analysis phase (and thus may not be the actual offset in the original content). ">TermVectorOffsetInfo</a> t1 <span class="keywordflow">in</span> offsets)</div> |
| <div class="line"><a name="l00190"></a><span class="lineno"> 190</span>  {</div> |
| <div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  var token = <span class="keyword">new</span> <a class="code" href="_snowball_filter_8cs.html#a6f0cf8ca71e14fbb50a33080d0492fff">Token</a>(t1.<a class="code" href="struct_lucene_1_1_net_1_1_index_1_1_term_vector_offset_info.html#aebd2e92bb6a6bc26f503663ad41c36ba" title="The accessor for the starting offset of the term.">StartOffset</a>, t1.<a class="code" href="struct_lucene_1_1_net_1_1_index_1_1_term_vector_offset_info.html#a3b536e68fd1c75861625a794d2755671" title="The accessor for the ending offset for the term">EndOffset</a>);</div> |
| <div class="line"><a name="l00192"></a><span class="lineno"> 192</span>  token.SetTermBuffer(terms[t]);</div> |
| <div class="line"><a name="l00193"></a><span class="lineno"> 193</span>  unsortedTokens.Add(token);</div> |
| <div class="line"><a name="l00194"></a><span class="lineno"> 194</span>  }</div> |
| <div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  }</div> |
| <div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  <span class="keywordflow">else</span></div> |
| <div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  {</div> |
| <div class="line"><a name="l00198"></a><span class="lineno"> 198</span>  <span class="comment">//We have positions stored and a guarantee that the token position information is contiguous</span></div> |
| <div class="line"><a name="l00199"></a><span class="lineno"> 199</span> </div> |
| <div class="line"><a name="l00200"></a><span class="lineno"> 200</span>  <span class="comment">// This may be fast BUT wont work if Tokenizers used which create >1 token in same position or</span></div> |
| <div class="line"><a name="l00201"></a><span class="lineno"> 201</span>  <span class="comment">// creates jumps in position numbers - this code would fail under those circumstances</span></div> |
| <div class="line"><a name="l00202"></a><span class="lineno"> 202</span> </div> |
| <div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  <span class="comment">//tokens stored with positions - can use this to index straight into sorted array</span></div> |
| <div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> tp = 0; tp < pos.Length; tp++)</div> |
| <div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  {</div> |
| <div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  var token = <span class="keyword">new</span> <a class="code" href="_snowball_filter_8cs.html#a6f0cf8ca71e14fbb50a33080d0492fff">Token</a>(terms[t], offsets[tp].StartOffset, offsets[tp].EndOffset);</div> |
| <div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  tokensInOriginalOrder[pos[tp]] = token;</div> |
| <div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  }</div> |
| <div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  }</div> |
| <div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  }</div> |
| <div class="line"><a name="l00211"></a><span class="lineno"> 211</span>  <span class="comment">//If the field has been stored without position data we must perform a sort </span></div> |
| <div class="line"><a name="l00212"></a><span class="lineno"> 212</span>  <span class="keywordflow">if</span> (unsortedTokens != null)</div> |
| <div class="line"><a name="l00213"></a><span class="lineno"> 213</span>  {</div> |
| <div class="line"><a name="l00214"></a><span class="lineno"> 214</span>  tokensInOriginalOrder = unsortedTokens.ToArray();</div> |
| <div class="line"><a name="l00215"></a><span class="lineno"> 215</span>  Array.Sort(tokensInOriginalOrder, (t1, t2) =></div> |
| <div class="line"><a name="l00216"></a><span class="lineno"> 216</span>  {</div> |
| <div class="line"><a name="l00217"></a><span class="lineno"> 217</span>  <span class="keywordflow">if</span> (t1.StartOffset > t2.EndOffset)</div> |
| <div class="line"><a name="l00218"></a><span class="lineno"> 218</span>  <span class="keywordflow">return</span> 1;</div> |
| <div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  if (t1.StartOffset < t2.StartOffset)</div> |
| <div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="keywordflow">return</span> -1;</div> |
| <div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  <span class="keywordflow">return</span> 0;</div> |
| <div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  });</div> |
| <div class="line"><a name="l00223"></a><span class="lineno"> 223</span>  }</div> |
| <div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="keywordflow">return</span> <span class="keyword">new</span> <a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources_1_1_stored_token_stream.html">StoredTokenStream</a>(tokensInOriginalOrder);</div> |
| <div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  }</div> |
| <div class="line"><a name="l00226"></a><span class="lineno"> 226</span> </div> |
| <div class="line"><a name="l00227"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#aca08f3b1f6029695716d9a33c0ba8903"> 227</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetTokenStream(<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html" title="IndexReader is an abstract class, providing an interface for accessing an index. Search of an index i...">IndexReader</a> reader, <span class="keywordtype">int</span> docId, System.String field)</div> |
| <div class="line"><a name="l00228"></a><span class="lineno"> 228</span>  {</div> |
| <div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  var tfv = reader.<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html#a93ef270cec13352696660ca154d4a1be" title="Return a term frequency vector for the specified document and field. The returned vector contains ter...">GetTermFreqVector</a>(docId, field);</div> |
| <div class="line"><a name="l00230"></a><span class="lineno"> 230</span>  <span class="keywordflow">if</span> (tfv == null)</div> |
| <div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  {</div> |
| <div class="line"><a name="l00232"></a><span class="lineno"> 232</span>  <span class="keywordflow">throw</span> <span class="keyword">new</span> ArgumentException(field + <span class="stringliteral">" in doc #"</span> + docId</div> |
| <div class="line"><a name="l00233"></a><span class="lineno"> 233</span>  + <span class="stringliteral">"does not have any term position data stored"</span>);</div> |
| <div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  }</div> |
| <div class="line"><a name="l00235"></a><span class="lineno"> 235</span>  <span class="keywordflow">if</span> (tfv is <a class="code" href="interface_lucene_1_1_net_1_1_index_1_1_term_position_vector.html" title="Extends TermFreqVector to provide additional information about positions in which each of the terms i...">TermPositionVector</a>)</div> |
| <div class="line"><a name="l00236"></a><span class="lineno"> 236</span>  {</div> |
| <div class="line"><a name="l00237"></a><span class="lineno"> 237</span>  var tpv = (TermPositionVector) reader.<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html#a93ef270cec13352696660ca154d4a1be" title="Return a term frequency vector for the specified document and field. The returned vector contains ter...">GetTermFreqVector</a>(docId, field);</div> |
| <div class="line"><a name="l00238"></a><span class="lineno"> 238</span>  <span class="keywordflow">return</span> GetTokenStream(tpv);</div> |
| <div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  }</div> |
| <div class="line"><a name="l00240"></a><span class="lineno"> 240</span>  <span class="keywordflow">throw</span> <span class="keyword">new</span> ArgumentException(field + <span class="stringliteral">" in doc #"</span> + docId</div> |
| <div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  + <span class="stringliteral">"does not have any term position data stored"</span>);</div> |
| <div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  }</div> |
| <div class="line"><a name="l00243"></a><span class="lineno"> 243</span> </div> |
| <div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  <span class="comment">//convenience method</span></div> |
| <div class="line"><a name="l00245"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#ab7f36fbbbc70b9a9d8ffaa78552a1c42"> 245</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetTokenStream(<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html" title="IndexReader is an abstract class, providing an interface for accessing an index. Search of an index i...">IndexReader</a> reader, <span class="keywordtype">int</span> docId, String field, <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_analyzer.html" title="An Analyzer builds TokenStreams, which analyze text. It thus represents a policy for extracting index...">Analyzer</a> analyzer)</div> |
| <div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  {</div> |
| <div class="line"><a name="l00247"></a><span class="lineno"> 247</span>  <a class="code" href="class_lucene_1_1_net_1_1_documents_1_1_document.html" title="Documents are the unit of indexing and search.">Document</a> doc = reader.<a class="code" href="class_lucene_1_1_net_1_1_index_1_1_index_reader.html#ae5a8698ada40868908ea27f502337ee2" title="Returns the stored fields of the nth Document in this index. NOTE: for performance reasons...">Document</a>(docId);</div> |
| <div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  <span class="keywordflow">return</span> GetTokenStream(doc, field, analyzer);</div> |
| <div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  }</div> |
| <div class="line"><a name="l00250"></a><span class="lineno"> 250</span> </div> |
| <div class="line"><a name="l00251"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#a356b40956ea4a7bdec0a96f156b8a84c"> 251</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetTokenStream(<a class="code" href="class_lucene_1_1_net_1_1_documents_1_1_document.html" title="Documents are the unit of indexing and search.">Document</a> doc, String field, <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_analyzer.html" title="An Analyzer builds TokenStreams, which analyze text. It thus represents a policy for extracting index...">Analyzer</a> analyzer)</div> |
| <div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  {</div> |
| <div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  String contents = doc.<a class="code" href="class_lucene_1_1_net_1_1_documents_1_1_document.html#ab279a2d40744d95895e01cc16965887b" title="Returns the string value of the field with the given name if any exist in this document, or null. If multiple fields exist with this name, this method returns the first value added. If only binary fields with this name exist, returns null. ">Get</a>(field);</div> |
| <div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  <span class="keywordflow">if</span> (contents == null)</div> |
| <div class="line"><a name="l00255"></a><span class="lineno"> 255</span>  {</div> |
| <div class="line"><a name="l00256"></a><span class="lineno"> 256</span>  <span class="keywordflow">throw</span> <span class="keyword">new</span> ArgumentException(<span class="stringliteral">"Field "</span> + field + <span class="stringliteral">" in document is not stored and cannot be analyzed"</span>);</div> |
| <div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  }</div> |
| <div class="line"><a name="l00258"></a><span class="lineno"> 258</span>  <span class="keywordflow">return</span> GetTokenStream(field, contents, analyzer);</div> |
| <div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  }</div> |
| <div class="line"><a name="l00260"></a><span class="lineno"> 260</span> </div> |
| <div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  <span class="comment">//convenience method</span></div> |
| <div class="line"><a name="l00262"></a><span class="lineno"><a class="code" href="class_lucene_1_1_net_1_1_search_1_1_highlight_1_1_token_sources.html#ad91941c8a0bb901e71b032ae8de2c420"> 262</a></span>  <span class="keyword">public</span> <span class="keyword">static</span> <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_token_stream.html" title="A TokenStream enumerates the sequence of tokens, either from Fields of a Document or from query text...">TokenStream</a> GetTokenStream(String field, String contents, <a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_analyzer.html" title="An Analyzer builds TokenStreams, which analyze text. It thus represents a policy for extracting index...">Analyzer</a> analyzer)</div> |
| <div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  {</div> |
| <div class="line"><a name="l00264"></a><span class="lineno"> 264</span>  <span class="keywordflow">return</span> analyzer.<a class="code" href="class_lucene_1_1_net_1_1_analysis_1_1_analyzer.html#ac031bae47f64e6bbe2117c991996f5fc" title="Creates a TokenStream which tokenizes all the text in the provided Reader. Must be able to handle nul...">TokenStream</a>(field, <span class="keyword">new</span> StringReader(contents));</div> |
| <div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  }</div> |
| <div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  }</div> |
| <div class="line"><a name="l00267"></a><span class="lineno"> 267</span> }</div> |
| </div><!-- fragment --></div><!-- contents --> |
| <!-- start footer part --> |
| <hr class="footer"/><address class="footer"><small> |
| Generated on Thu Jan 3 2013 02:12:43 for Lucene.Net by  <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/> |
| </a> 1.8.3 |
| </small></address> |
| </body> |
| </html> |
