| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: be/src/exec/base-sequence-scanner.cc Source File</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('base-sequence-scanner_8cc_source.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">base-sequence-scanner.cc</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| <a href="base-sequence-scanner_8cc.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">// Copyright 2012 Cloudera Inc.</span></div> |
| <div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment">// Licensed under the Apache License, Version 2.0 (the "License");</span></div> |
| <div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment">// you may not use this file except in compliance with the License.</span></div> |
| <div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment">// You may obtain a copy of the License at</span></div> |
| <div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment">// http://www.apache.org/licenses/LICENSE-2.0</span></div> |
| <div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment">// Unless required by applicable law or agreed to in writing, software</span></div> |
| <div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment">// distributed under the License is distributed on an "AS IS" BASIS,</span></div> |
| <div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment">// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div> |
| <div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment">// See the License for the specific language governing permissions and</span></div> |
| <div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment">// limitations under the License.</span></div> |
| <div class="line"><a name="l00014"></a><span class="lineno"> 14</span> </div> |
| <div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="preprocessor">#include <boost/bind.hpp></span></div> |
| <div class="line"><a name="l00016"></a><span class="lineno"> 16</span> </div> |
| <div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="preprocessor">#include "<a class="code" href="base-sequence-scanner_8h.html">exec/base-sequence-scanner.h</a>"</span></div> |
| <div class="line"><a name="l00018"></a><span class="lineno"> 18</span> </div> |
| <div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="preprocessor">#include "<a class="code" href="hdfs-scan-node_8h.html">exec/hdfs-scan-node.h</a>"</span></div> |
| <div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="preprocessor">#include "<a class="code" href="scanner-context_8inline_8h.html">exec/scanner-context.inline.h</a>"</span></div> |
| <div class="line"><a name="l00021"></a><span class="lineno"> 21</span> <span class="preprocessor">#include "<a class="code" href="runtime-state_8h.html">runtime/runtime-state.h</a>"</span></div> |
| <div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="preprocessor">#include "<a class="code" href="string-search_8h.html">runtime/string-search.h</a>"</span></div> |
| <div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="preprocessor">#include "<a class="code" href="codec_8h.html">util/codec.h</a>"</span></div> |
| <div class="line"><a name="l00024"></a><span class="lineno"> 24</span> </div> |
| <div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="preprocessor">#include "<a class="code" href="names_8h.html">common/names.h</a>"</span></div> |
| <div class="line"><a name="l00026"></a><span class="lineno"> 26</span> </div> |
| <div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="keyword">using namespace </span>impala;</div> |
| <div class="line"><a name="l00028"></a><span class="lineno"> 28</span> </div> |
| <div class="line"><a name="l00029"></a><span class="lineno"> 29</span> <span class="keyword">const</span> <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac62358e62350918ee1a0c429cd74fd5c">BaseSequenceScanner::HEADER_SIZE</a> = 1024;</div> |
| <div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="keyword">const</span> <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ae3d4eec40706fa6d653246a19d17e006">BaseSequenceScanner::SYNC_MARKER</a> = -1;</div> |
| <div class="line"><a name="l00031"></a><span class="lineno"> 31</span> </div> |
| <div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="comment">// Constants used in ReadPastSize()</span></div> |
| <div class="line"><a name="l00033"></a><span class="lineno"><a class="line" href="base-sequence-scanner_8cc.html#ac5a0880457028b83916eadd56b9dcf20"> 33</a></span> <span class="keyword">static</span> <span class="keyword">const</span> <span class="keywordtype">double</span> <a class="code" href="base-sequence-scanner_8cc.html#ac5a0880457028b83916eadd56b9dcf20">BLOCK_SIZE_PADDING_PERCENT</a> = 0.1;</div> |
| <div class="line"><a name="l00034"></a><span class="lineno"><a class="line" href="base-sequence-scanner_8cc.html#a4fd070e0de326e5cfa25a3bf0a8747e7"> 34</a></span> <span class="keyword">static</span> <span class="keyword">const</span> <span class="keywordtype">int</span> <a class="code" href="base-sequence-scanner_8cc.html#a4fd070e0de326e5cfa25a3bf0a8747e7">REMAINING_BLOCK_SIZE_GUESS</a> = 100 * 1024; <span class="comment">// bytes</span></div> |
| <div class="line"><a name="l00035"></a><span class="lineno"><a class="line" href="base-sequence-scanner_8cc.html#a3286d10cafc052ea336a292d9d9ab686"> 35</a></span> <span class="keyword">static</span> <span class="keyword">const</span> <span class="keywordtype">int</span> <a class="code" href="base-sequence-scanner_8cc.html#a3286d10cafc052ea336a292d9d9ab686">MIN_SYNC_READ_SIZE</a> = 10 * 1024; <span class="comment">// bytes</span></div> |
| <div class="line"><a name="l00036"></a><span class="lineno"> 36</span> </div> |
| <div class="line"><a name="l00037"></a><span class="lineno"> 37</span> <span class="comment">// Macro to convert between SerdeUtil errors to Status returns.</span></div> |
| <div class="line"><a name="l00038"></a><span class="lineno"><a class="line" href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565"> 38</a></span> <span class="preprocessor">#define RETURN_IF_FALSE(x) if (UNLIKELY(!(x))) return parse_status_</span></div> |
| <div class="line"><a name="l00039"></a><span class="lineno"> 39</span> <span class="preprocessor"></span></div> |
| <div class="line"><a name="l00040"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#a6b84385660f88ed592fd61ac62c15793"> 40</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a6b84385660f88ed592fd61ac62c15793">BaseSequenceScanner::IssueInitialRanges</a>(<a class="code" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a>* scan_node,</div> |
| <div class="line"><a name="l00041"></a><span class="lineno"> 41</span>  <span class="keyword">const</span> vector<HdfsFileDesc*>& files) {</div> |
| <div class="line"><a name="l00042"></a><span class="lineno"> 42</span>  <span class="comment">// Issue just the header range for each file. When the header is complete,</span></div> |
| <div class="line"><a name="l00043"></a><span class="lineno"> 43</span>  <span class="comment">// we'll issue the splits for that file. Splits cannot be processed until the</span></div> |
| <div class="line"><a name="l00044"></a><span class="lineno"> 44</span>  <span class="comment">// header is parsed (the header object is then shared across splits for that file).</span></div> |
| <div class="line"><a name="l00045"></a><span class="lineno"> 45</span>  vector<DiskIoMgr::ScanRange*> header_ranges;</div> |
| <div class="line"><a name="l00046"></a><span class="lineno"> 46</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < files.size(); ++i) {</div> |
| <div class="line"><a name="l00047"></a><span class="lineno"> 47</span>  <a class="code" href="structimpala_1_1ScanRangeMetadata.html">ScanRangeMetadata</a>* metadata =</div> |
| <div class="line"><a name="l00048"></a><span class="lineno"> 48</span>  <span class="keyword">reinterpret_cast<</span><a class="code" href="structimpala_1_1ScanRangeMetadata.html">ScanRangeMetadata</a>*<span class="keyword">></span>(files[i]->splits[0]->meta_data());</div> |
| <div class="line"><a name="l00049"></a><span class="lineno"> 49</span>  int64_t header_size = min(static_cast<int64_t>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac62358e62350918ee1a0c429cd74fd5c">HEADER_SIZE</a>), files[i]->file_length);</div> |
| <div class="line"><a name="l00050"></a><span class="lineno"> 50</span>  <span class="comment">// The header is almost always a remote read. Set the disk id to -1 and indicate</span></div> |
| <div class="line"><a name="l00051"></a><span class="lineno"> 51</span>  <span class="comment">// it is not cached.</span></div> |
| <div class="line"><a name="l00052"></a><span class="lineno"> 52</span>  <span class="comment">// TODO: add remote disk id and plumb that through to the io mgr. It should have</span></div> |
| <div class="line"><a name="l00053"></a><span class="lineno"> 53</span>  <span class="comment">// 1 queue for each NIC as well?</span></div> |
| <div class="line"><a name="l00054"></a><span class="lineno"> 54</span>  <a class="code" href="classimpala_1_1DiskIoMgr_1_1ScanRange.html">DiskIoMgr::ScanRange</a>* header_range = scan_node-><a class="code" href="classimpala_1_1HdfsScanNode.html#a5314b840efa7625e54ce930ee1986bb1">AllocateScanRange</a>(</div> |
| <div class="line"><a name="l00055"></a><span class="lineno"> 55</span>  files[i]->fs, files[i]->filename.c_str(), header_size, 0, metadata-><a class="code" href="structimpala_1_1ScanRangeMetadata.html#a7a7171d1dc269bea3f3dd5636e4bb774">partition_id</a>,</div> |
| <div class="line"><a name="l00056"></a><span class="lineno"> 56</span>  -1, <span class="keyword">false</span>, <span class="keyword">false</span>, files[i]->mtime);</div> |
| <div class="line"><a name="l00057"></a><span class="lineno"> 57</span>  header_ranges.push_back(header_range);</div> |
| <div class="line"><a name="l00058"></a><span class="lineno"> 58</span>  }</div> |
| <div class="line"><a name="l00059"></a><span class="lineno"> 59</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(scan_node-><a class="code" href="classimpala_1_1HdfsScanNode.html#afe9bc42ad3846bde9f7113e5f1c133dc">AddDiskIoRanges</a>(header_ranges));</div> |
| <div class="line"><a name="l00060"></a><span class="lineno"> 60</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00061"></a><span class="lineno"> 61</span> }</div> |
| <div class="line"><a name="l00062"></a><span class="lineno"> 62</span> </div> |
| <div class="line"><a name="l00063"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#a240e63ad0129c83cb784f96031ad9f6d"> 63</a></span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a240e63ad0129c83cb784f96031ad9f6d">BaseSequenceScanner::BaseSequenceScanner</a>(<a class="code" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a>* node, <a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state)</div> |
| <div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  : <a class="code" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a>(node, state),</div> |
| <div class="line"><a name="l00065"></a><span class="lineno"> 65</span>  header_(NULL),</div> |
| <div class="line"><a name="l00066"></a><span class="lineno"> 66</span>  block_start_(0),</div> |
| <div class="line"><a name="l00067"></a><span class="lineno"> 67</span>  total_block_size_(0),</div> |
| <div class="line"><a name="l00068"></a><span class="lineno"> 68</span>  num_syncs_(0) {</div> |
| <div class="line"><a name="l00069"></a><span class="lineno"> 69</span> }</div> |
| <div class="line"><a name="l00070"></a><span class="lineno"> 70</span> </div> |
| <div class="line"><a name="l00071"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#aae0d5cf9f400c07db437b4fa94b6be8a"> 71</a></span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aae0d5cf9f400c07db437b4fa94b6be8a">BaseSequenceScanner::~BaseSequenceScanner</a>() {</div> |
| <div class="line"><a name="l00072"></a><span class="lineno"> 72</span> }</div> |
| <div class="line"><a name="l00073"></a><span class="lineno"> 73</span> </div> |
| <div class="line"><a name="l00074"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#acc3c090b8b5e1c282c828be221697d5b"> 74</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#acc3c090b8b5e1c282c828be221697d5b">BaseSequenceScanner::Prepare</a>(<a class="code" href="classimpala_1_1ScannerContext.html">ScannerContext</a>* context) {</div> |
| <div class="line"><a name="l00075"></a><span class="lineno"> 75</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#a99165297c6e6546edbee741a54dfd194">HdfsScanner::Prepare</a>(context));</div> |
| <div class="line"><a name="l00076"></a><span class="lineno"> 76</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#acb22ac5214eff45da580c5be90c4d08f">set_read_past_size_cb</a>(bind(&<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a9586fc33865674ed488abbbd0d7e2003">BaseSequenceScanner::ReadPastSize</a>, <span class="keyword">this</span>, _1));</div> |
| <div class="line"><a name="l00077"></a><span class="lineno"> 77</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a393b2ff9306ac4becc7ae3838099a4c7">bytes_skipped_counter_</a> = <a class="code" href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a>(</div> |
| <div class="line"><a name="l00078"></a><span class="lineno"> 78</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1ExecNode.html#ad062f28c78add1a869fffdab9419b09d">runtime_profile</a>(), <span class="stringliteral">"BytesSkipped"</span>, TUnit::BYTES);</div> |
| <div class="line"><a name="l00079"></a><span class="lineno"> 79</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00080"></a><span class="lineno"> 80</span> }</div> |
| <div class="line"><a name="l00081"></a><span class="lineno"> 81</span> </div> |
| <div class="line"><a name="l00082"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#a1a37621b600b55d162832e8744fc6a17"> 82</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a1a37621b600b55d162832e8744fc6a17">BaseSequenceScanner::Close</a>() {</div> |
| <div class="line"><a name="l00083"></a><span class="lineno"> 83</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"Bytes read past scan range: "</span> << -<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a48c8ea0f83bb58e810e5148e0addccb0">bytes_left</a>();</div> |
| <div class="line"><a name="l00084"></a><span class="lineno"> 84</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"Average block size: "</span></div> |
| <div class="line"><a name="l00085"></a><span class="lineno"> 85</span>  << (<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">num_syncs_</a> > 1 ? <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a4e114262860e1f844d498c802336973c">total_block_size_</a> / (<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">num_syncs_</a> - 1) : 0);</div> |
| <div class="line"><a name="l00086"></a><span class="lineno"> 86</span>  <span class="comment">// Need to close the decompressor before releasing the resources at AddFinalRowBatch(),</span></div> |
| <div class="line"><a name="l00087"></a><span class="lineno"> 87</span>  <span class="comment">// because in some cases there is memory allocated in decompressor_'s temp_memory_pool_.</span></div> |
| <div class="line"><a name="l00088"></a><span class="lineno"> 88</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsScanner.html#a8d995c8326def52de037ecf074fc2654">decompressor_</a>.get() != NULL) {</div> |
| <div class="line"><a name="l00089"></a><span class="lineno"> 89</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a8d995c8326def52de037ecf074fc2654">decompressor_</a>->Close();</div> |
| <div class="line"><a name="l00090"></a><span class="lineno"> 90</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a8d995c8326def52de037ecf074fc2654">decompressor_</a>.reset(NULL);</div> |
| <div class="line"><a name="l00091"></a><span class="lineno"> 91</span>  }</div> |
| <div class="line"><a name="l00092"></a><span class="lineno"> 92</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a94b437624601d6e7626e9b3d59cd87da">AttachPool</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#ae86f3019c271de0d0593204003ee1653">data_buffer_pool_</a>.get(), <span class="keyword">false</span>);</div> |
| <div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a36e661fbaff23879b78f53cdc1a56552">AddFinalRowBatch</a>();</div> |
| <div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a0184e49248bc243ed45c77b8fb7aeac4">only_parsing_header_</a>) {</div> |
| <div class="line"><a name="l00095"></a><span class="lineno"> 95</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#a6193af89a13de0a3b8d45bc0e5cd288c">RangeComplete</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3ac35eed32d440edbd401600d9ee0c4">file_format</a>(), <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a394cc7bb9ae9309bce7f4d9167b083fc">compression_type</a>);</div> |
| <div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  }</div> |
| <div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a61d4a073305f6befe9f12bad12f9de48">HdfsScanner::Close</a>();</div> |
| <div class="line"><a name="l00098"></a><span class="lineno"> 98</span> }</div> |
| <div class="line"><a name="l00099"></a><span class="lineno"> 99</span> </div> |
| <div class="line"><a name="l00100"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#acc59a2d650453eb5c12670519342c846"> 100</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#acc59a2d650453eb5c12670519342c846">BaseSequenceScanner::ProcessSplit</a>() {</div> |
| <div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a> = <span class="keyword">reinterpret_cast<</span><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html">FileHeader</a>*<span class="keyword">></span>(</div> |
| <div class="line"><a name="l00102"></a><span class="lineno"> 102</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#adab35319d4dbff9af621387591a32c2c">GetFileMetadata</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">filename</a>()));</div> |
| <div class="line"><a name="l00103"></a><span class="lineno"> 103</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a> == NULL) {</div> |
| <div class="line"><a name="l00104"></a><span class="lineno"> 104</span>  <span class="comment">// This is the initial scan range just to parse the header</span></div> |
| <div class="line"><a name="l00105"></a><span class="lineno"> 105</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a0184e49248bc243ed45c77b8fb7aeac4">only_parsing_header_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00106"></a><span class="lineno"> 106</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a> = <a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a519c086c9914fa0c9e2a2f608814a46b">AllocateFileHeader</a>());</div> |
| <div class="line"><a name="l00107"></a><span class="lineno"> 107</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> status = <a class="code" href="classimpala_1_1BaseSequenceScanner.html#af924a2719c3aedb31c17bd3f0bd80af3">ReadFileHeader</a>();</div> |
| <div class="line"><a name="l00108"></a><span class="lineno"> 108</span>  <span class="keywordflow">if</span> (!status.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) {</div> |
| <div class="line"><a name="l00109"></a><span class="lineno"> 109</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#ad449a8708ed26241be77b023c95ac7f0">abort_on_error</a>()) <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#aaf076b1879d9e712b08ba3a6e24607b9">LogError</a>(status.<a class="code" href="classimpala_1_1Status.html#afe14635279d3cd633d08ecbadee51019">msg</a>());</div> |
| <div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <span class="comment">// We need to complete the ranges for this file.</span></div> |
| <div class="line"><a name="l00112"></a><span class="lineno"> 112</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#adff67c7bd7dd6522b6db5682deefcdfe">CloseFileRanges</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">filename</a>());</div> |
| <div class="line"><a name="l00113"></a><span class="lineno"> 113</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  }</div> |
| <div class="line"><a name="l00115"></a><span class="lineno"> 115</span> </div> |
| <div class="line"><a name="l00116"></a><span class="lineno"> 116</span>  <span class="comment">// Header is parsed, set the metadata in the scan node and issue more ranges</span></div> |
| <div class="line"><a name="l00117"></a><span class="lineno"> 117</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#a3405cd6d94586c152efae17e6cfbd787">SetFileMetadata</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">filename</a>(), <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>);</div> |
| <div class="line"><a name="l00118"></a><span class="lineno"> 118</span>  <a class="code" href="structimpala_1_1HdfsFileDesc.html">HdfsFileDesc</a>* desc = <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#a0521140f7c60d795fb7bd2f44c42c9bf">GetFileDesc</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">filename</a>());</div> |
| <div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#afe9bc42ad3846bde9f7113e5f1c133dc">AddDiskIoRanges</a>(desc);</div> |
| <div class="line"><a name="l00120"></a><span class="lineno"> 120</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  }</div> |
| <div class="line"><a name="l00122"></a><span class="lineno"> 122</span> </div> |
| <div class="line"><a name="l00123"></a><span class="lineno"> 123</span>  <span class="comment">// Initialize state for new scan range</span></div> |
| <div class="line"><a name="l00124"></a><span class="lineno"> 124</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a> = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00125"></a><span class="lineno"> 125</span>  <span class="comment">// If the file is compressed, the buffers in the stream_ are not used directly.</span></div> |
| <div class="line"><a name="l00126"></a><span class="lineno"> 126</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a17899a9d57bf0246139d62029d40a7a3">is_compressed</a>) <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a53b77f52ccee137fbe9a39083bac3488">set_contains_tuple_data</a>(<span class="keyword">false</span>);</div> |
| <div class="line"><a name="l00127"></a><span class="lineno"> 127</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#a542176620ff5194866253070a539134a">InitNewRange</a>());</div> |
| <div class="line"><a name="l00128"></a><span class="lineno"> 128</span> </div> |
| <div class="line"><a name="l00129"></a><span class="lineno"> 129</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> status = <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00130"></a><span class="lineno"> 130</span> </div> |
| <div class="line"><a name="l00131"></a><span class="lineno"> 131</span>  <span class="comment">// Skip to the first record</span></div> |
| <div class="line"><a name="l00132"></a><span class="lineno"> 132</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>() < <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a87d5f2145212f3a762255c4713357965">header_size</a>) {</div> |
| <div class="line"><a name="l00133"></a><span class="lineno"> 133</span>  <span class="comment">// If the scan range starts within the header, skip to the end of the header so we</span></div> |
| <div class="line"><a name="l00134"></a><span class="lineno"> 134</span>  <span class="comment">// don't accidentally skip to an extra sync within the header</span></div> |
| <div class="line"><a name="l00135"></a><span class="lineno"> 135</span>  <a class="code" href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565">RETURN_IF_FALSE</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac1bae8ae5208e9d2c2547bc0a7efddd6">SkipBytes</a>(</div> |
| <div class="line"><a name="l00136"></a><span class="lineno"> 136</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a87d5f2145212f3a762255c4713357965">header_size</a> - <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>(), &<a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a>));</div> |
| <div class="line"><a name="l00137"></a><span class="lineno"> 137</span>  }</div> |
| <div class="line"><a name="l00138"></a><span class="lineno"> 138</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac583e95bbd1397d0ecbe526e7a35a445">SkipToSync</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#ab57f8bf94e3f23b8e9ed076b786d7a23">sync</a>, <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a>));</div> |
| <div class="line"><a name="l00139"></a><span class="lineno"> 139</span> </div> |
| <div class="line"><a name="l00140"></a><span class="lineno"> 140</span>  <span class="comment">// Process Range.</span></div> |
| <div class="line"><a name="l00141"></a><span class="lineno"> 141</span>  <span class="keywordflow">while</span> (!<a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a>) {</div> |
| <div class="line"><a name="l00142"></a><span class="lineno"> 142</span>  status = <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a7fb6c7a9e0bdf14e594d412f7e1bc600">ProcessRange</a>();</div> |
| <div class="line"><a name="l00143"></a><span class="lineno"> 143</span>  <span class="keywordflow">if</span> (status.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00144"></a><span class="lineno"> 144</span>  <span class="keywordflow">if</span> (status.<a class="code" href="classimpala_1_1Status.html#a327e57a4c61d1387f8b52fce20e89a34">IsCancelled</a>() || status.<a class="code" href="classimpala_1_1Status.html#a39b7bde73a8803e2ceaf3b97d0fe94eb">IsMemLimitExceeded</a>()) <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l00145"></a><span class="lineno"> 145</span> </div> |
| <div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  <span class="comment">// Log error from file format parsing.</span></div> |
| <div class="line"><a name="l00147"></a><span class="lineno"> 147</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#aaf076b1879d9e712b08ba3a6e24607b9">LogError</a>(<a class="code" href="classimpala_1_1ErrorMsg.html">ErrorMsg</a>(TErrorCode::SEQUENCE_SCANNER_PARSE_ERROR,</div> |
| <div class="line"><a name="l00148"></a><span class="lineno"> 148</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">filename</a>(), <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>(),</div> |
| <div class="line"><a name="l00149"></a><span class="lineno"> 149</span>  (<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a1a423b75d11aaf04deb06633355bfb08">eof</a>() ? <span class="stringliteral">"(EOF)"</span> : <span class="stringliteral">""</span>)));</div> |
| <div class="line"><a name="l00150"></a><span class="lineno"> 150</span> </div> |
| <div class="line"><a name="l00151"></a><span class="lineno"> 151</span>  <span class="comment">// Make sure errors specified in the status are logged as well</span></div> |
| <div class="line"><a name="l00152"></a><span class="lineno"> 152</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#aaf076b1879d9e712b08ba3a6e24607b9">LogError</a>(status.<a class="code" href="classimpala_1_1Status.html#afe14635279d3cd633d08ecbadee51019">msg</a>());</div> |
| <div class="line"><a name="l00153"></a><span class="lineno"> 153</span> </div> |
| <div class="line"><a name="l00154"></a><span class="lineno"> 154</span>  <span class="comment">// If abort on error then return, otherwise try to recover.</span></div> |
| <div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#ad449a8708ed26241be77b023c95ac7f0">abort_on_error</a>()) <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l00156"></a><span class="lineno"> 156</span> </div> |
| <div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  <span class="comment">// Recover by skipping to the next sync.</span></div> |
| <div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a> = <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00159"></a><span class="lineno"> 159</span>  int64_t error_offset = <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>();</div> |
| <div class="line"><a name="l00160"></a><span class="lineno"> 160</span>  status = <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac583e95bbd1397d0ecbe526e7a35a445">SkipToSync</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#ab57f8bf94e3f23b8e9ed076b786d7a23">sync</a>, <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a>);</div> |
| <div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <a class="code" href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a393b2ff9306ac4becc7ae3838099a4c7">bytes_skipped_counter_</a>, <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>() - error_offset);</div> |
| <div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(status);</div> |
| <div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  DCHECK(<a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>());</div> |
| <div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  }</div> |
| <div class="line"><a name="l00165"></a><span class="lineno"> 165</span> </div> |
| <div class="line"><a name="l00166"></a><span class="lineno"> 166</span>  <span class="comment">// All done with this scan range.</span></div> |
| <div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00168"></a><span class="lineno"> 168</span> }</div> |
| <div class="line"><a name="l00169"></a><span class="lineno"> 169</span> </div> |
| <div class="line"><a name="l00170"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#a38bb92ef6416325aefafbcaed8f5010c"> 170</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a38bb92ef6416325aefafbcaed8f5010c">BaseSequenceScanner::ReadSync</a>() {</div> |
| <div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  <span class="comment">// We are finished when we read a sync marker occurring completely in the next</span></div> |
| <div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  <span class="comment">// scan range</span></div> |
| <div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a> = <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a0f3599f41ee84455e186a3d1b02fcbb7">eosr</a>();</div> |
| <div class="line"><a name="l00174"></a><span class="lineno"> 174</span> </div> |
| <div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  uint8_t* <a class="code" href="namespaceimpala.html#a50b59c9628eedf49aa82643ce601967d">hash</a>;</div> |
| <div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  int64_t out_len;</div> |
| <div class="line"><a name="l00177"></a><span class="lineno"> 177</span>  <a class="code" href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565">RETURN_IF_FALSE</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#acfded8cb494dbda9eb700d32295e87be">GetBytes</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a>, &hash, &out_len, &<a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a>));</div> |
| <div class="line"><a name="l00178"></a><span class="lineno"> 178</span>  <span class="keywordflow">if</span> (out_len != <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a> || memcmp(hash, <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#ab57f8bf94e3f23b8e9ed076b786d7a23">sync</a>, <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a>)) {</div> |
| <div class="line"><a name="l00179"></a><span class="lineno"> 179</span>  stringstream ss;</div> |
| <div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  ss << <span class="stringliteral">"Bad synchronization marker"</span> << endl</div> |
| <div class="line"><a name="l00181"></a><span class="lineno"> 181</span>  << <span class="stringliteral">" Expected: '"</span></div> |
| <div class="line"><a name="l00182"></a><span class="lineno"> 182</span>  << <a class="code" href="classimpala_1_1ReadWriteUtil.html#a9c6a8fcabb5a5edb77de5866d5a4f7f4">ReadWriteUtil::HexDump</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">header_</a>-><a class="code" href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#ab57f8bf94e3f23b8e9ed076b786d7a23">sync</a>, <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a>) << <span class="stringliteral">"'"</span> << endl</div> |
| <div class="line"><a name="l00183"></a><span class="lineno"> 183</span>  << <span class="stringliteral">" Actual: '"</span></div> |
| <div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  << <a class="code" href="classimpala_1_1ReadWriteUtil.html#a9c6a8fcabb5a5edb77de5866d5a4f7f4">ReadWriteUtil::HexDump</a>(hash, <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">SYNC_HASH_SIZE</a>) << <span class="stringliteral">"'"</span>;</div> |
| <div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  }</div> |
| <div class="line"><a name="l00187"></a><span class="lineno"> 187</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a> |= <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a1a423b75d11aaf04deb06633355bfb08">eof</a>();</div> |
| <div class="line"><a name="l00188"></a><span class="lineno"> 188</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a4e114262860e1f844d498c802336973c">total_block_size_</a> += <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>() - <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5c06eddcf4b24c054328e7eeb28c5f1b">block_start_</a>;</div> |
| <div class="line"><a name="l00189"></a><span class="lineno"> 189</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5c06eddcf4b24c054328e7eeb28c5f1b">block_start_</a> = <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>();</div> |
| <div class="line"><a name="l00190"></a><span class="lineno"> 190</span>  ++<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">num_syncs_</a>;</div> |
| <div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00192"></a><span class="lineno"> 192</span> }</div> |
| <div class="line"><a name="l00193"></a><span class="lineno"> 193</span> </div> |
| <div class="line"><a name="l00194"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#a92d958bf8e7eaee48dad0398def2589d"> 194</a></span> <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a92d958bf8e7eaee48dad0398def2589d">BaseSequenceScanner::FindSyncBlock</a>(<span class="keyword">const</span> uint8_t* buffer, <span class="keywordtype">int</span> buffer_len,</div> |
| <div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  <span class="keyword">const</span> uint8_t* sync, <span class="keywordtype">int</span> sync_len) {</div> |
| <div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  <span class="keywordtype">char</span>* sync_str = <span class="keyword">reinterpret_cast<</span><span class="keywordtype">char</span>*<span class="keyword">></span>(<span class="keyword">const_cast<</span>uint8_t*<span class="keyword">></span>(sync));</div> |
| <div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  <a class="code" href="structimpala_1_1StringValue.html">StringValue</a> needle = <a class="code" href="structimpala_1_1StringValue.html">StringValue</a>(sync_str, sync_len);</div> |
| <div class="line"><a name="l00198"></a><span class="lineno"> 198</span> </div> |
| <div class="line"><a name="l00199"></a><span class="lineno"> 199</span>  <a class="code" href="structimpala_1_1StringValue.html">StringValue</a> haystack(</div> |
| <div class="line"><a name="l00200"></a><span class="lineno"> 200</span>  const_cast<char*>(reinterpret_cast<const char*>(buffer)), buffer_len);</div> |
| <div class="line"><a name="l00201"></a><span class="lineno"> 201</span> </div> |
| <div class="line"><a name="l00202"></a><span class="lineno"> 202</span>  <a class="code" href="classimpala_1_1StringSearch.html">StringSearch</a> search(&needle);</div> |
| <div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  <span class="keywordtype">int</span> <a class="code" href="partitioning-throughput-test_8cc.html#ad172ecfd5c31c5df6282c2b4bc322bf7">offset</a> = search.<a class="code" href="classimpala_1_1StringSearch.html#a4ab30c893cd9036d6ae5684e91b19012">Search</a>(&haystack);</div> |
| <div class="line"><a name="l00204"></a><span class="lineno"> 204</span> </div> |
| <div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  <span class="keywordflow">if</span> (offset != -1) {</div> |
| <div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  <span class="comment">// Advance offset past sync</span></div> |
| <div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  offset += sync_len;</div> |
| <div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  }</div> |
| <div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  <span class="keywordflow">return</span> <a class="code" href="partitioning-throughput-test_8cc.html#ad172ecfd5c31c5df6282c2b4bc322bf7">offset</a>;</div> |
| <div class="line"><a name="l00210"></a><span class="lineno"> 210</span> }</div> |
| <div class="line"><a name="l00211"></a><span class="lineno"> 211</span> </div> |
| <div class="line"><a name="l00212"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#ac583e95bbd1397d0ecbe526e7a35a445"> 212</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#ac583e95bbd1397d0ecbe526e7a35a445">BaseSequenceScanner::SkipToSync</a>(<span class="keyword">const</span> uint8_t* sync, <span class="keywordtype">int</span> sync_size) {</div> |
| <div class="line"><a name="l00213"></a><span class="lineno"> 213</span>  <span class="comment">// offset into current buffer of end of sync (once found, -1 until then)</span></div> |
| <div class="line"><a name="l00214"></a><span class="lineno"> 214</span>  <span class="keywordtype">int</span> <a class="code" href="partitioning-throughput-test_8cc.html#ad172ecfd5c31c5df6282c2b4bc322bf7">offset</a> = -1;</div> |
| <div class="line"><a name="l00215"></a><span class="lineno"> 215</span>  uint8_t* buffer;</div> |
| <div class="line"><a name="l00216"></a><span class="lineno"> 216</span>  int64_t buffer_len;</div> |
| <div class="line"><a name="l00217"></a><span class="lineno"> 217</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> status;</div> |
| <div class="line"><a name="l00218"></a><span class="lineno"> 218</span> </div> |
| <div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  <span class="comment">// Read buffers until we find a sync or reach the end of the scan range. If we read all</span></div> |
| <div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="comment">// the buffers remaining in the scan range and none of them contain a sync (including a</span></div> |
| <div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  <span class="comment">// sync that starts at the end of this scan range and continues into the next one), then</span></div> |
| <div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  <span class="comment">// there are no more syncs in this scan range and we're finished.</span></div> |
| <div class="line"><a name="l00223"></a><span class="lineno"> 223</span>  <span class="keywordflow">while</span> (!<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a0f3599f41ee84455e186a3d1b02fcbb7">eosr</a>()) {</div> |
| <div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="comment">// Check if there's a sync fully contained in the current buffer</span></div> |
| <div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a92a40b73fc3be37a22db492f1bd9071b">GetBuffer</a>(<span class="keyword">true</span>, &buffer, &buffer_len));</div> |
| <div class="line"><a name="l00226"></a><span class="lineno"> 226</span>  offset = <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a92d958bf8e7eaee48dad0398def2589d">FindSyncBlock</a>(buffer, buffer_len, sync, sync_size);</div> |
| <div class="line"><a name="l00227"></a><span class="lineno"> 227</span>  DCHECK_LE(offset, buffer_len);</div> |
| <div class="line"><a name="l00228"></a><span class="lineno"> 228</span>  <span class="keywordflow">if</span> (offset != -1) <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00229"></a><span class="lineno"> 229</span> </div> |
| <div class="line"><a name="l00230"></a><span class="lineno"> 230</span>  <span class="comment">// No sync found in the current buffer, so check if there's a sync spanning the</span></div> |
| <div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  <span class="comment">// current buffer and the next. First we skip so there are sync_size - 1 bytes left,</span></div> |
| <div class="line"><a name="l00232"></a><span class="lineno"> 232</span>  <span class="comment">// then we read these bytes plus the first sync_size - 1 bytes of the next buffer.</span></div> |
| <div class="line"><a name="l00233"></a><span class="lineno"> 233</span>  <span class="comment">// This guarantees that we find any syncs that start in the current buffer and end in</span></div> |
| <div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  <span class="comment">// the next buffer.</span></div> |
| <div class="line"><a name="l00235"></a><span class="lineno"> 235</span>  int64_t to_skip = max(static_cast<int64_t>(0), buffer_len - (sync_size - 1));</div> |
| <div class="line"><a name="l00236"></a><span class="lineno"> 236</span>  <a class="code" href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565">RETURN_IF_FALSE</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac1bae8ae5208e9d2c2547bc0a7efddd6">SkipBytes</a>(to_skip, &<a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a>));</div> |
| <div class="line"><a name="l00237"></a><span class="lineno"> 237</span>  <span class="comment">// Peek so we don't advance stream_ into the next buffer. If we don't find a sync here</span></div> |
| <div class="line"><a name="l00238"></a><span class="lineno"> 238</span>  <span class="comment">// then we'll need to check all of the next buffer, including the first sync_size -1</span></div> |
| <div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  <span class="comment">// bytes.</span></div> |
| <div class="line"><a name="l00240"></a><span class="lineno"> 240</span>  <a class="code" href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565">RETURN_IF_FALSE</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#acfded8cb494dbda9eb700d32295e87be">GetBytes</a>(</div> |
| <div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  (sync_size - 1) * 2, &buffer, &buffer_len, &<a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a>, <span class="keyword">true</span>));</div> |
| <div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  offset = <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a92d958bf8e7eaee48dad0398def2589d">FindSyncBlock</a>(buffer, buffer_len, sync, sync_size);</div> |
| <div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  DCHECK_LE(offset, buffer_len);</div> |
| <div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  <span class="keywordflow">if</span> (offset != -1) <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00245"></a><span class="lineno"> 245</span> </div> |
| <div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  <span class="comment">// No sync starting in this buffer, so advance stream_ to the beginning of the next</span></div> |
| <div class="line"><a name="l00247"></a><span class="lineno"> 247</span>  <span class="comment">// buffer.</span></div> |
| <div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a92a40b73fc3be37a22db492f1bd9071b">GetBuffer</a>(<span class="keyword">false</span>, &buffer, &buffer_len));</div> |
| <div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  }</div> |
| <div class="line"><a name="l00250"></a><span class="lineno"> 250</span> </div> |
| <div class="line"><a name="l00251"></a><span class="lineno"> 251</span>  <span class="keywordflow">if</span> (offset == -1) {</div> |
| <div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  <span class="comment">// No more syncs in this scan range</span></div> |
| <div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  DCHECK(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a0f3599f41ee84455e186a3d1b02fcbb7">eosr</a>());</div> |
| <div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00255"></a><span class="lineno"> 255</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00256"></a><span class="lineno"> 256</span>  }</div> |
| <div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  DCHECK_GE(offset, sync_size);</div> |
| <div class="line"><a name="l00258"></a><span class="lineno"> 258</span> </div> |
| <div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  <span class="comment">// Make sure sync starts in our scan range</span></div> |
| <div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  <span class="keywordflow">if</span> (offset - sync_size >= <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a48c8ea0f83bb58e810e5148e0addccb0">bytes_left</a>()) {</div> |
| <div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  }</div> |
| <div class="line"><a name="l00264"></a><span class="lineno"> 264</span> </div> |
| <div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  <a class="code" href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565">RETURN_IF_FALSE</a>(<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac1bae8ae5208e9d2c2547bc0a7efddd6">SkipBytes</a>(offset, &<a class="code" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a>));</div> |
| <div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"Found sync for: "</span> << <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">filename</a>()</div> |
| <div class="line"><a name="l00267"></a><span class="lineno"> 267</span>  << <span class="stringliteral">" at "</span> << <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>() - sync_size;</div> |
| <div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#a1a423b75d11aaf04deb06633355bfb08">eof</a>()) <a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">finished_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5c06eddcf4b24c054328e7eeb28c5f1b">block_start_</a> = <a class="code" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a>-><a class="code" href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">file_offset</a>();</div> |
| <div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  ++<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">num_syncs_</a>;</div> |
| <div class="line"><a name="l00271"></a><span class="lineno"> 271</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00272"></a><span class="lineno"> 272</span> }</div> |
| <div class="line"><a name="l00273"></a><span class="lineno"> 273</span> </div> |
| <div class="line"><a name="l00274"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#adff67c7bd7dd6522b6db5682deefcdfe"> 274</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#adff67c7bd7dd6522b6db5682deefcdfe">BaseSequenceScanner::CloseFileRanges</a>(<span class="keyword">const</span> <span class="keywordtype">char</span>* filename) {</div> |
| <div class="line"><a name="l00275"></a><span class="lineno"> 275</span>  DCHECK(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a0184e49248bc243ed45c77b8fb7aeac4">only_parsing_header_</a>);</div> |
| <div class="line"><a name="l00276"></a><span class="lineno"> 276</span>  <a class="code" href="structimpala_1_1HdfsFileDesc.html">HdfsFileDesc</a>* desc = <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#a0521140f7c60d795fb7bd2f44c42c9bf">GetFileDesc</a>(filename);</div> |
| <div class="line"><a name="l00277"></a><span class="lineno"> 277</span>  <span class="keyword">const</span> vector<DiskIoMgr::ScanRange*>& splits = desc-><a class="code" href="structimpala_1_1HdfsFileDesc.html#a513879a3999adaee5fbde411757e3703">splits</a>;</div> |
| <div class="line"><a name="l00278"></a><span class="lineno"> 278</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < splits.size(); ++i) {</div> |
| <div class="line"><a name="l00279"></a><span class="lineno"> 279</span>  <a class="code" href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a393b2ff9306ac4becc7ae3838099a4c7">bytes_skipped_counter_</a>, splits[i]->len());</div> |
| <div class="line"><a name="l00280"></a><span class="lineno"> 280</span>  <a class="code" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a>-><a class="code" href="classimpala_1_1HdfsScanNode.html#a6193af89a13de0a3b8d45bc0e5cd288c">RangeComplete</a>(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#aa3ac35eed32d440edbd401600d9ee0c4">file_format</a>(), THdfsCompression::NONE);</div> |
| <div class="line"><a name="l00281"></a><span class="lineno"> 281</span>  }</div> |
| <div class="line"><a name="l00282"></a><span class="lineno"> 282</span> }</div> |
| <div class="line"><a name="l00283"></a><span class="lineno"> 283</span> </div> |
| <div class="line"><a name="l00284"></a><span class="lineno"><a class="line" href="classimpala_1_1BaseSequenceScanner.html#a9586fc33865674ed488abbbd0d7e2003"> 284</a></span> <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a9586fc33865674ed488abbbd0d7e2003">BaseSequenceScanner::ReadPastSize</a>(int64_t file_offset) {</div> |
| <div class="line"><a name="l00285"></a><span class="lineno"> 285</span>  DCHECK_GE(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a4e114262860e1f844d498c802336973c">total_block_size_</a>, 0);</div> |
| <div class="line"><a name="l00286"></a><span class="lineno"> 286</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a4e114262860e1f844d498c802336973c">total_block_size_</a> == 0) {</div> |
| <div class="line"><a name="l00287"></a><span class="lineno"> 287</span>  <span class="comment">// This scan range didn't include a complete block, so we have no idea how many bytes</span></div> |
| <div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  <span class="comment">// remain in the block. Guess.</span></div> |
| <div class="line"><a name="l00289"></a><span class="lineno"> 289</span>  <span class="keywordflow">return</span> <a class="code" href="base-sequence-scanner_8cc.html#a4fd070e0de326e5cfa25a3bf0a8747e7">REMAINING_BLOCK_SIZE_GUESS</a>;</div> |
| <div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  }</div> |
| <div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  DCHECK_GE(<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">num_syncs_</a>, 2);</div> |
| <div class="line"><a name="l00292"></a><span class="lineno"> 292</span>  <span class="keywordtype">int</span> average_block_size = <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a4e114262860e1f844d498c802336973c">total_block_size_</a> / (<a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">num_syncs_</a> - 1);</div> |
| <div class="line"><a name="l00293"></a><span class="lineno"> 293</span> </div> |
| <div class="line"><a name="l00294"></a><span class="lineno"> 294</span>  <span class="comment">// Number of bytes read in the current block</span></div> |
| <div class="line"><a name="l00295"></a><span class="lineno"> 295</span>  <span class="keywordtype">int</span> block_bytes_read = file_offset - <a class="code" href="classimpala_1_1BaseSequenceScanner.html#a5c06eddcf4b24c054328e7eeb28c5f1b">block_start_</a>;</div> |
| <div class="line"><a name="l00296"></a><span class="lineno"> 296</span>  DCHECK_GE(block_bytes_read, 0);</div> |
| <div class="line"><a name="l00297"></a><span class="lineno"> 297</span>  <span class="keywordtype">int</span> bytes_left = max(average_block_size - block_bytes_read, 0);</div> |
| <div class="line"><a name="l00298"></a><span class="lineno"> 298</span>  <span class="comment">// Include some padding</span></div> |
| <div class="line"><a name="l00299"></a><span class="lineno"> 299</span>  bytes_left += average_block_size * <a class="code" href="base-sequence-scanner_8cc.html#ac5a0880457028b83916eadd56b9dcf20">BLOCK_SIZE_PADDING_PERCENT</a>;</div> |
| <div class="line"><a name="l00300"></a><span class="lineno"> 300</span> </div> |
| <div class="line"><a name="l00301"></a><span class="lineno"> 301</span>  <span class="keywordtype">int</span> max_read_size = <a class="code" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#a5e664a4e409bd8e13916e138a9b6e4a9">io_mgr</a>()-><a class="code" href="classimpala_1_1DiskIoMgr.html#a4d68607c8cf8f9003f47891af6627103">max_read_buffer_size</a>();</div> |
| <div class="line"><a name="l00302"></a><span class="lineno"> 302</span>  <span class="keywordflow">return</span> min(max(bytes_left, <a class="code" href="base-sequence-scanner_8cc.html#a3286d10cafc052ea336a292d9d9ab686">MIN_SYNC_READ_SIZE</a>), max_read_size);</div> |
| <div class="line"><a name="l00303"></a><span class="lineno"> 303</span> }</div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a6b84385660f88ed592fd61ac62c15793"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a6b84385660f88ed592fd61ac62c15793">impala::BaseSequenceScanner::IssueInitialRanges</a></div><div class="ttdeci">static Status IssueInitialRanges(HdfsScanNode *scan_node, const std::vector< HdfsFileDesc * > &files)</div><div class="ttdoc">Issue the initial ranges for all sequence container files. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00040">base-sequence-scanner.cc:40</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_acb22ac5214eff45da580c5be90c4d08f"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#acb22ac5214eff45da580c5be90c4d08f">impala::ScannerContext::Stream::set_read_past_size_cb</a></div><div class="ttdeci">void set_read_past_size_cb(ReadPastSizeCallback cb)</div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00106">scanner-context.h:106</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a8d995c8326def52de037ecf074fc2654"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a8d995c8326def52de037ecf074fc2654">impala::HdfsScanner::decompressor_</a></div><div class="ttdeci">boost::scoped_ptr< Codec > decompressor_</div><div class="ttdoc">Decompressor class to use, if any. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00198">hdfs-scanner.h:198</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a542176620ff5194866253070a539134a"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a542176620ff5194866253070a539134a">impala::HdfsScanner::InitNewRange</a></div><div class="ttdeci">virtual Status InitNewRange()=0</div><div class="ttdoc">Reset internal state for a new scan range. </div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a1b361d5cc69ef26b606349a26c1a8415"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">impala::HdfsScanner::scan_node_</a></div><div class="ttdeci">HdfsScanNode * scan_node_</div><div class="ttdoc">The scan node that started this scanner. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00141">hdfs-scanner.h:141</a></div></div> |
| <div class="ttc" id="hdfs-scan-node_8h_html"><div class="ttname"><a href="hdfs-scan-node_8h.html">hdfs-scan-node.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_a92a40b73fc3be37a22db492f1bd9071b"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#a92a40b73fc3be37a22db492f1bd9071b">impala::ScannerContext::Stream::GetBuffer</a></div><div class="ttdeci">Status GetBuffer(bool peek, uint8_t **buffer, int64_t *out_len)</div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8cc_source.html#l00171">scanner-context.cc:171</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a0184e49248bc243ed45c77b8fb7aeac4"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a0184e49248bc243ed45c77b8fb7aeac4">impala::BaseSequenceScanner::only_parsing_header_</a></div><div class="ttdeci">bool only_parsing_header_</div><div class="ttdoc">If true, this scanner object is only for processing the header. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00130">base-sequence-scanner.h:130</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_ac7f1cf14b61ed091ee80c4f2dbadba4c"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#ac7f1cf14b61ed091ee80c4f2dbadba4c">impala::BaseSequenceScanner::header_</a></div><div class="ttdeci">FileHeader * header_</div><div class="ttdoc">File header for this scan range. This is not owned by the parent scan node. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00127">base-sequence-scanner.h:127</a></div></div> |
| <div class="ttc" id="structimpala_1_1StringValue_html"><div class="ttname"><a href="structimpala_1_1StringValue.html">impala::StringValue</a></div><div class="ttdef"><b>Definition:</b> <a href="string-value_8h_source.html#l00033">string-value.h:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_a0f3599f41ee84455e186a3d1b02fcbb7"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#a0f3599f41ee84455e186a3d1b02fcbb7">impala::ScannerContext::Stream::eosr</a></div><div class="ttdeci">bool eosr() const </div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00113">scanner-context.h:113</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_a48c8ea0f83bb58e810e5148e0addccb0"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#a48c8ea0f83bb58e810e5148e0addccb0">impala::ScannerContext::Stream::bytes_left</a></div><div class="ttdeci">int64_t bytes_left()</div><div class="ttdoc">Return the number of bytes left in the range for this stream. </div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00109">scanner-context.h:109</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_ae86f3019c271de0d0593204003ee1653"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#ae86f3019c271de0d0593204003ee1653">impala::HdfsScanner::data_buffer_pool_</a></div><div class="ttdeci">boost::scoped_ptr< MemPool > data_buffer_pool_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00205">hdfs-scanner.h:205</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a5814338e23714b1d66022f75bbb48c5b"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a5814338e23714b1d66022f75bbb48c5b">impala::BaseSequenceScanner::SYNC_HASH_SIZE</a></div><div class="ttdeci">static const int SYNC_HASH_SIZE</div><div class="ttdoc">Size of the sync hash field. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00049">base-sequence-scanner.h:49</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a7fb6c7a9e0bdf14e594d412f7e1bc600"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a7fb6c7a9e0bdf14e594d412f7e1bc600">impala::BaseSequenceScanner::ProcessRange</a></div><div class="ttdeci">virtual Status ProcessRange()=0</div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_ac583e95bbd1397d0ecbe526e7a35a445"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#ac583e95bbd1397d0ecbe526e7a35a445">impala::BaseSequenceScanner::SkipToSync</a></div><div class="ttdeci">Status SkipToSync(const uint8_t *sync, int sync_size)</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00212">base-sequence-scanner.cc:212</a></div></div> |
| <div class="ttc" id="classimpala_1_1StringSearch_html_a4ab30c893cd9036d6ae5684e91b19012"><div class="ttname"><a href="classimpala_1_1StringSearch.html#a4ab30c893cd9036d6ae5684e91b19012">impala::StringSearch::Search</a></div><div class="ttdeci">int Search(const StringValue *str) const </div><div class="ttdef"><b>Definition:</b> <a href="string-search_8h_source.html#l00110">string-search.h:110</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a8196c5dc7e842622df7ab43b8cd68397"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">impala::HdfsScanner::parse_status_</a></div><div class="ttdeci">Status parse_status_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00195">hdfs-scanner.h:195</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a50b59c9628eedf49aa82643ce601967d"><div class="ttname"><a href="namespaceimpala.html#a50b59c9628eedf49aa82643ce601967d">impala::hash</a></div><div class="ttdeci">const StringSearch UrlParser::hash_search & hash</div><div class="ttdef"><b>Definition:</b> <a href="url-parser_8cc_source.html#l00041">url-parser.cc:41</a></div></div> |
| <div class="ttc" id="status_8h_html_a85f7d0e774e15eb35b74f53264305e16"><div class="ttname"><a href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a></div><div class="ttdeci">#define RETURN_IF_ERROR(stmt)</div><div class="ttdoc">some generally useful macros </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00242">status.h:242</a></div></div> |
| <div class="ttc" id="structimpala_1_1ScanRangeMetadata_html"><div class="ttname"><a href="structimpala_1_1ScanRangeMetadata.html">impala::ScanRangeMetadata</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8h_source.html#l00078">hdfs-scan-node.h:78</a></div></div> |
| <div class="ttc" id="classimpala_1_1StringSearch_html"><div class="ttname"><a href="classimpala_1_1StringSearch.html">impala::StringSearch</a></div><div class="ttdef"><b>Definition:</b> <a href="string-search_8h_source.html#l00083">string-search.h:83</a></div></div> |
| <div class="ttc" id="codec_8h_html"><div class="ttname"><a href="codec_8h.html">codec.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a36e661fbaff23879b78f53cdc1a56552"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a36e661fbaff23879b78f53cdc1a56552">impala::HdfsScanner::AddFinalRowBatch</a></div><div class="ttdeci">void AddFinalRowBatch()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8cc_source.html#l00145">hdfs-scanner.cc:145</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a5c06eddcf4b24c054328e7eeb28c5f1b"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a5c06eddcf4b24c054328e7eeb28c5f1b">impala::BaseSequenceScanner::block_start_</a></div><div class="ttdeci">int64_t block_start_</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00150">base-sequence-scanner.h:150</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html_a3405cd6d94586c152efae17e6cfbd787"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html#a3405cd6d94586c152efae17e6cfbd787">impala::HdfsScanNode::SetFileMetadata</a></div><div class="ttdeci">void SetFileMetadata(const std::string &filename, void *metadata)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8cc_source.html#l00211">hdfs-scan-node.cc:211</a></div></div> |
| <div class="ttc" id="structimpala_1_1BaseSequenceScanner_1_1FileHeader_html_a87d5f2145212f3a762255c4713357965"><div class="ttname"><a href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a87d5f2145212f3a762255c4713357965">impala::BaseSequenceScanner::FileHeader::header_size</a></div><div class="ttdeci">int64_t header_size</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00070">base-sequence-scanner.h:70</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html_adab35319d4dbff9af621387591a32c2c"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html#adab35319d4dbff9af621387591a32c2c">impala::HdfsScanNode::GetFileMetadata</a></div><div class="ttdeci">void * GetFileMetadata(const std::string &filename)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8cc_source.html#l00217">hdfs-scan-node.cc:217</a></div></div> |
| <div class="ttc" id="string-search_8h_html"><div class="ttname"><a href="string-search_8h.html">string-search.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_html"><div class="ttname"><a href="classimpala_1_1ScannerContext.html">impala::ScannerContext</a></div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00055">scanner-context.h:55</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_acc3c090b8b5e1c282c828be221697d5b"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#acc3c090b8b5e1c282c828be221697d5b">impala::BaseSequenceScanner::Prepare</a></div><div class="ttdeci">virtual Status Prepare(ScannerContext *context)</div><div class="ttdoc">One-time initialisation of state that is constant across scan ranges. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00074">base-sequence-scanner.cc:74</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_a0842f330c18b3e4e53e13655add0310a"><div class="ttname"><a href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a></div><div class="ttdeci">#define COUNTER_ADD(c, v)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00055">runtime-profile.h:55</a></div></div> |
| <div class="ttc" id="structimpala_1_1ScanRangeMetadata_html_a7a7171d1dc269bea3f3dd5636e4bb774"><div class="ttname"><a href="structimpala_1_1ScanRangeMetadata.html#a7a7171d1dc269bea3f3dd5636e4bb774">impala::ScanRangeMetadata::partition_id</a></div><div class="ttdeci">int64_t partition_id</div><div class="ttdoc">The partition id that this range is part of. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8h_source.html#l00080">hdfs-scan-node.h:80</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_ac50f8d6efde3334ce557e19e9b2ab7f9"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#ac50f8d6efde3334ce557e19e9b2ab7f9">impala::ScannerContext::Stream::file_offset</a></div><div class="ttdeci">int64_t file_offset() const </div><div class="ttdoc">Returns the buffer's current offset in the file. </div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00123">scanner-context.h:123</a></div></div> |
| <div class="ttc" id="structimpala_1_1BaseSequenceScanner_1_1FileHeader_html_a394cc7bb9ae9309bce7f4d9167b083fc"><div class="ttname"><a href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a394cc7bb9ae9309bce7f4d9167b083fc">impala::BaseSequenceScanner::FileHeader::compression_type</a></div><div class="ttdeci">THdfsCompression::type compression_type</div><div class="ttdoc">Enum for compression type. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00066">base-sequence-scanner.h:66</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html_a6193af89a13de0a3b8d45bc0e5cd288c"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html#a6193af89a13de0a3b8d45bc0e5cd288c">impala::HdfsScanNode::RangeComplete</a></div><div class="ttdeci">void RangeComplete(const THdfsFileFormat::type &file_type, const THdfsCompression::type &compression_type)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8cc_source.html#l00924">hdfs-scan-node.cc:924</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_a8e06eee6a1aae5aa71b936be43d6aee4"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#a8e06eee6a1aae5aa71b936be43d6aee4">impala::ScannerContext::Stream::filename</a></div><div class="ttdeci">const char * filename()</div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00118">scanner-context.h:118</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a61d4a073305f6befe9f12bad12f9de48"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a61d4a073305f6befe9f12bad12f9de48">impala::HdfsScanner::Close</a></div><div class="ttdeci">virtual void Close()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8cc_source.html#l00082">hdfs-scanner.cc:82</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_acfded8cb494dbda9eb700d32295e87be"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#acfded8cb494dbda9eb700d32295e87be">impala::ScannerContext::Stream::GetBytes</a></div><div class="ttdeci">bool GetBytes(int64_t requested_len, uint8_t **buffer, int64_t *out_len, Status *status, bool peek=false)</div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8inline_8h_source.html#l00031">scanner-context.inline.h:31</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html"><div class="ttname"><a href="classimpala_1_1Status.html">impala::Status</a></div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00081">status.h:81</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a8d1c8ea0d46e9378c6db2c24395a3f78"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">impala::HdfsScanner::state_</a></div><div class="ttdeci">RuntimeState * state_</div><div class="ttdoc">RuntimeState for error reporting. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00144">hdfs-scanner.h:144</a></div></div> |
| <div class="ttc" id="base-sequence-scanner_8cc_html_a4fd070e0de326e5cfa25a3bf0a8747e7"><div class="ttname"><a href="base-sequence-scanner_8cc.html#a4fd070e0de326e5cfa25a3bf0a8747e7">REMAINING_BLOCK_SIZE_GUESS</a></div><div class="ttdeci">static const int REMAINING_BLOCK_SIZE_GUESS</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00034">base-sequence-scanner.cc:34</a></div></div> |
| <div class="ttc" id="base-sequence-scanner_8cc_html_a2a5a24ebaf26175881d20696ab510565"><div class="ttname"><a href="base-sequence-scanner_8cc.html#a2a5a24ebaf26175881d20696ab510565">RETURN_IF_FALSE</a></div><div class="ttdeci">#define RETURN_IF_FALSE(x)</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00038">base-sequence-scanner.cc:38</a></div></div> |
| <div class="ttc" id="base-sequence-scanner_8cc_html_a3286d10cafc052ea336a292d9d9ab686"><div class="ttname"><a href="base-sequence-scanner_8cc.html#a3286d10cafc052ea336a292d9d9ab686">MIN_SYNC_READ_SIZE</a></div><div class="ttdeci">static const int MIN_SYNC_READ_SIZE</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00035">base-sequence-scanner.cc:35</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_af924a2719c3aedb31c17bd3f0bd80af3"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#af924a2719c3aedb31c17bd3f0bd80af3">impala::BaseSequenceScanner::ReadFileHeader</a></div><div class="ttdeci">virtual Status ReadFileHeader()=0</div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_acc59a2d650453eb5c12670519342c846"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#acc59a2d650453eb5c12670519342c846">impala::BaseSequenceScanner::ProcessSplit</a></div><div class="ttdeci">virtual Status ProcessSplit()</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00100">base-sequence-scanner.cc:100</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a38bb92ef6416325aefafbcaed8f5010c"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a38bb92ef6416325aefafbcaed8f5010c">impala::BaseSequenceScanner::ReadSync</a></div><div class="ttdeci">Status ReadSync()</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00170">base-sequence-scanner.cc:170</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a9586fc33865674ed488abbbd0d7e2003"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a9586fc33865674ed488abbbd0d7e2003">impala::BaseSequenceScanner::ReadPastSize</a></div><div class="ttdeci">int ReadPastSize(int64_t file_offset)</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00284">base-sequence-scanner.cc:284</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html_a0521140f7c60d795fb7bd2f44c42c9bf"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html#a0521140f7c60d795fb7bd2f44c42c9bf">impala::HdfsScanNode::GetFileDesc</a></div><div class="ttdeci">HdfsFileDesc * GetFileDesc(const std::string &filename)</div><div class="ttdoc">Returns the file desc for 'filename'. Returns NULL if filename is invalid. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8cc_source.html#l00206">hdfs-scan-node.cc:206</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_aaf076b1879d9e712b08ba3a6e24607b9"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#aaf076b1879d9e712b08ba3a6e24607b9">impala::RuntimeState::LogError</a></div><div class="ttdeci">bool LogError(const ErrorMsg &msg)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8cc_source.html#l00224">runtime-state.cc:224</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html_afe9bc42ad3846bde9f7113e5f1c133dc"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html#afe9bc42ad3846bde9f7113e5f1c133dc">impala::HdfsScanNode::AddDiskIoRanges</a></div><div class="ttdeci">Status AddDiskIoRanges(const std::vector< DiskIoMgr::ScanRange * > &ranges)</div><div class="ttdoc">Adds ranges to the io mgr queue and starts up new scanner threads if possible. </div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html"><div class="ttname"><a href="classimpala_1_1RuntimeState.html">impala::RuntimeState</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00069">runtime-state.h:69</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a327e57a4c61d1387f8b52fce20e89a34"><div class="ttname"><a href="classimpala_1_1Status.html#a327e57a4c61d1387f8b52fce20e89a34">impala::Status::IsCancelled</a></div><div class="ttdeci">bool IsCancelled() const </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00174">status.h:174</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_a1a423b75d11aaf04deb06633355bfb08"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#a1a423b75d11aaf04deb06633355bfb08">impala::ScannerContext::Stream::eof</a></div><div class="ttdeci">bool eof() const </div><div class="ttdoc">If true, the stream has reached the end of the file. </div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00116">scanner-context.h:116</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a4c01f227dc12d859d5824e7c91d6d1aa"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">impala::RuntimeState::obj_pool</a></div><div class="ttdeci">ObjectPool * obj_pool() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00092">runtime-state.h:92</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a5be274fb82550166b716199805307992"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a5be274fb82550166b716199805307992">impala::BaseSequenceScanner::num_syncs_</a></div><div class="ttdeci">int num_syncs_</div><div class="ttdoc">The number of syncs seen by this scanner so far. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00157">base-sequence-scanner.h:157</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_aae0d5cf9f400c07db437b4fa94b6be8a"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#aae0d5cf9f400c07db437b4fa94b6be8a">impala::BaseSequenceScanner::~BaseSequenceScanner</a></div><div class="ttdeci">virtual ~BaseSequenceScanner()</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00071">base-sequence-scanner.cc:71</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_ac1bae8ae5208e9d2c2547bc0a7efddd6"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#ac1bae8ae5208e9d2c2547bc0a7efddd6">impala::ScannerContext::Stream::SkipBytes</a></div><div class="ttdeci">bool SkipBytes(int64_t length, Status *)</div><div class="ttdoc">Skip over the next length bytes in the specified HDFS file. </div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8inline_8h_source.html#l00070">scanner-context.inline.h:70</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_aac035c52016117af399543521f069c51"><div class="ttname"><a href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a></div><div class="ttdeci">#define ADD_COUNTER(profile, name, unit)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00047">runtime-profile.h:47</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a393b2ff9306ac4becc7ae3838099a4c7"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a393b2ff9306ac4becc7ae3838099a4c7">impala::BaseSequenceScanner::bytes_skipped_counter_</a></div><div class="ttdeci">RuntimeProfile::Counter * bytes_skipped_counter_</div><div class="ttdoc">Number of bytes skipped when advancing to next sync on error. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00174">base-sequence-scanner.h:174</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html">impala::HdfsScanNode</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8h_source.html#l00104">hdfs-scan-node.h:104</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a1a37621b600b55d162832e8744fc6a17"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a1a37621b600b55d162832e8744fc6a17">impala::BaseSequenceScanner::Close</a></div><div class="ttdeci">virtual void Close()</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00082">base-sequence-scanner.cc:82</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_ae3d4eec40706fa6d653246a19d17e006"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#ae3d4eec40706fa6d653246a19d17e006">impala::BaseSequenceScanner::SYNC_MARKER</a></div><div class="ttdeci">static const int SYNC_MARKER</div><div class="ttdoc">Sync indicator. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00124">base-sequence-scanner.h:124</a></div></div> |
| <div class="ttc" id="runtime-state_8h_html"><div class="ttname"><a href="runtime-state_8h.html">runtime-state.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_aa3ac35eed32d440edbd401600d9ee0c4"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#aa3ac35eed32d440edbd401600d9ee0c4">impala::BaseSequenceScanner::file_format</a></div><div class="ttdeci">virtual THdfsFileFormat::type file_format() const =0</div><div class="ttdoc">Returns type of scanner: e.g. rcfile, seqfile. </div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a94b437624601d6e7626e9b3d59cd87da"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a94b437624601d6e7626e9b3d59cd87da">impala::HdfsScanner::AttachPool</a></div><div class="ttdeci">void AttachPool(MemPool *pool, bool commit_batch)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00256">hdfs-scanner.h:256</a></div></div> |
| <div class="ttc" id="classimpala_1_1ErrorMsg_html"><div class="ttname"><a href="classimpala_1_1ErrorMsg.html">impala::ErrorMsg</a></div><div class="ttdef"><b>Definition:</b> <a href="error-util_8h_source.html#l00047">error-util.h:47</a></div></div> |
| <div class="ttc" id="scanner-context_8inline_8h_html"><div class="ttname"><a href="scanner-context_8inline_8h.html">scanner-context.inline.h</a></div></div> |
| <div class="ttc" id="structimpala_1_1BaseSequenceScanner_1_1FileHeader_html"><div class="ttname"><a href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html">impala::BaseSequenceScanner::FileHeader</a></div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00053">base-sequence-scanner.h:53</a></div></div> |
| <div class="ttc" id="base-sequence-scanner_8cc_html_ac5a0880457028b83916eadd56b9dcf20"><div class="ttname"><a href="base-sequence-scanner_8cc.html#ac5a0880457028b83916eadd56b9dcf20">BLOCK_SIZE_PADDING_PERCENT</a></div><div class="ttdeci">static const double BLOCK_SIZE_PADDING_PERCENT</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00033">base-sequence-scanner.cc:33</a></div></div> |
| <div class="ttc" id="structimpala_1_1BaseSequenceScanner_1_1FileHeader_html_ab57f8bf94e3f23b8e9ed076b786d7a23"><div class="ttname"><a href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#ab57f8bf94e3f23b8e9ed076b786d7a23">impala::BaseSequenceScanner::FileHeader::sync</a></div><div class="ttdeci">uint8_t sync[SYNC_HASH_SIZE]</div><div class="ttdoc">The sync hash for this file. </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00057">base-sequence-scanner.h:57</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsFileDesc_html_a513879a3999adaee5fbde411757e3703"><div class="ttname"><a href="structimpala_1_1HdfsFileDesc.html#a513879a3999adaee5fbde411757e3703">impala::HdfsFileDesc::splits</a></div><div class="ttdeci">std::vector< DiskIoMgr::ScanRange * > splits</div><div class="ttdoc">Splits (i.e. raw byte ranges) for this file, assigned to this scan node. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8h_source.html#l00070">hdfs-scan-node.h:70</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanNode_html_a5314b840efa7625e54ce930ee1986bb1"><div class="ttname"><a href="classimpala_1_1HdfsScanNode.html#a5314b840efa7625e54ce930ee1986bb1">impala::HdfsScanNode::AllocateScanRange</a></div><div class="ttdeci">DiskIoMgr::ScanRange * AllocateScanRange(hdfsFS fs, const char *file, int64_t len, int64_t offset, int64_t partition_id, int disk_id, bool try_cache, bool expected_local, int64_t mtime)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8cc_source.html#l00183">hdfs-scan-node.cc:183</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a580565665ea944eb64f3f495b1bee1e0"><div class="ttname"><a href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">impala::Status::OK</a></div><div class="ttdeci">static const Status OK</div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00087">status.h:87</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html">impala::HdfsScanner</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00091">hdfs-scanner.h:91</a></div></div> |
| <div class="ttc" id="partitioning-throughput-test_8cc_html_ad172ecfd5c31c5df6282c2b4bc322bf7"><div class="ttname"><a href="partitioning-throughput-test_8cc.html#ad172ecfd5c31c5df6282c2b4bc322bf7">offset</a></div><div class="ttdeci">uint8_t offset[7 *64-sizeof(uint64_t)]</div><div class="ttdef"><b>Definition:</b> <a href="partitioning-throughput-test_8cc_source.html#l00037">partitioning-throughput-test.cc:37</a></div></div> |
| <div class="ttc" id="names_8h_html"><div class="ttname"><a href="names_8h.html">names.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ObjectPool_html_ab191078c99825682a0a9915ca1e0c05c"><div class="ttname"><a href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">impala::ObjectPool::Add</a></div><div class="ttdeci">T * Add(T *t)</div><div class="ttdef"><b>Definition:</b> <a href="object-pool_8h_source.html#l00042">object-pool.h:42</a></div></div> |
| <div class="ttc" id="classimpala_1_1ReadWriteUtil_html_a9c6a8fcabb5a5edb77de5866d5a4f7f4"><div class="ttname"><a href="classimpala_1_1ReadWriteUtil.html#a9c6a8fcabb5a5edb77de5866d5a4f7f4">impala::ReadWriteUtil::HexDump</a></div><div class="ttdeci">static std::string HexDump(const uint8_t *buf, int64_t length)</div><div class="ttdoc">Dump the first length bytes of buf to a Hex string. </div><div class="ttdef"><b>Definition:</b> <a href="read-write-util_8cc_source.html#l00072">read-write-util.cc:72</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_afe14635279d3cd633d08ecbadee51019"><div class="ttname"><a href="classimpala_1_1Status.html#afe14635279d3cd633d08ecbadee51019">impala::Status::msg</a></div><div class="ttdeci">const ErrorMsg & msg() const </div><div class="ttdoc">Returns the error message associated with a non-successful status. </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00189">status.h:189</a></div></div> |
| <div class="ttc" id="logging_8h_html_a0bdc3628c740982e847cd24114c07994"><div class="ttname"><a href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a></div><div class="ttdeci">#define VLOG_FILE</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00058">logging.h:58</a></div></div> |
| <div class="ttc" id="classimpala_1_1DiskIoMgr_html_a4d68607c8cf8f9003f47891af6627103"><div class="ttname"><a href="classimpala_1_1DiskIoMgr.html#a4d68607c8cf8f9003f47891af6627103">impala::DiskIoMgr::max_read_buffer_size</a></div><div class="ttdeci">int max_read_buffer_size() const </div><div class="ttdoc">Returns the maximum read buffer size. </div><div class="ttdef"><b>Definition:</b> <a href="disk-io-mgr_8h_source.html#l00590">disk-io-mgr.h:590</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a92d958bf8e7eaee48dad0398def2589d"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a92d958bf8e7eaee48dad0398def2589d">impala::BaseSequenceScanner::FindSyncBlock</a></div><div class="ttdeci">int FindSyncBlock(const uint8_t *buffer, int buffer_len, const uint8_t *sync, int sync_len)</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00194">base-sequence-scanner.cc:194</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a4e114262860e1f844d498c802336973c"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a4e114262860e1f844d498c802336973c">impala::BaseSequenceScanner::total_block_size_</a></div><div class="ttdeci">int total_block_size_</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00154">base-sequence-scanner.h:154</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_ad449a8708ed26241be77b023c95ac7f0"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#ad449a8708ed26241be77b023c95ac7f0">impala::RuntimeState::abort_on_error</a></div><div class="ttdeci">bool abort_on_error() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00099">runtime-state.h:99</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a95ba859e42fe93445b340533220836ac"><div class="ttname"><a href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">impala::Status::ok</a></div><div class="ttdeci">bool ok() const </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00172">status.h:172</a></div></div> |
| <div class="ttc" id="base-sequence-scanner_8h_html"><div class="ttname"><a href="base-sequence-scanner_8h.html">base-sequence-scanner.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a5e664a4e409bd8e13916e138a9b6e4a9"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a5e664a4e409bd8e13916e138a9b6e4a9">impala::RuntimeState::io_mgr</a></div><div class="ttdeci">DiskIoMgr * io_mgr()</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00139">runtime-state.h:139</a></div></div> |
| <div class="ttc" id="structimpala_1_1BaseSequenceScanner_1_1FileHeader_html_a17899a9d57bf0246139d62029d40a7a3"><div class="ttname"><a href="structimpala_1_1BaseSequenceScanner_1_1FileHeader.html#a17899a9d57bf0246139d62029d40a7a3">impala::BaseSequenceScanner::FileHeader::is_compressed</a></div><div class="ttdeci">bool is_compressed</div><div class="ttdoc">true if the file is compressed </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00060">base-sequence-scanner.h:60</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_ac62358e62350918ee1a0c429cd74fd5c"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#ac62358e62350918ee1a0c429cd74fd5c">impala::BaseSequenceScanner::HEADER_SIZE</a></div><div class="ttdeci">static const int HEADER_SIZE</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00121">base-sequence-scanner.h:121</a></div></div> |
| <div class="ttc" id="classimpala_1_1DiskIoMgr_1_1ScanRange_html"><div class="ttname"><a href="classimpala_1_1DiskIoMgr_1_1ScanRange.html">impala::DiskIoMgr::ScanRange</a></div><div class="ttdef"><b>Definition:</b> <a href="disk-io-mgr_8h_source.html#l00295">disk-io-mgr.h:295</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_aa539050780939fc85267c63e9f262e05"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">impala::HdfsScanner::stream_</a></div><div class="ttdeci">ScannerContext::Stream * stream_</div><div class="ttdoc">The first stream for context_. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8h_source.html#l00150">hdfs-scanner.h:150</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScannerContext_1_1Stream_html_a53b77f52ccee137fbe9a39083bac3488"><div class="ttname"><a href="classimpala_1_1ScannerContext_1_1Stream.html#a53b77f52ccee137fbe9a39083bac3488">impala::ScannerContext::Stream::set_contains_tuple_data</a></div><div class="ttdeci">void set_contains_tuple_data(bool v)</div><div class="ttdef"><b>Definition:</b> <a href="scanner-context_8h_source.html#l00097">scanner-context.h:97</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_aa3430957bf7a888fddda01ac950ea46b"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#aa3430957bf7a888fddda01ac950ea46b">impala::BaseSequenceScanner::finished_</a></div><div class="ttdeci">bool finished_</div><div class="ttdoc">finished_ is set by ReadSync() and SkipToSync(). </div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8h_source.html#l00146">base-sequence-scanner.h:146</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a39b7bde73a8803e2ceaf3b97d0fe94eb"><div class="ttname"><a href="classimpala_1_1Status.html#a39b7bde73a8803e2ceaf3b97d0fe94eb">impala::Status::IsMemLimitExceeded</a></div><div class="ttdeci">bool IsMemLimitExceeded() const </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00178">status.h:178</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsScanner_html_a99165297c6e6546edbee741a54dfd194"><div class="ttname"><a href="classimpala_1_1HdfsScanner.html#a99165297c6e6546edbee741a54dfd194">impala::HdfsScanner::Prepare</a></div><div class="ttdeci">virtual Status Prepare(ScannerContext *context)</div><div class="ttdoc">One-time initialisation of state that is constant across scan ranges. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scanner_8cc_source.html#l00071">hdfs-scanner.cc:71</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_adff67c7bd7dd6522b6db5682deefcdfe"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#adff67c7bd7dd6522b6db5682deefcdfe">impala::BaseSequenceScanner::CloseFileRanges</a></div><div class="ttdeci">void CloseFileRanges(const char *file)</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00274">base-sequence-scanner.cc:274</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsFileDesc_html"><div class="ttname"><a href="structimpala_1_1HdfsFileDesc.html">impala::HdfsFileDesc</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-scan-node_8h_source.html#l00053">hdfs-scan-node.h:53</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExecNode_html_ad062f28c78add1a869fffdab9419b09d"><div class="ttname"><a href="classimpala_1_1ExecNode.html#ad062f28c78add1a869fffdab9419b09d">impala::ExecNode::runtime_profile</a></div><div class="ttdeci">RuntimeProfile * runtime_profile()</div><div class="ttdef"><b>Definition:</b> <a href="exec-node_8h_source.html#l00161">exec-node.h:161</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a240e63ad0129c83cb784f96031ad9f6d"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a240e63ad0129c83cb784f96031ad9f6d">impala::BaseSequenceScanner::BaseSequenceScanner</a></div><div class="ttdeci">BaseSequenceScanner(HdfsScanNode *, RuntimeState *)</div><div class="ttdef"><b>Definition:</b> <a href="base-sequence-scanner_8cc_source.html#l00063">base-sequence-scanner.cc:63</a></div></div> |
| <div class="ttc" id="classimpala_1_1BaseSequenceScanner_html_a519c086c9914fa0c9e2a2f608814a46b"><div class="ttname"><a href="classimpala_1_1BaseSequenceScanner.html#a519c086c9914fa0c9e2a2f608814a46b">impala::BaseSequenceScanner::AllocateFileHeader</a></div><div class="ttdeci">virtual FileHeader * AllocateFileHeader()=0</div></div> |
| </div><!-- fragment --></div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_e5d120be6b5e8a44336cbfd013b25604.html">be</a></li><li class="navelem"><a class="el" href="dir_68cf7cafb51a962d5bc4848b83cab0de.html">src</a></li><li class="navelem"><a class="el" href="dir_479336c0a15f2c0737bcafcf969f884c.html">exec</a></li><li class="navelem"><a class="el" href="base-sequence-scanner_8cc.html">base-sequence-scanner.cc</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:35 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
