| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: impala::HdfsScanner Class Reference</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li class="current"><a href="annotated.html"><span>Classes</span></a></li> |
| <li><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="annotated.html"><span>Class List</span></a></li> |
| <li><a href="classes.html"><span>Class Index</span></a></li> |
| <li><a href="inherits.html"><span>Class Hierarchy</span></a></li> |
| <li><a href="functions.html"><span>Class Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('classimpala_1_1HdfsScanner.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="summary"> |
| <a href="#pub-methods">Public Member Functions</a> | |
| <a href="#pub-static-attribs">Static Public Attributes</a> | |
| <a href="#pro-types">Protected Types</a> | |
| <a href="#pro-methods">Protected Member Functions</a> | |
| <a href="#pro-static-methods">Static Protected Member Functions</a> | |
| <a href="#pro-attribs">Protected Attributes</a> | |
| <a href="classimpala_1_1HdfsScanner-members.html">List of all members</a> </div> |
| <div class="headertitle"> |
| <div class="title">impala::HdfsScanner Class Reference<span class="mlabels"><span class="mlabel">abstract</span></span></div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| |
| <p><code>#include <<a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>></code></p> |
| <div class="dynheader"> |
| Inheritance diagram for impala::HdfsScanner:</div> |
| <div class="dyncontent"> |
| <div class="center"><div class="zoom"><iframe scrolling="no" frameborder="0" src="classimpala_1_1HdfsScanner__inherit__graph.svg" width="100%" height="600"><p><b>This browser is not able to show SVG: try Firefox, Chrome, Safari, or Opera instead.</b></p></iframe></div></div> |
| </div> |
| <div class="dynheader"> |
| Collaboration diagram for impala::HdfsScanner:</div> |
| <div class="dyncontent"> |
| <div class="center"><div class="zoom"><iframe scrolling="no" frameborder="0" src="classimpala_1_1HdfsScanner__coll__graph.svg" width="100%" height="600"><p><b>This browser is not able to show SVG: try Firefox, Chrome, Safari, or Opera instead.</b></p></iframe></div></div> |
| </div> |
| <table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-methods"></a> |
| Public Member Functions</h2></td></tr> |
| <tr class="memitem:ae876ce6b45a20b8f9664eda0cc17881d"><td class="memItemLeft" align="right" valign="top"> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae876ce6b45a20b8f9664eda0cc17881d">HdfsScanner</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *scan_node, <a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> *state)</td></tr> |
| <tr class="separator:ae876ce6b45a20b8f9664eda0cc17881d"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:acdb23592a90aeb791cd802928b8af317"><td class="memItemLeft" align="right" valign="top">virtual </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#acdb23592a90aeb791cd802928b8af317">~HdfsScanner</a> ()</td></tr> |
| <tr class="separator:acdb23592a90aeb791cd802928b8af317"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a99165297c6e6546edbee741a54dfd194"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a99165297c6e6546edbee741a54dfd194">Prepare</a> (<a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> *context)</td></tr> |
| <tr class="memdesc:a99165297c6e6546edbee741a54dfd194"><td class="mdescLeft"> </td><td class="mdescRight">One-time initialisation of state that is constant across scan ranges. <a href="#a99165297c6e6546edbee741a54dfd194">More...</a><br/></td></tr> |
| <tr class="separator:a99165297c6e6546edbee741a54dfd194"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a2f5d1478537643952a179471165bf1b5"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a2f5d1478537643952a179471165bf1b5">ProcessSplit</a> ()=0</td></tr> |
| <tr class="separator:a2f5d1478537643952a179471165bf1b5"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a61d4a073305f6befe9f12bad12f9de48"><td class="memItemLeft" align="right" valign="top">virtual void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a61d4a073305f6befe9f12bad12f9de48">Close</a> ()</td></tr> |
| <tr class="separator:a61d4a073305f6befe9f12bad12f9de48"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-static-attribs"></a> |
| Static Public Attributes</h2></td></tr> |
| <tr class="memitem:a0df2c19265535245917d3bb3b6095ba7"><td class="memItemLeft" align="right" valign="top">static const int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a0df2c19265535245917d3bb3b6095ba7">FILE_BLOCK_SIZE</a> = 4096</td></tr> |
| <tr class="separator:a0df2c19265535245917d3bb3b6095ba7"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa0fa3a63201936de45a12623cef65e74"><td class="memItemLeft" align="right" valign="top">static const char * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aa0fa3a63201936de45a12623cef65e74">LLVM_CLASS_NAME</a> = "class.impala::HdfsScanner"</td></tr> |
| <tr class="separator:aa0fa3a63201936de45a12623cef65e74"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-types"></a> |
| Protected Types</h2></td></tr> |
| <tr class="memitem:a1875e5409fedbadf979001ef761ae753"><td class="memItemLeft" align="right" valign="top">typedef int(* </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a1875e5409fedbadf979001ef761ae753">WriteTuplesFn</a> )(<a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> *, <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *, int, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *, int, int, int, int)</td></tr> |
| <tr class="separator:a1875e5409fedbadf979001ef761ae753"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-methods"></a> |
| Protected Member Functions</h2></td></tr> |
| <tr class="memitem:adab08ccc8e27d64a6473683eabaadb84"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#adab08ccc8e27d64a6473683eabaadb84">InitializeWriteTuplesFn</a> (<a class="el" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a> *partition, THdfsFileFormat::type type, const std::string &scanner_name)</td></tr> |
| <tr class="separator:adab08ccc8e27d64a6473683eabaadb84"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a370d072e6ebd37891c012fc9da453133"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a370d072e6ebd37891c012fc9da453133">StartNewRowBatch</a> ()</td></tr> |
| <tr class="memdesc:a370d072e6ebd37891c012fc9da453133"><td class="mdescLeft"> </td><td class="mdescRight">Set batch_ to a new row batch and update tuple_mem_ accordingly. <a href="#a370d072e6ebd37891c012fc9da453133">More...</a><br/></td></tr> |
| <tr class="separator:a370d072e6ebd37891c012fc9da453133"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a542176620ff5194866253070a539134a"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a542176620ff5194866253070a539134a">InitNewRange</a> ()=0</td></tr> |
| <tr class="memdesc:a542176620ff5194866253070a539134a"><td class="mdescLeft"> </td><td class="mdescRight">Reset internal state for a new scan range. <a href="#a542176620ff5194866253070a539134a">More...</a><br/></td></tr> |
| <tr class="separator:a542176620ff5194866253070a539134a"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ac2a391565eba0ab7cf58a7a024032220"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ac2a391565eba0ab7cf58a7a024032220">GetMemory</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> **<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> **tuple_mem, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> **tuple_row_mem)</td></tr> |
| <tr class="separator:ac2a391565eba0ab7cf58a7a024032220"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab72a9901f5c1c2cb0f053ce6ad45d9b6"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ab72a9901f5c1c2cb0f053ce6ad45d9b6">CommitRows</a> (int num_rows)</td></tr> |
| <tr class="separator:ab72a9901f5c1c2cb0f053ce6ad45d9b6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a36e661fbaff23879b78f53cdc1a56552"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a36e661fbaff23879b78f53cdc1a56552">AddFinalRowBatch</a> ()</td></tr> |
| <tr class="separator:a36e661fbaff23879b78f53cdc1a56552"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a94b437624601d6e7626e9b3d59cd87da"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a94b437624601d6e7626e9b3d59cd87da">AttachPool</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="classbool.html">bool</a> commit_batch)</td></tr> |
| <tr class="separator:a94b437624601d6e7626e9b3d59cd87da"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa329170edaabadb0f1ff94de121b3cec"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> <a class="el" href="impala-ir_8h.html#af2b653018219cda8c9a2d9efc0ce3f44">IR_ALWAYS_INLINE</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aa329170edaabadb0f1ff94de121b3cec">EvalConjuncts</a> (<a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *row)</td></tr> |
| <tr class="separator:aa329170edaabadb0f1ff94de121b3cec"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af5a25b22a0fb0b5bea62602fe224b021"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#af5a25b22a0fb0b5bea62602fe224b021">WriteEmptyTuples</a> (<a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a> *row_batch, int num_tuples)</td></tr> |
| <tr class="separator:af5a25b22a0fb0b5bea62602fe224b021"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae748aa64d908ff8a1fdcf3ae942ee51c"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae748aa64d908ff8a1fdcf3ae942ee51c">WriteEmptyTuples</a> (<a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> *context, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row, int num_tuples)</td></tr> |
| <tr class="memdesc:ae748aa64d908ff8a1fdcf3ae942ee51c"><td class="mdescLeft"> </td><td class="mdescRight">Write empty tuples and commit them to the context object. <a href="#ae748aa64d908ff8a1fdcf3ae942ee51c">More...</a><br/></td></tr> |
| <tr class="separator:ae748aa64d908ff8a1fdcf3ae942ee51c"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ac95a9d24d8da6d038c93b177c9c48e6b"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ac95a9d24d8da6d038c93b177c9c48e6b">WriteAlignedTuples</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row_mem, int row_size, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *fields, int num_tuples, int max_added_tuples, int slots_per_tuple, int row_start_indx)</td></tr> |
| <tr class="separator:ac95a9d24d8da6d038c93b177c9c48e6b"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab7c78e78c88f81f7ff9d87ed12eece4e"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ab7c78e78c88f81f7ff9d87ed12eece4e">UpdateDecompressor</a> (const THdfsCompression::type &compression)</td></tr> |
| <tr class="separator:ab7c78e78c88f81f7ff9d87ed12eece4e"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aace11af0dd18b9d9dbf7afbc74dab1ba"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aace11af0dd18b9d9dbf7afbc74dab1ba">UpdateDecompressor</a> (const std::string &codec)</td></tr> |
| <tr class="separator:aace11af0dd18b9d9dbf7afbc74dab1ba"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:afd378fe0f2500a3a63012c7b33be4c90"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#afd378fe0f2500a3a63012c7b33be4c90">ReportTupleParseError</a> (<a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *fields, uint8_t *errors, int row_idx)</td></tr> |
| <tr class="separator:afd378fe0f2500a3a63012c7b33be4c90"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:adfe2965c538acda676b3a3627ff614ee"><td class="memItemLeft" align="right" valign="top">virtual void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#adfe2965c538acda676b3a3627ff614ee">LogRowParseError</a> (int row_idx, std::stringstream *)</td></tr> |
| <tr class="separator:adfe2965c538acda676b3a3627ff614ee"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ad6bf05f906b32b58d7df914cbd115492"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ad6bf05f906b32b58d7df914cbd115492">WriteCompleteTuple</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *fields, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *tuple, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *template_tuple, uint8_t *error_fields, uint8_t *error_in_row)</td></tr> |
| <tr class="separator:ad6bf05f906b32b58d7df914cbd115492"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae05ee8b2f0b254228afcf6abf540f8f4"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae05ee8b2f0b254228afcf6abf540f8f4">ReportColumnParseError</a> (const <a class="el" href="classimpala_1_1SlotDescriptor.html">SlotDescriptor</a> *desc, const char *data, int len)</td></tr> |
| <tr class="separator:ae05ee8b2f0b254228afcf6abf540f8f4"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:accd7ea935f503dfe61c1570d1b9c07d8"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#accd7ea935f503dfe61c1570d1b9c07d8">InitTuple</a> (<a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *template_tuple, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *tuple)</td></tr> |
| <tr class="separator:accd7ea935f503dfe61c1570d1b9c07d8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a2ea24bc3f271386fe3e24336c7956ca7"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a2ea24bc3f271386fe3e24336c7956ca7">next_tuple</a> (<a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *t) const </td></tr> |
| <tr class="separator:a2ea24bc3f271386fe3e24336c7956ca7"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae2cf3a0764b67bb1a1733daaa772e045"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae2cf3a0764b67bb1a1733daaa772e045">next_row</a> (<a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *r) const </td></tr> |
| <tr class="separator:ae2cf3a0764b67bb1a1733daaa772e045"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a77ca8ddf41e61984b20255881759ef99"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a77ca8ddf41e61984b20255881759ef99">GetConjunctCtx</a> (int idx) const </td></tr> |
| <tr class="separator:a77ca8ddf41e61984b20255881759ef99"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-static-methods"></a> |
| Static Protected Member Functions</h2></td></tr> |
| <tr class="memitem:ab0d4c09c4bacaf57bf5bf238da3cd5e8"><td class="memItemLeft" align="right" valign="top">static llvm::Function * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ab0d4c09c4bacaf57bf5bf238da3cd5e8">CodegenWriteCompleteTuple</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *, <a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> *, const std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > &conjunct_ctxs)</td></tr> |
| <tr class="separator:ab0d4c09c4bacaf57bf5bf238da3cd5e8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aac18b5e53c864fa9ac9a574e00518a4a"><td class="memItemLeft" align="right" valign="top">static llvm::Function * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aac18b5e53c864fa9ac9a574e00518a4a">CodegenWriteAlignedTuples</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *, <a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> *, llvm::Function *write_tuple_fn)</td></tr> |
| <tr class="separator:aac18b5e53c864fa9ac9a574e00518a4a"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-attribs"></a> |
| Protected Attributes</h2></td></tr> |
| <tr class="memitem:a1b361d5cc69ef26b606349a26c1a8415"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a></td></tr> |
| <tr class="memdesc:a1b361d5cc69ef26b606349a26c1a8415"><td class="mdescLeft"> </td><td class="mdescRight">The scan node that started this scanner. <a href="#a1b361d5cc69ef26b606349a26c1a8415">More...</a><br/></td></tr> |
| <tr class="separator:a1b361d5cc69ef26b606349a26c1a8415"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8d1c8ea0d46e9378c6db2c24395a3f78"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a></td></tr> |
| <tr class="memdesc:a8d1c8ea0d46e9378c6db2c24395a3f78"><td class="mdescLeft"> </td><td class="mdescRight"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> for error reporting. <a href="#a8d1c8ea0d46e9378c6db2c24395a3f78">More...</a><br/></td></tr> |
| <tr class="separator:a8d1c8ea0d46e9378c6db2c24395a3f78"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a6c1f7899de6ac5baca6914640f4859f4"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a6c1f7899de6ac5baca6914640f4859f4">context_</a></td></tr> |
| <tr class="memdesc:a6c1f7899de6ac5baca6914640f4859f4"><td class="mdescLeft"> </td><td class="mdescRight">Context for this scanner. <a href="#a6c1f7899de6ac5baca6914640f4859f4">More...</a><br/></td></tr> |
| <tr class="separator:a6c1f7899de6ac5baca6914640f4859f4"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa539050780939fc85267c63e9f262e05"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1ScannerContext_1_1Stream.html">ScannerContext::Stream</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a></td></tr> |
| <tr class="memdesc:aa539050780939fc85267c63e9f262e05"><td class="mdescLeft"> </td><td class="mdescRight">The first stream for context_. <a href="#aa539050780939fc85267c63e9f262e05">More...</a><br/></td></tr> |
| <tr class="separator:aa539050780939fc85267c63e9f262e05"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:afb3a3d0ceb454e1ea3aa549786e45fac"><td class="memItemLeft" align="right" valign="top">std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#afb3a3d0ceb454e1ea3aa549786e45fac">conjunct_ctxs_</a></td></tr> |
| <tr class="separator:afb3a3d0ceb454e1ea3aa549786e45fac"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af1088c4f13acd1a6374c2693041f372e"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#af1088c4f13acd1a6374c2693041f372e">template_tuple_</a></td></tr> |
| <tr class="separator:af1088c4f13acd1a6374c2693041f372e"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a80ed99fe7814a4047c9220d31fc57536"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a80ed99fe7814a4047c9220d31fc57536">tuple_byte_size_</a></td></tr> |
| <tr class="memdesc:a80ed99fe7814a4047c9220d31fc57536"><td class="mdescLeft"> </td><td class="mdescRight">Fixed size of each tuple, in bytes. <a href="#a80ed99fe7814a4047c9220d31fc57536">More...</a><br/></td></tr> |
| <tr class="separator:a80ed99fe7814a4047c9220d31fc57536"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a0f92b70b3120fbf0b5ed833d26176c39"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a0f92b70b3120fbf0b5ed833d26176c39">tuple_</a></td></tr> |
| <tr class="memdesc:a0f92b70b3120fbf0b5ed833d26176c39"><td class="mdescLeft"> </td><td class="mdescRight">Current tuple pointer into tuple_mem_. <a href="#a0f92b70b3120fbf0b5ed833d26176c39">More...</a><br/></td></tr> |
| <tr class="separator:a0f92b70b3120fbf0b5ed833d26176c39"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a5c6954a13e4a285d2e4b544574c03b76"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a5c6954a13e4a285d2e4b544574c03b76">batch_</a></td></tr> |
| <tr class="separator:a5c6954a13e4a285d2e4b544574c03b76"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8b0b36b42ab140cbb793aa986683c432"><td class="memItemLeft" align="right" valign="top">uint8_t * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8b0b36b42ab140cbb793aa986683c432">tuple_mem_</a></td></tr> |
| <tr class="memdesc:a8b0b36b42ab140cbb793aa986683c432"><td class="mdescLeft"> </td><td class="mdescRight">The tuple memory of batch_. <a href="#a8b0b36b42ab140cbb793aa986683c432">More...</a><br/></td></tr> |
| <tr class="separator:a8b0b36b42ab140cbb793aa986683c432"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a06581f363b634869efc5d09839bc3e25"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a06581f363b634869efc5d09839bc3e25">num_errors_in_file_</a></td></tr> |
| <tr class="memdesc:a06581f363b634869efc5d09839bc3e25"><td class="mdescLeft"> </td><td class="mdescRight">number of errors in current file <a href="#a06581f363b634869efc5d09839bc3e25">More...</a><br/></td></tr> |
| <tr class="separator:a06581f363b634869efc5d09839bc3e25"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aebc25f28c691efb3b377e2cf7bc728a8"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1TextConverter.html">TextConverter</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aebc25f28c691efb3b377e2cf7bc728a8">text_converter_</a></td></tr> |
| <tr class="memdesc:aebc25f28c691efb3b377e2cf7bc728a8"><td class="mdescLeft"> </td><td class="mdescRight">Helper class for converting text to other types;. <a href="#aebc25f28c691efb3b377e2cf7bc728a8">More...</a><br/></td></tr> |
| <tr class="separator:aebc25f28c691efb3b377e2cf7bc728a8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a922115776b26544df5d86ad82dff87ef"><td class="memItemLeft" align="right" valign="top">int32_t </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a922115776b26544df5d86ad82dff87ef">num_null_bytes_</a></td></tr> |
| <tr class="memdesc:a922115776b26544df5d86ad82dff87ef"><td class="mdescLeft"> </td><td class="mdescRight">Number of null bytes in the tuple. <a href="#a922115776b26544df5d86ad82dff87ef">More...</a><br/></td></tr> |
| <tr class="separator:a922115776b26544df5d86ad82dff87ef"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8196c5dc7e842622df7ab43b8cd68397"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a></td></tr> |
| <tr class="separator:a8196c5dc7e842622df7ab43b8cd68397"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8d995c8326def52de037ecf074fc2654"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1Codec.html">Codec</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8d995c8326def52de037ecf074fc2654">decompressor_</a></td></tr> |
| <tr class="memdesc:a8d995c8326def52de037ecf074fc2654"><td class="mdescLeft"> </td><td class="mdescRight">Decompressor class to use, if any. <a href="#a8d995c8326def52de037ecf074fc2654">More...</a><br/></td></tr> |
| <tr class="separator:a8d995c8326def52de037ecf074fc2654"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aba0c7e8797db0a61444e50f342c775c6"><td class="memItemLeft" align="right" valign="top">THdfsCompression::type </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aba0c7e8797db0a61444e50f342c775c6">decompression_type_</a></td></tr> |
| <tr class="memdesc:aba0c7e8797db0a61444e50f342c775c6"><td class="mdescLeft"> </td><td class="mdescRight">The most recently used decompression type. <a href="#aba0c7e8797db0a61444e50f342c775c6">More...</a><br/></td></tr> |
| <tr class="separator:aba0c7e8797db0a61444e50f342c775c6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae86f3019c271de0d0593204003ee1653"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae86f3019c271de0d0593204003ee1653">data_buffer_pool_</a></td></tr> |
| <tr class="separator:ae86f3019c271de0d0593204003ee1653"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a7bd58a77c6acecf09cbfcc9b4e592618"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a7bd58a77c6acecf09cbfcc9b4e592618">decompress_timer_</a></td></tr> |
| <tr class="memdesc:a7bd58a77c6acecf09cbfcc9b4e592618"><td class="mdescLeft"> </td><td class="mdescRight">Time spent decompressing bytes. <a href="#a7bd58a77c6acecf09cbfcc9b4e592618">More...</a><br/></td></tr> |
| <tr class="separator:a7bd58a77c6acecf09cbfcc9b4e592618"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aea086d8b6a0604df26216cacea3feeed"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1HdfsScanner.html#a1875e5409fedbadf979001ef761ae753">WriteTuplesFn</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aea086d8b6a0604df26216cacea3feeed">write_tuples_fn_</a></td></tr> |
| <tr class="memdesc:aea086d8b6a0604df26216cacea3feeed"><td class="mdescLeft"> </td><td class="mdescRight">Jitted write tuples function pointer. Null if codegen is disabled. <a href="#aea086d8b6a0604df26216cacea3feeed">More...</a><br/></td></tr> |
| <tr class="separator:aea086d8b6a0604df26216cacea3feeed"><td class="memSeparator" colspan="2"> </td></tr> |
| </table> |
| <a name="details" id="details"></a><h2 class="groupheader">Detailed Description</h2> |
| <div class="textblock"><p><a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> is the superclass for different hdfs file format parsers. There is an instance of the scanner object created for each split, each driven by a different thread created by the scan node. The scan node calls:</p> |
| <ol type="1"> |
| <li>Prepare</li> |
| <li>ProcessSplit</li> |
| <li>Close ProcessSplit does not return until the split is complete (or an error) occurred. The <a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> works in tandem with the <a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> to interleave IO and parsing. If a split is compressed, then a decompressor will be created, either during <a class="el" href="classimpala_1_1HdfsScanner.html#a99165297c6e6546edbee741a54dfd194" title="One-time initialisation of state that is constant across scan ranges. ">Prepare()</a> or at the beginning of <a class="el" href="classimpala_1_1HdfsScanner.html#a2f5d1478537643952a179471165bf1b5">ProcessSplit()</a>, and used for decompressing and reading the split. For codegen, the implementation is split into two parts.</li> |
| </ol> |
| <ol type="1"> |
| <li>During the <a class="el" href="classimpala_1_1HdfsScanner.html#a99165297c6e6546edbee741a54dfd194" title="One-time initialisation of state that is constant across scan ranges. ">Prepare()</a> phase of the <a class="el" href="classimpala_1_1ScanNode.html" title="Abstract base class of all scan nodes; introduces SetScanRange(). ">ScanNode</a>, the scanner subclass's static Codegen() function will be called to perform codegen for that scanner type for the specific tuple desc. This codegen'd function is cached in the <a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a>.</li> |
| <li>During the <a class="el" href="namespaceimpala.html#af279911558f793413306452d5b16e6d6">GetNext()</a> phase (where we create one Scanner for each scan range), the created scanner subclass can retrieve, from the scan node, the codegen'd function to use. This way, we only codegen once per scanner type, rather than once per scanner object. This class also encapsulates row batch management. Subclasses should call <a class="el" href="classimpala_1_1HdfsScanner.html#ab72a9901f5c1c2cb0f053ce6ad45d9b6">CommitRows()</a> after writing to the current row batch, which handles creating row batches, attaching resources (IO buffers and mem pools) to the current row batch, and passing row batches up to the scan node. Subclasses can also use <a class="el" href="classimpala_1_1HdfsScanner.html#ac2a391565eba0ab7cf58a7a024032220">GetMemory()</a> to help with per-row memory management. </li> |
| </ol> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00091">91</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| </div><h2 class="groupheader">Member Typedef Documentation</h2> |
| <a class="anchor" id="a1875e5409fedbadf979001ef761ae753"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">typedef int(* impala::HdfsScanner::WriteTuplesFn)(<a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> *, <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *, int, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *, int, int, int, int)</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Matching typedef for WriteAlignedTuples for codegen. Refer to comments for that function. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00212">212</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Constructor & Destructor Documentation</h2> |
| <a class="anchor" id="ae876ce6b45a20b8f9664eda0cc17881d"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">HdfsScanner::HdfsScanner </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname"><em>scan_node</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> * </td> |
| <td class="paramname"><em>state</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00053">53</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="acdb23592a90aeb791cd802928b8af317"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">HdfsScanner::~HdfsScanner </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00067">67</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Member Function Documentation</h2> |
| <a class="anchor" id="a36e661fbaff23879b78f53cdc1a56552"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::AddFinalRowBatch </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Attach all remaining resources from context_ to batch_ and send batch_ to the scan node. This must be called after all rows have been committed and no further resources are needed from context_ (in practice this will happen in each scanner subclass's <a class="el" href="classimpala_1_1HdfsScanner.html#a61d4a073305f6befe9f12bad12f9de48">Close()</a> implementation). </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00145">145</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scan-node_8cc_source.html#l00688">impala::HdfsScanNode::AddMaterializedRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">context_</a>, <a class="el" href="scanner-context_8cc_source.html#l00045">impala::ScannerContext::ReleaseCompletedResources()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, and <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a94b437624601d6e7626e9b3d59cd87da"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void impala::HdfsScanner::AttachPool </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> </td> |
| <td class="paramname"><em>commit_batch</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Release all memory in 'pool' to batch_. If commit_batch is true, the row batch will be committed. commit_batch should be true if the attached pool is expected to be non-trivial (i.e. a decompression buffer) to minimize scanner mem usage. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00256">256</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="mem-pool_8cc_source.html#l00161">impala::MemPool::AcquireData()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, and <a class="el" href="row-batch_8h_source.html#l00148">impala::RowBatch::tuple_data_pool()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00532">impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00231">impala::HdfsRCFileScanner::ResetRowGroup()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a61d4a073305f6befe9f12bad12f9de48"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::Close </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Release all resources the scanner has allocated. This is the last chance for the scanner to attach any resources to the <a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> object. </p> |
| |
| <p>Reimplemented in <a class="el" href="classimpala_1_1HdfsParquetScanner.html#af159ad563155f231e9d760b218538355">impala::HdfsParquetScanner</a>, <a class="el" href="classimpala_1_1BaseSequenceScanner.html#a1a37621b600b55d162832e8744fc6a17">impala::BaseSequenceScanner</a>, and <a class="el" href="classimpala_1_1HdfsTextScanner.html#a10b654ff3364a09cde00aec5dc64a21a">impala::HdfsTextScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00082">82</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="classimpala_1_1Expr.html#a70300b0ba7c91fddb9adfaeb0dfc09a5">impala::Expr::Close()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00154">conjunct_ctxs_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">decompressor_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00144">state_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>, and <a class="el" href="hdfs-scan-node_8cc_source.html#l00797">impala::HdfsScanNode::ScannerThread()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aac18b5e53c864fa9ac9a574e00518a4a"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Function * HdfsScanner::CodegenWriteAlignedTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">llvm::Function * </td> |
| <td class="paramname"><em>write_tuple_fn</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Codegen function to replace WriteAlignedTuples. WriteAlignedTuples is cross compiled to IR. This function loads the precompiled IR function, modifies it and returns the resulting function. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00495">495</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="llvm-codegen_8h_source.html#l00135">impala::LlvmCodeGen::codegen_timer()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00596">impala::LlvmCodeGen::FinalizeFunction()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00421">impala::LlvmCodeGen::GetFunction()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00489">impala::LlvmCodeGen::ReplaceCallSites()</a>, and <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00609">impala::HdfsTextScanner::Codegen()</a>, and <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00050">impala::HdfsSequenceScanner::Codegen()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab0d4c09c4bacaf57bf5bf238da3cd5e8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Function * HdfsScanner::CodegenWriteCompleteTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > & </td> |
| <td class="paramname"><em>conjunct_ctxs</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Codegen function to replace WriteCompleteTuple. Should behave identically to WriteCompleteTuple. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00296">296</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="llvm-codegen_8h_source.html#l00171">impala::LlvmCodeGen::FnPrototype::AddArgument()</a>, <a class="el" href="descriptors_8h_source.html#l00300">impala::TupleDescriptor::byte_size()</a>, <a class="el" href="llvm-codegen_8h_source.html#l00135">impala::LlvmCodeGen::codegen_timer()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00933">impala::LlvmCodeGen::CodegenMemcpy()</a>, <a class="el" href="text-converter_8cc_source.html#l00099">impala::TextConverter::CodegenWriteSlot()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00956">impala::HdfsScanNode::ComputeSlotMaterializationOrder()</a>, <a class="el" href="llvm-codegen_8h_source.html#l00214">impala::LlvmCodeGen::context()</a>, <a class="el" href="codegen-anyval_8cc_source.html#l00148">impala::CodegenAnyVal::CreateCallWrapped()</a>, <a class="el" href="llvm-codegen_8h_source.html#l00381">impala::LlvmCodeGen::false_value()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00596">impala::LlvmCodeGen::FinalizeFunction()</a>, <a class="el" href="descriptors_8cc_source.html#l00556">impala::TupleDescriptor::GenerateLlvmStruct()</a>, <a class="el" href="status_8cc_source.html#l00184">impala::Status::GetDetail()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00421">impala::LlvmCodeGen::GetFunction()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00371">impala::LlvmCodeGen::GetIntConstant()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00312">impala::LlvmCodeGen::GetType()</a>, <a class="el" href="codegen-anyval_8cc_source.html#l00258">impala::CodegenAnyVal::GetVal()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00134">impala::HdfsScanNode::hdfs_table()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00061">impala::FieldLocation::LLVM_CLASS_NAME</a>, <a class="el" href="tuple-row_8h_source.html#l00076">impala::TupleRow::LLVM_CLASS_NAME</a>, <a class="el" href="tuple_8h_source.html#l00134">impala::Tuple::LLVM_CLASS_NAME</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00137">LLVM_CLASS_NAME</a>, <a class="el" href="mem-pool_8h_source.html#l00177">impala::MemPool::LLVM_CLASS_NAME</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="descriptors_8h_source.html#l00233">impala::HdfsTableDescriptor::null_column_value()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00130">impala::HdfsScanNode::num_materialized_partition_keys()</a>, <a class="el" href="descriptors_8h_source.html#l00301">impala::TupleDescriptor::num_null_bytes()</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00583">impala::LlvmCodeGen::OptimizeFunctionWithExprs()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00136">impala::HdfsScanNode::runtime_state()</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, <a class="el" href="llvm-codegen_8h_source.html#l00380">impala::LlvmCodeGen::true_value()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>, <a class="el" href="types_8h_source.html#l00060">impala::ColumnType::type</a>, <a class="el" href="descriptors_8h_source.html#l00078">impala::SlotDescriptor::type()</a>, <a class="el" href="types_8h_source.html#l00030">impala::TYPE_BOOLEAN</a>, <a class="el" href="types_8h_source.html#l00042">impala::TYPE_DECIMAL</a>, <a class="el" href="types_8h_source.html#l00033">impala::TYPE_INT</a>, <a class="el" href="types_8h_source.html#l00037">impala::TYPE_TIMESTAMP</a>, and <a class="el" href="types_8h_source.html#l00031">impala::TYPE_TINYINT</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00609">impala::HdfsTextScanner::Codegen()</a>, and <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00050">impala::HdfsSequenceScanner::Codegen()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab72a9901f5c1c2cb0f053ce6ad45d9b6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::CommitRows </td> |
| <td>(</td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_rows</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Commit num_rows to the current row batch. If this completes, the row batch is enqueued with the scan node and <a class="el" href="classimpala_1_1HdfsScanner.html#a370d072e6ebd37891c012fc9da453133" title="Set batch_ to a new row batch and update tuple_mem_ accordingly. ">StartNewRowBatch()</a> is called. Returns <a class="el" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a> if the query is not cancelled and hasn't exceeded any mem limits. Scanner can call this with 0 rows to flush any pending resources (attached pools and io buffers) to minimize memory consumption. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00124">124</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scan-node_8cc_source.html#l00688">impala::HdfsScanNode::AddMaterializedRowBatch()</a>, <a class="el" href="row-batch_8h_source.html#l00120">impala::RowBatch::AtCapacity()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>, <a class="el" href="descriptors_8h_source.html#l00300">impala::TupleDescriptor::byte_size()</a>, <a class="el" href="status_8h_source.html#l00088">impala::Status::CANCELLED</a>, <a class="el" href="scanner-context_8cc_source.html#l00282">impala::ScannerContext::cancelled()</a>, <a class="el" href="row-batch_8h_source.html#l00216">impala::RowBatch::capacity()</a>, <a class="el" href="runtime-state_8cc_source.html#l00286">impala::RuntimeState::CheckQueryState()</a>, <a class="el" href="row-batch_8h_source.html#l00102">impala::RowBatch::CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00154">conjunct_ctxs_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">context_</a>, <a class="el" href="expr-context_8cc_source.html#l00109">impala::ExprContext::FreeLocalAllocations()</a>, <a class="el" href="scanner-context_8h_source.html#l00277">impala::ScannerContext::num_completed_io_buffers()</a>, <a class="el" href="row-batch_8h_source.html#l00215">impala::RowBatch::num_rows()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="scanner-context_8cc_source.html#l00045">impala::ScannerContext::ReleaseCompletedResources()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">state_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00180">tuple_mem_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00256">AttachPool()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, and <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa329170edaabadb0f1ff94de121b3cec"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> <a class="el" href="impala-ir_8h.html#af2b653018219cda8c9a2d9efc0ce3f44">IR_ALWAYS_INLINE</a> impala::HdfsScanner::EvalConjuncts </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>row</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Convenience function for evaluating conjuncts using this scanner's ExprContexts. This must always be inlined so we can correctly replace the call to <a class="el" href="classimpala_1_1ExecNode.html#a48ec693b9eb748682b58479751812f08">ExecNode::EvalConjuncts()</a> during codegen. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00266">266</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00154">conjunct_ctxs_</a>, and <a class="el" href="exec-node_8cc_source.html#l00393">impala::ExecNode::EvalConjuncts()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">WriteCompleteTuple()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a77ca8ddf41e61984b20255881759ef99"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * HdfsScanner::GetConjunctCtx </td> |
| <td>(</td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>idx</em></td><td>)</td> |
| <td> const</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Simple wrapper around conjunct_ctxs_. Used in the codegen'd version of <a class="el" href="classimpala_1_1HdfsScanner.html#ad6bf05f906b32b58d7df914cbd115492">WriteCompleteTuple()</a> because it's easier than writing IR to access conjunct_ctxs_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00079">79</a> of file <a class="el" href="hdfs-scanner-ir_8cc_source.html">hdfs-scanner-ir.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00154">conjunct_ctxs_</a>, and <a class="el" href="gen__ir__descriptions_8py_source.html#l00215">gen_ir_descriptions::idx</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ac2a391565eba0ab7cf58a7a024032220"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::GetMemory </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> ** </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> ** </td> |
| <td class="paramname"><em>tuple_mem</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> ** </td> |
| <td class="paramname"><em>tuple_row_mem</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Gets memory for outputting tuples into batch_. *pool is the mem pool that should be used for memory allocated for those tuples. *tuple_mem should be the location to output tuples, and *tuple_row_mem for outputting tuple rows. Returns the maximum number of tuples/tuple rows that can be output (before the current row batch is complete and a new one is allocated). Memory returned from this call is invalidated after calling CommitRows. Callers must call GetMemory again after calling this function. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00115">115</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="row-batch_8h_source.html#l00100">impala::RowBatch::AddRow()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>, <a class="el" href="row-batch_8h_source.html#l00216">impala::RowBatch::capacity()</a>, <a class="el" href="row-batch_8h_source.html#l00140">impala::RowBatch::GetRow()</a>, <a class="el" href="row-batch_8h_source.html#l00215">impala::RowBatch::num_rows()</a>, <a class="el" href="row-batch_8h_source.html#l00148">impala::RowBatch::tuple_data_pool()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00180">tuple_mem_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="adab08ccc8e27d64a6473683eabaadb84"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::InitializeWriteTuplesFn </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a> * </td> |
| <td class="paramname"><em>partition</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">THdfsFileFormat::type </td> |
| <td class="paramname"><em>type</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const std::string & </td> |
| <td class="paramname"><em>scanner_name</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Initializes write_tuples_fn_ to the jitted function if codegen is possible.</p> |
| <ul> |
| <li>partition - partition descriptor for this scanner/scan range</li> |
| <li>type - type for this scanner</li> |
| <li>scanner_name - debug string name for this scanner (e.g. <a class="el" href="classimpala_1_1HdfsTextScanner.html">HdfsTextScanner</a>) </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00087">87</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="descriptors_8h_source.html#l00183">impala::HdfsPartitionDescriptor::escape_char()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00224">impala::HdfsScanNode::GetCodegenFn()</a>, <a class="el" href="exec-node_8h_source.html#l00154">impala::ExecNode::id()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00172">impala::HdfsScanNode::IncNumScannersCodegenDisabled()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00168">impala::HdfsScanNode::IncNumScannersCodegenEnabled()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="descriptors_8h_source.html#l00303">impala::TupleDescriptor::string_slots()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00215">write_tuples_fn_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">impala::HdfsTextScanner::ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a542176620ff5194866253070a539134a"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> impala::HdfsScanner::InitNewRange </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">pure virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Reset internal state for a new scan range. </p> |
| |
| <p>Implemented in <a class="el" href="classimpala_1_1HdfsRCFileScanner.html#a4260dd8dc9d776f9d7acbf94fe184af9">impala::HdfsRCFileScanner</a>, <a class="el" href="classimpala_1_1HdfsParquetScanner.html#af7f55fa3572a786681ef54b34a8e644c">impala::HdfsParquetScanner</a>, <a class="el" href="classimpala_1_1HdfsSequenceScanner.html#a6929ecd5b2aa8d140dde64dad3381a9a">impala::HdfsSequenceScanner</a>, <a class="el" href="classimpala_1_1HdfsAvroScanner.html#a38922c01fb625280a843da3cb934c137">impala::HdfsAvroScanner</a>, and <a class="el" href="classimpala_1_1HdfsTextScanner.html#a4cebfa743efe38c408fc73b53422e851">impala::HdfsTextScanner</a>.</p> |
| |
| <p>Referenced by <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="accd7ea935f503dfe61c1570d1b9c07d8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void impala::HdfsScanner::InitTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>template_tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>tuple</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Initialize a tuple. TODO: only copy over non-null slots. TODO: InitTuple is called frequently, avoid the if, perhaps via templatization. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00355">355</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00189">num_null_bytes_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00167">tuple_byte_size_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">WriteCompleteTuple()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="adfe2965c538acda676b3a3627ff614ee"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::LogRowParseError </td> |
| <td>(</td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_idx</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">std::stringstream * </td> |
| <td class="paramname"> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Utility function to append an error message for an invalid row. This is called from <a class="el" href="classimpala_1_1HdfsScanner.html#afd378fe0f2500a3a63012c7b33be4c90">ReportTupleParseError()</a> row_idx is the index of the row in the current batch. Subclasses should override this function (i.e. text needs to join boundary rows). Since this is only in the error path, vtable overhead is acceptable. </p> |
| |
| <p>Reimplemented in <a class="el" href="classimpala_1_1HdfsSequenceScanner.html#ad07951d5279f1c10f2c5c1e8157ce953">impala::HdfsSequenceScanner</a>, and <a class="el" href="classimpala_1_1HdfsTextScanner.html#aa54a763cd29781bd631158d72bd269ad">impala::HdfsTextScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00572">572</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae2cf3a0764b67bb1a1733daaa772e045"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a>* impala::HdfsScanner::next_row </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>r</em></td><td>)</td> |
| <td> const</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00368">368</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>, and <a class="el" href="row-batch_8h_source.html#l00147">impala::RowBatch::row_byte_size()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00195">WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a2ea24bc3f271386fe3e24336c7956ca7"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsScanner::next_tuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>t</em></td><td>)</td> |
| <td> const</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00363">363</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00167">tuple_byte_size_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a99165297c6e6546edbee741a54dfd194"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::Prepare </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> * </td> |
| <td class="paramname"><em>context</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>One-time initialisation of state that is constant across scan ranges. </p> |
| |
| <p>Reimplemented in <a class="el" href="classimpala_1_1HdfsRCFileScanner.html#ac619ad303abae210fffd496b9665e4e0">impala::HdfsRCFileScanner</a>, <a class="el" href="classimpala_1_1HdfsSequenceScanner.html#aa681e1868e2d7ea10a72c5230343af26">impala::HdfsSequenceScanner</a>, <a class="el" href="classimpala_1_1HdfsParquetScanner.html#ac2fd6d1aa6cb95dcd874460f03e5d379">impala::HdfsParquetScanner</a>, <a class="el" href="classimpala_1_1BaseSequenceScanner.html#acc3c090b8b5e1c282c828be221697d5b">impala::BaseSequenceScanner</a>, and <a class="el" href="classimpala_1_1HdfsTextScanner.html#a370ef8b83aa2f583182c0939d9ab1e10">impala::HdfsTextScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00071">71</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-profile_8h_source.html#l00050">ADD_TIMER</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00154">conjunct_ctxs_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">context_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00208">decompress_timer_</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00692">impala::HdfsScanNode::GetConjunctCtxs()</a>, <a class="el" href="scanner-context_8h_source.html#l00246">impala::ScannerContext::GetStream()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00270">impala::HdfsScanNode::InitTemplateTuple()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="scanner-context_8h_source.html#l00278">impala::ScannerContext::partition_descriptor()</a>, <a class="el" href="descriptors_8h_source.html#l00185">impala::HdfsPartitionDescriptor::partition_key_value_ctxs()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="exec-node_8h_source.html#l00161">impala::ExecNode::runtime_profile()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">state_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">stream_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00164">template_tuple_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scan-node_8cc_source.html#l00230">impala::HdfsScanNode::CreateAndPrepareScanner()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">impala::HdfsTextScanner::Prepare()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00074">impala::BaseSequenceScanner::Prepare()</a>, and <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00440">impala::HdfsParquetScanner::Prepare()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a2f5d1478537643952a179471165bf1b5"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> impala::HdfsScanner::ProcessSplit </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">pure virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Process an entire split, reading bytes from the context's streams. Context is initialized with the split data (e.g. template tuple, partition descriptor, etc). This function should only return on error or end of scan range. </p> |
| |
| <p>Implemented in <a class="el" href="classimpala_1_1HdfsParquetScanner.html#a718fbd685b69286bf3ece22757cf0556">impala::HdfsParquetScanner</a>, <a class="el" href="classimpala_1_1BaseSequenceScanner.html#acc59a2d650453eb5c12670519342c846">impala::BaseSequenceScanner</a>, and <a class="el" href="classimpala_1_1HdfsTextScanner.html#ad22f51e3ac67fce825aab6480477d839">impala::HdfsTextScanner</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scan-node_8cc_source.html#l00797">impala::HdfsScanNode::ScannerThread()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae05ee8b2f0b254228afcf6abf540f8f4"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::ReportColumnParseError </td> |
| <td>(</td> |
| <td class="paramtype">const <a class="el" href="classimpala_1_1SlotDescriptor.html">SlotDescriptor</a> * </td> |
| <td class="paramname"><em>desc</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const char * </td> |
| <td class="paramname"><em>data</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>len</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Report parse error for column @ desc. If abort_on_error is true, sets parse_status_ to the error message. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00577">577</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="descriptors_8h_source.html#l00084">impala::SlotDescriptor::col_pos()</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00127">impala::HdfsScanNode::num_partition_keys()</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">parse_status_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">state_</a>, and <a class="el" href="descriptors_8h_source.html#l00078">impala::SlotDescriptor::type()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">impala::HdfsTextScanner::WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="afd378fe0f2500a3a63012c7b33be4c90"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> HdfsScanner::ReportTupleParseError </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">uint8_t * </td> |
| <td class="paramname"><em>errors</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_idx</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Utility function to report parse errors for each field. If errors[i] is nonzero, fields[i] had a parse error. row_idx is the idx of the row in the current batch that had the parse error Returns false if parsing should be aborted. In this case parse_status_ is set to the error. This is called from WriteAlignedTuples. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00546">546</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="scanner-context_8h_source.html#l00118">impala::ScannerContext::Stream::filename()</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00572">LogRowParseError()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00183">num_errors_in_file_</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">parse_status_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">ReportColumnParseError()</a>, <a class="el" href="runtime-state_8cc_source.html#l00219">impala::RuntimeState::ReportFileErrors()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">state_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00150">stream_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">WriteAlignedTuples()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a370d072e6ebd37891c012fc9da453133"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::StartNewRowBatch </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Set batch_ to a new row batch and update tuple_mem_ accordingly. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00108">108</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="mem-pool_8h_source.html#l00092">impala::MemPool::Allocate()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">batch_</a>, <a class="el" href="runtime-state_8h_source.html#l00098">impala::RuntimeState::batch_size()</a>, <a class="el" href="exec-node_8h_source.html#l00162">impala::ExecNode::mem_tracker()</a>, <a class="el" href="exec-node_8h_source.html#l00156">impala::ExecNode::row_desc()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">state_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00167">tuple_byte_size_</a>, <a class="el" href="row-batch_8h_source.html#l00148">impala::RowBatch::tuple_data_pool()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00180">tuple_mem_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab7c78e78c88f81f7ff9d87ed12eece4e"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::UpdateDecompressor </td> |
| <td>(</td> |
| <td class="paramtype">const THdfsCompression::type & </td> |
| <td class="paramname"><em>compression</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Update the decompressor_ object given a compression type or codec name. Depending on the old compression type and the new one, it may close the old decompressor and/or create a new one of different type. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00513">513</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="classimpala_1_1Codec.html#a0d8e74ac086a91c1cc7dbd7fe166f307">impala::Codec::CreateDecompressor()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00205">data_buffer_pool_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00201">decompression_type_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">decompressor_</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="descriptors_8h_source.html#l00303">impala::TupleDescriptor::string_slots()</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00493">impala::HdfsAvroScanner::InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">impala::HdfsTextScanner::ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aace11af0dd18b9d9dbf7afbc74dab1ba"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> impala::HdfsScanner::UpdateDecompressor </td> |
| <td>(</td> |
| <td class="paramtype">const std::string & </td> |
| <td class="paramname"><em>codec</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ac95a9d24d8da6d038c93b177c9c48e6b"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::WriteAlignedTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row_mem</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_size</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>max_added_tuples</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>slots_per_tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_start_indx</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Processes batches of fields and writes them out to tuple_row_mem.</p> |
| <ul> |
| <li>'pool' mempool to allocate from for auxiliary tuple memory</li> |
| <li>'tuple_row_mem' preallocated tuple_row memory this function must use.</li> |
| <li>'fields' must start at the beginning of a tuple.</li> |
| <li>'num_tuples' number of tuples to process</li> |
| <li>'max_added_tuples' the maximum number of tuples that should be added to the batch.</li> |
| <li>'row_start_index' is the number of rows that have already been processed as part of WritePartialTuple. Returns the number of tuples added to the row batch. This can be less than num_tuples/tuples_till_limit because of failed conjuncts. Returns -1 if parsing should be aborted due to parse errors. </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">33</a> of file <a class="el" href="hdfs-scanner-ir_8cc_source.html">hdfs-scanner-ir.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">template_tuple_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00170">tuple_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00167">tuple_byte_size_</a>, <a class="el" href="compiler-util_8h_source.html#l00033">UNLIKELY</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00217">WriteCompleteTuple()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ad6bf05f906b32b58d7df914cbd115492"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> HdfsScanner::WriteCompleteTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>template_tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">uint8_t * </td> |
| <td class="paramname"><em>error_fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">uint8_t * </td> |
| <td class="paramname"><em>error_in_row</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Writes out all slots for 'tuple' from 'fields'. 'fields' must be aligned to the start of the tuple (e.g. fields[0] maps to slots[0]). After writing the tuple, it will be evaluated against the conjuncts.</p> |
| <ul> |
| <li>error_fields is an out array. error_fields[i] will be set to true if the ith field had a parse error</li> |
| <li>error_in_row is an out bool. It is set to true if any field had parse errors Returns whether the resulting tuplerow passed the conjuncts. The parsing of the fields and evaluating against conjuncts is combined in this function. This is done so it can be possible to evaluate conjuncts as slots are materialized (on partial tuples). This function is replaced by a codegen'd function at runtime. This is the reason that the out error parameters are typed uint8_t instead of bool. We need to be able to match this function's signature identically for the codegen'd function. Bool's as out parameters can get converted to bytes by the compiler and rather than implicitly depending on that to happen, we will explicitly type them to bytes. TODO: revisit this </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00217">217</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00266">EvalConjuncts()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00355">InitTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00059">impala::FieldLocation::len</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00186">text_converter_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>, and <a class="el" href="compiler-util_8h_source.html#l00033">UNLIKELY</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">WriteAlignedTuples()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af5a25b22a0fb0b5bea62602fe224b021"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::WriteEmptyTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a> * </td> |
| <td class="paramname"><em>row_batch</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Utility method to write out tuples when there are no materialized fields (e.g. select count(*) or only partition keys). num_tuples - Total number of tuples to write out. Returns the number of tuples added to the row batch. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00157">157</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="row-batch_8h_source.html#l00100">impala::RowBatch::AddRow()</a>, <a class="el" href="row-batch_8h_source.html#l00094">impala::RowBatch::AddRows()</a>, <a class="el" href="row-batch_8h_source.html#l00120">impala::RowBatch::AtCapacity()</a>, <a class="el" href="row-batch_8h_source.html#l00216">impala::RowBatch::capacity()</a>, <a class="el" href="row-batch_8h_source.html#l00109">impala::RowBatch::CommitLastRow()</a>, <a class="el" href="row-batch_8h_source.html#l00102">impala::RowBatch::CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00266">EvalConjuncts()</a>, <a class="el" href="row-batch_8h_source.html#l00140">impala::RowBatch::GetRow()</a>, <a class="el" href="row-batch_8h_source.html#l00087">impala::RowBatch::INVALID_ROW_INDEX</a>, <a class="el" href="row-batch_8h_source.html#l00215">impala::RowBatch::num_rows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">template_tuple_</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae748aa64d908ff8a1fdcf3ae942ee51c"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::WriteEmptyTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> * </td> |
| <td class="paramname"><em>context</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Write empty tuples and commit them to the context object. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00195">195</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00266">EvalConjuncts()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00368">next_row()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">scan_node_</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">template_tuple_</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Member Data Documentation</h2> |
| <a class="anchor" id="a5c6954a13e4a285d2e4b544574c03b76"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a>* impala::HdfsScanner::batch_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>The current row batch being populated. Creating new row batches, attaching context resources, and handing off to the scan node is handled by this class in <a class="el" href="classimpala_1_1HdfsScanner.html#ab72a9901f5c1c2cb0f053ce6ad45d9b6">CommitRows()</a>, but <a class="el" href="classimpala_1_1HdfsScanner.html#a94b437624601d6e7626e9b3d59cd87da">AttachPool()</a> must be called by scanner subclasses to attach any memory allocated by that subclass. All row batches created by this class are transferred to the scan node (i.e., all batches are ultimately owned by the scan node). </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00177">177</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00145">AddFinalRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00256">AttachPool()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00115">GetMemory()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00368">next_row()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00067">~HdfsScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="afb3a3d0ceb454e1ea3aa549786e45fac"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">std::vector<<a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a>*> impala::HdfsScanner::conjunct_ctxs_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p><a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> for each conjunct. Each scanner has its own ExprContexts so the conjuncts can be safely evaluated in parallel. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00154">154</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00082">Close()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00266">EvalConjuncts()</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00079">GetConjunctCtx()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a6c1f7899de6ac5baca6914640f4859f4"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a>* impala::HdfsScanner::context_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Context for this scanner. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00147">147</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00145">AddFinalRowBatch()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">impala::HdfsTextScanner::FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01093">impala::HdfsParquetScanner::InitColumns()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">impala::HdfsTextScanner::InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">impala::HdfsTextScanner::ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae86f3019c271de0d0593204003ee1653"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1MemPool.html">MemPool</a>> impala::HdfsScanner::data_buffer_pool_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Pool to allocate per data block memory. This should be used with the decompressor and any other per data block allocations. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00205">205</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00254">impala::HdfsRCFileScanner::ReadRowGroup()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00231">impala::HdfsRCFileScanner::ResetRowGroup()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00513">UpdateDecompressor()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">impala::HdfsTextScanner::WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a7bd58a77c6acecf09cbfcc9b4e592618"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a>* impala::HdfsScanner::decompress_timer_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Time spent decompressing bytes. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00208">208</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">impala::HdfsTextScanner::FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00532">impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aba0c7e8797db0a61444e50f342c775c6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">THdfsCompression::type impala::HdfsScanner::decompression_type_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The most recently used decompression type. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00201">201</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">impala::HdfsTextScanner::FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">impala::HdfsTextScanner::FillByteBufferCompressedFile()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00513">UpdateDecompressor()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8d995c8326def52de037ecf074fc2654"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1Codec.html">Codec</a>> impala::HdfsScanner::decompressor_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Decompressor class to use, if any. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00198">198</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00082">Close()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">impala::HdfsTextScanner::FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">impala::HdfsTextScanner::FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00068">impala::HdfsRCFileScanner::InitNewRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00513">UpdateDecompressor()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a0df2c19265535245917d3bb3b6095ba7"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int impala::HdfsScanner::FILE_BLOCK_SIZE = 4096</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Assumed size of an OS file block. Used mostly when reading file format headers, etc. This probably ought to be a derived number from the environment. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00095">95</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa0fa3a63201936de45a12623cef65e74"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const char * HdfsScanner::LLVM_CLASS_NAME = "class.impala::HdfsScanner"</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Scanner subclasses must implement these static functions as well. Unfortunately, c++ does not allow static virtual functions. Issue the initial ranges for 'files'. <a class="el" href="structimpala_1_1HdfsFileDesc.html">HdfsFileDesc</a> groups all the splits assigned to this scan node by file. This is called before any of the scanner subclasses are created to process splits in 'files'. The strategy on how to parse the scan ranges depends on the file format.</p> |
| <ul> |
| <li>For simple text files, all the splits are simply issued to the io mgr and one split == one scan range.</li> |
| <li>For formats with a header, the metadata is first parsed, and then the ranges are issued to the io mgr. There is one scan range for the header and one range for each split.</li> |
| <li>For columnar formats, the header is parsed and only the relevant byte ranges should be issued to the io mgr. This is one range for the metadata and one range for each column, for each split. This function is how scanners can pick their strategy. void IssueInitialRanges(HdfsScanNode* scan_node, const std::vector<HdfsFileDesc*>& files); Codegen all functions for this scanner. The codegen'd function is specific to the scanner subclass but not specific to each scanner object. We don't want to codegen the functions for each scanner object. llvm::Function* Codegen(HdfsScanNode* scan_node); </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00137">137</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00296">CodegenWriteCompleteTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a06581f363b634869efc5d09839bc3e25"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int impala::HdfsScanner::num_errors_in_file_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>number of errors in current file </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00183">183</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a922115776b26544df5d86ad82dff87ef"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int32_t impala::HdfsScanner::num_null_bytes_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Number of null bytes in the tuple. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00189">189</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8h_source.html#l00355">InitTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8196c5dc7e842622df7ab43b8cd68397"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> impala::HdfsScanner::parse_status_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Contains current parse status to minimize the number of <a class="el" href="classimpala_1_1Status.html">Status</a> objects returned. This significantly minimizes the cross compile dependencies for llvm since status objects inline a bunch of string functions. Also, status objects aren't extremely cheap to create and destroy. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00195">195</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00125">impala::HdfsAvroScanner::ParseMetadata()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00419">impala::HdfsSequenceScanner::ReadBlockHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00099">impala::HdfsAvroScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00345">impala::HdfsSequenceScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00107">impala::HdfsRCFileScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00197">impala::HdfsRCFileScanner::ReadNumColumnsMetadata()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00278">impala::HdfsRCFileScanner::ReadRowGroupHeader()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00170">impala::BaseSequenceScanner::ReadSync()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00212">impala::BaseSequenceScanner::SkipToSync()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a1b361d5cc69ef26b606349a26c1a8415"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a>* impala::HdfsScanner::scan_node_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The scan node that started this scanner. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00141">141</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00145">AddFinalRowBatch()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00240">impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00274">impala::BaseSequenceScanner::CloseFileRanges()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01050">impala::HdfsParquetScanner::CreateColumnReaders()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00472">impala::HdfsParquetScanner::CreateReader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00564">impala::HdfsRCFileScanner::DebugString()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">impala::HdfsTextScanner::FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01093">impala::HdfsParquetScanner::InitColumns()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00087">InitializeWriteTuplesFn()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">impala::HdfsTextScanner::InitNewRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00493">impala::HdfsAvroScanner::InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00068">impala::HdfsRCFileScanner::InitNewRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00125">impala::HdfsAvroScanner::ParseMetadata()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">impala::HdfsTextScanner::Prepare()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00074">impala::BaseSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00440">impala::HdfsParquetScanner::Prepare()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00088">impala::HdfsSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00060">impala::HdfsRCFileScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00532">impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00254">impala::HdfsRCFileScanner::ReadRowGroup()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">impala::HdfsTextScanner::ResetScanner()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00220">impala::HdfsAvroScanner::ResolveSchemas()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00513">UpdateDecompressor()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00426">impala::HdfsAvroScanner::VerifyTypesMatch()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">WriteCompleteTuple()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">WriteEmptyTuples()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">impala::HdfsTextScanner::WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8d1c8ea0d46e9378c6db2c24395a3f78"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* impala::HdfsScanner::state_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> for error reporting. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00144">144</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00082">Close()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">impala::HdfsTextScanner::Prepare()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00088">impala::HdfsSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00284">impala::BaseSequenceScanner::ReadPastSize()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00254">impala::HdfsRCFileScanner::ReadRowGroup()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00220">impala::HdfsAvroScanner::ResolveSchemas()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01290">impala::HdfsParquetScanner::ValidateColumn()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa539050780939fc85267c63e9f262e05"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1ScannerContext_1_1Stream.html">ScannerContext::Stream</a>* impala::HdfsScanner::stream_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The first stream for context_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00150">150</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">impala::HdfsTextScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01050">impala::HdfsParquetScanner::CreateColumnReaders()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00564">impala::HdfsRCFileScanner::DebugString()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">impala::HdfsTextScanner::FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">impala::HdfsTextScanner::FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">impala::HdfsTextScanner::FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">impala::HdfsTextScanner::FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">impala::HdfsTextScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00068">impala::HdfsRCFileScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00368">impala::HdfsRCFileScanner::NextField()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00125">impala::HdfsAvroScanner::ParseMetadata()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00074">impala::BaseSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">impala::HdfsTextScanner::ProcessSplit()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00419">impala::HdfsSequenceScanner::ReadBlockHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00099">impala::HdfsAvroScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00345">impala::HdfsSequenceScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00107">impala::HdfsRCFileScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00197">impala::HdfsRCFileScanner::ReadNumColumnsMetadata()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00278">impala::HdfsRCFileScanner::ReadRowGroupHeader()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00170">impala::BaseSequenceScanner::ReadSync()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">ReportTupleParseError()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00212">impala::BaseSequenceScanner::SkipToSync()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01263">impala::HdfsParquetScanner::ValidateFileMetadata()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00426">impala::HdfsAvroScanner::VerifyTypesMatch()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af1088c4f13acd1a6374c2693041f372e"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsScanner::template_tuple_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>A partially materialized tuple with only partition key slots set. The non-partition key slots are set to NULL. The template tuple must be copied into tuple_ before any of the other slots are materialized. Pointer is NULL if there are no partition key slots. This template tuple is computed once for each file and valid for the duration of that file. It is owned by the HDFS scan node. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00164">164</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00093">impala::HdfsAvroScanner::AllocateFileHeader()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01050">impala::HdfsParquetScanner::CreateColumnReaders()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00493">impala::HdfsAvroScanner::InitNewRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00220">impala::HdfsAvroScanner::ResolveSchemas()</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">WriteAlignedTuples()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aebc25f28c691efb3b377e2cf7bc728a8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1TextConverter.html">TextConverter</a>> impala::HdfsScanner::text_converter_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Helper class for converting text to other types;. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00186">186</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">impala::HdfsTextScanner::InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00060">impala::HdfsRCFileScanner::Prepare()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">WriteCompleteTuple()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">impala::HdfsTextScanner::WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a0f92b70b3120fbf0b5ed833d26176c39"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsScanner::tuple_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Current tuple pointer into tuple_mem_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00170">170</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">impala::HdfsTextScanner::FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">impala::HdfsTextScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">WriteAlignedTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a80ed99fe7814a4047c9220d31fc57536"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int impala::HdfsScanner::tuple_byte_size_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Fixed size of each tuple, in bytes. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00167">167</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00355">InitTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00363">next_tuple()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>, and <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">WriteAlignedTuples()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8b0b36b42ab140cbb793aa986683c432"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">uint8_t* impala::HdfsScanner::tuple_mem_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The tuple memory of batch_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00180">180</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00124">CommitRows()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00115">GetMemory()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00108">StartNewRowBatch()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aea086d8b6a0604df26216cacea3feeed"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1HdfsScanner.html#a1875e5409fedbadf979001ef761ae753">WriteTuplesFn</a> impala::HdfsScanner::write_tuples_fn_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Jitted write tuples function pointer. Null if codegen is disabled. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00215">215</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00087">InitializeWriteTuplesFn()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">impala::HdfsTextScanner::WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <hr/>The documentation for this class was generated from the following files:<ul> |
| <li>be/src/exec/<a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a></li> |
| <li>be/src/exec/<a class="el" href="hdfs-scanner-ir_8cc_source.html">hdfs-scanner-ir.cc</a></li> |
| <li>be/src/exec/<a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a></li> |
| </ul> |
| </div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="namespaceimpala.html">impala</a></li><li class="navelem"><a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:47 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
