| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: impala::HdfsTextScanner Class Reference</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li class="current"><a href="annotated.html"><span>Classes</span></a></li> |
| <li><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="annotated.html"><span>Class List</span></a></li> |
| <li><a href="classes.html"><span>Class Index</span></a></li> |
| <li><a href="inherits.html"><span>Class Hierarchy</span></a></li> |
| <li><a href="functions.html"><span>Class Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('classimpala_1_1HdfsTextScanner.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="summary"> |
| <a href="#pub-methods">Public Member Functions</a> | |
| <a href="#pub-static-methods">Static Public Member Functions</a> | |
| <a href="#pub-static-attribs">Static Public Attributes</a> | |
| <a href="#pro-types">Protected Types</a> | |
| <a href="#pro-methods">Protected Member Functions</a> | |
| <a href="#pro-static-methods">Static Protected Member Functions</a> | |
| <a href="#pro-attribs">Protected Attributes</a> | |
| <a href="#pri-methods">Private Member Functions</a> | |
| <a href="#pri-attribs">Private Attributes</a> | |
| <a href="#pri-static-attribs">Static Private Attributes</a> | |
| <a href="classimpala_1_1HdfsTextScanner-members.html">List of all members</a> </div> |
| <div class="headertitle"> |
| <div class="title">impala::HdfsTextScanner Class Reference</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| |
| <p><code>#include <<a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>></code></p> |
| <div class="dynheader"> |
| Inheritance diagram for impala::HdfsTextScanner:</div> |
| <div class="dyncontent"> |
| <div class="center"><div class="zoom"><iframe scrolling="no" frameborder="0" src="classimpala_1_1HdfsTextScanner__inherit__graph.svg" width="100%" height="600"><p><b>This browser is not able to show SVG: try Firefox, Chrome, Safari, or Opera instead.</b></p></iframe></div></div> |
| </div> |
| <div class="dynheader"> |
| Collaboration diagram for impala::HdfsTextScanner:</div> |
| <div class="dyncontent"> |
| <div class="center"><div class="zoom"><iframe scrolling="no" frameborder="0" src="classimpala_1_1HdfsTextScanner__coll__graph.svg" width="100%" height="600"><p><b>This browser is not able to show SVG: try Firefox, Chrome, Safari, or Opera instead.</b></p></iframe></div></div> |
| </div> |
| <table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-methods"></a> |
| Public Member Functions</h2></td></tr> |
| <tr class="memitem:a4d21be24ac5c7ee3d8803f6cec1e6d29"><td class="memItemLeft" align="right" valign="top"> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a4d21be24ac5c7ee3d8803f6cec1e6d29">HdfsTextScanner</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *scan_node, <a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> *state)</td></tr> |
| <tr class="separator:a4d21be24ac5c7ee3d8803f6cec1e6d29"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa0baa5f9785a91e8af28ada950689781"><td class="memItemLeft" align="right" valign="top">virtual </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#aa0baa5f9785a91e8af28ada950689781">~HdfsTextScanner</a> ()</td></tr> |
| <tr class="separator:aa0baa5f9785a91e8af28ada950689781"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a370ef8b83aa2f583182c0939d9ab1e10"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a370ef8b83aa2f583182c0939d9ab1e10">Prepare</a> (<a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> *context)</td></tr> |
| <tr class="memdesc:a370ef8b83aa2f583182c0939d9ab1e10"><td class="mdescLeft"> </td><td class="mdescRight">Implementation of <a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> interface. <a href="#a370ef8b83aa2f583182c0939d9ab1e10">More...</a><br/></td></tr> |
| <tr class="separator:a370ef8b83aa2f583182c0939d9ab1e10"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ad22f51e3ac67fce825aab6480477d839"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#ad22f51e3ac67fce825aab6480477d839">ProcessSplit</a> ()</td></tr> |
| <tr class="separator:ad22f51e3ac67fce825aab6480477d839"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a10b654ff3364a09cde00aec5dc64a21a"><td class="memItemLeft" align="right" valign="top">virtual void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a10b654ff3364a09cde00aec5dc64a21a">Close</a> ()</td></tr> |
| <tr class="separator:a10b654ff3364a09cde00aec5dc64a21a"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-static-methods"></a> |
| Static Public Member Functions</h2></td></tr> |
| <tr class="memitem:af79bef6374f38e24303a824b0952dfc8"><td class="memItemLeft" align="right" valign="top">static <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#af79bef6374f38e24303a824b0952dfc8">IssueInitialRanges</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *scan_node, const std::vector< <a class="el" href="structimpala_1_1HdfsFileDesc.html">HdfsFileDesc</a> * > &files)</td></tr> |
| <tr class="memdesc:af79bef6374f38e24303a824b0952dfc8"><td class="mdescLeft"> </td><td class="mdescRight">Issue io manager byte ranges for 'files'. <a href="#af79bef6374f38e24303a824b0952dfc8">More...</a><br/></td></tr> |
| <tr class="separator:af79bef6374f38e24303a824b0952dfc8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:abd5837c883c0e0490a228e7809ee5478"><td class="memItemLeft" align="right" valign="top">static llvm::Function * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#abd5837c883c0e0490a228e7809ee5478">Codegen</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *, const std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > &conjunct_ctxs)</td></tr> |
| <tr class="memdesc:abd5837c883c0e0490a228e7809ee5478"><td class="mdescLeft"> </td><td class="mdescRight">Codegen writing tuples and evaluating predicates. <a href="#abd5837c883c0e0490a228e7809ee5478">More...</a><br/></td></tr> |
| <tr class="separator:abd5837c883c0e0490a228e7809ee5478"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-static-attribs"></a> |
| Static Public Attributes</h2></td></tr> |
| <tr class="memitem:a4e11f799fcf9fd729da778e8167bf74a"><td class="memItemLeft" align="right" valign="top">static const std::string </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a4e11f799fcf9fd729da778e8167bf74a">LZO_INDEX_SUFFIX</a> = ".index"</td></tr> |
| <tr class="memdesc:a4e11f799fcf9fd729da778e8167bf74a"><td class="mdescLeft"> </td><td class="mdescRight">Suffix for lzo index files. <a href="#a4e11f799fcf9fd729da778e8167bf74a">More...</a><br/></td></tr> |
| <tr class="separator:a4e11f799fcf9fd729da778e8167bf74a"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a65fed1adb4fde1c960b9ae522ba20aaa"><td class="memItemLeft" align="right" valign="top">static const char * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a65fed1adb4fde1c960b9ae522ba20aaa">LLVM_CLASS_NAME</a> = "class.impala::HdfsTextScanner"</td></tr> |
| <tr class="separator:a65fed1adb4fde1c960b9ae522ba20aaa"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a0df2c19265535245917d3bb3b6095ba7"><td class="memItemLeft" align="right" valign="top">static const int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a0df2c19265535245917d3bb3b6095ba7">FILE_BLOCK_SIZE</a> = 4096</td></tr> |
| <tr class="separator:a0df2c19265535245917d3bb3b6095ba7"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-types"></a> |
| Protected Types</h2></td></tr> |
| <tr class="memitem:a1875e5409fedbadf979001ef761ae753"><td class="memItemLeft" align="right" valign="top">typedef int(* </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a1875e5409fedbadf979001ef761ae753">WriteTuplesFn</a> )(<a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> *, <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *, int, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *, int, int, int, int)</td></tr> |
| <tr class="separator:a1875e5409fedbadf979001ef761ae753"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-methods"></a> |
| Protected Member Functions</h2></td></tr> |
| <tr class="memitem:a3078e6f9306e5cbd2aa7cd0032e69ce8"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a3078e6f9306e5cbd2aa7cd0032e69ce8">ResetScanner</a> ()</td></tr> |
| <tr class="separator:a3078e6f9306e5cbd2aa7cd0032e69ce8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:adab08ccc8e27d64a6473683eabaadb84"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#adab08ccc8e27d64a6473683eabaadb84">InitializeWriteTuplesFn</a> (<a class="el" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a> *partition, THdfsFileFormat::type type, const std::string &scanner_name)</td></tr> |
| <tr class="separator:adab08ccc8e27d64a6473683eabaadb84"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a370d072e6ebd37891c012fc9da453133"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a370d072e6ebd37891c012fc9da453133">StartNewRowBatch</a> ()</td></tr> |
| <tr class="memdesc:a370d072e6ebd37891c012fc9da453133"><td class="mdescLeft"> </td><td class="mdescRight">Set batch_ to a new row batch and update tuple_mem_ accordingly. <a href="#a370d072e6ebd37891c012fc9da453133">More...</a><br/></td></tr> |
| <tr class="separator:a370d072e6ebd37891c012fc9da453133"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ac2a391565eba0ab7cf58a7a024032220"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ac2a391565eba0ab7cf58a7a024032220">GetMemory</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> **<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> **tuple_mem, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> **tuple_row_mem)</td></tr> |
| <tr class="separator:ac2a391565eba0ab7cf58a7a024032220"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab72a9901f5c1c2cb0f053ce6ad45d9b6"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ab72a9901f5c1c2cb0f053ce6ad45d9b6">CommitRows</a> (int num_rows)</td></tr> |
| <tr class="separator:ab72a9901f5c1c2cb0f053ce6ad45d9b6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a36e661fbaff23879b78f53cdc1a56552"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a36e661fbaff23879b78f53cdc1a56552">AddFinalRowBatch</a> ()</td></tr> |
| <tr class="separator:a36e661fbaff23879b78f53cdc1a56552"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a94b437624601d6e7626e9b3d59cd87da"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a94b437624601d6e7626e9b3d59cd87da">AttachPool</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="classbool.html">bool</a> commit_batch)</td></tr> |
| <tr class="separator:a94b437624601d6e7626e9b3d59cd87da"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa329170edaabadb0f1ff94de121b3cec"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> <a class="el" href="impala-ir_8h.html#af2b653018219cda8c9a2d9efc0ce3f44">IR_ALWAYS_INLINE</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aa329170edaabadb0f1ff94de121b3cec">EvalConjuncts</a> (<a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *row)</td></tr> |
| <tr class="separator:aa329170edaabadb0f1ff94de121b3cec"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af5a25b22a0fb0b5bea62602fe224b021"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#af5a25b22a0fb0b5bea62602fe224b021">WriteEmptyTuples</a> (<a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a> *row_batch, int num_tuples)</td></tr> |
| <tr class="separator:af5a25b22a0fb0b5bea62602fe224b021"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae748aa64d908ff8a1fdcf3ae942ee51c"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae748aa64d908ff8a1fdcf3ae942ee51c">WriteEmptyTuples</a> (<a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> *context, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row, int num_tuples)</td></tr> |
| <tr class="memdesc:ae748aa64d908ff8a1fdcf3ae942ee51c"><td class="mdescLeft"> </td><td class="mdescRight">Write empty tuples and commit them to the context object. <a href="#ae748aa64d908ff8a1fdcf3ae942ee51c">More...</a><br/></td></tr> |
| <tr class="separator:ae748aa64d908ff8a1fdcf3ae942ee51c"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ac95a9d24d8da6d038c93b177c9c48e6b"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ac95a9d24d8da6d038c93b177c9c48e6b">WriteAlignedTuples</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row_mem, int row_size, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *fields, int num_tuples, int max_added_tuples, int slots_per_tuple, int row_start_indx)</td></tr> |
| <tr class="separator:ac95a9d24d8da6d038c93b177c9c48e6b"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab7c78e78c88f81f7ff9d87ed12eece4e"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ab7c78e78c88f81f7ff9d87ed12eece4e">UpdateDecompressor</a> (const THdfsCompression::type &compression)</td></tr> |
| <tr class="separator:ab7c78e78c88f81f7ff9d87ed12eece4e"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aace11af0dd18b9d9dbf7afbc74dab1ba"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aace11af0dd18b9d9dbf7afbc74dab1ba">UpdateDecompressor</a> (const std::string &codec)</td></tr> |
| <tr class="separator:aace11af0dd18b9d9dbf7afbc74dab1ba"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:afd378fe0f2500a3a63012c7b33be4c90"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#afd378fe0f2500a3a63012c7b33be4c90">ReportTupleParseError</a> (<a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *fields, uint8_t *errors, int row_idx)</td></tr> |
| <tr class="separator:afd378fe0f2500a3a63012c7b33be4c90"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ad6bf05f906b32b58d7df914cbd115492"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ad6bf05f906b32b58d7df914cbd115492">WriteCompleteTuple</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *fields, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *tuple, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *template_tuple, uint8_t *error_fields, uint8_t *error_in_row)</td></tr> |
| <tr class="separator:ad6bf05f906b32b58d7df914cbd115492"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae05ee8b2f0b254228afcf6abf540f8f4"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae05ee8b2f0b254228afcf6abf540f8f4">ReportColumnParseError</a> (const <a class="el" href="classimpala_1_1SlotDescriptor.html">SlotDescriptor</a> *desc, const char *data, int len)</td></tr> |
| <tr class="separator:ae05ee8b2f0b254228afcf6abf540f8f4"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:accd7ea935f503dfe61c1570d1b9c07d8"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#accd7ea935f503dfe61c1570d1b9c07d8">InitTuple</a> (<a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *template_tuple, <a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *tuple)</td></tr> |
| <tr class="separator:accd7ea935f503dfe61c1570d1b9c07d8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a2ea24bc3f271386fe3e24336c7956ca7"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a2ea24bc3f271386fe3e24336c7956ca7">next_tuple</a> (<a class="el" href="classimpala_1_1Tuple.html">Tuple</a> *t) const </td></tr> |
| <tr class="separator:a2ea24bc3f271386fe3e24336c7956ca7"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae2cf3a0764b67bb1a1733daaa772e045"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae2cf3a0764b67bb1a1733daaa772e045">next_row</a> (<a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *r) const </td></tr> |
| <tr class="separator:ae2cf3a0764b67bb1a1733daaa772e045"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a77ca8ddf41e61984b20255881759ef99"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a77ca8ddf41e61984b20255881759ef99">GetConjunctCtx</a> (int idx) const </td></tr> |
| <tr class="separator:a77ca8ddf41e61984b20255881759ef99"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-static-methods"></a> |
| Static Protected Member Functions</h2></td></tr> |
| <tr class="memitem:ab0d4c09c4bacaf57bf5bf238da3cd5e8"><td class="memItemLeft" align="right" valign="top">static llvm::Function * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ab0d4c09c4bacaf57bf5bf238da3cd5e8">CodegenWriteCompleteTuple</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *, <a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> *, const std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > &conjunct_ctxs)</td></tr> |
| <tr class="separator:ab0d4c09c4bacaf57bf5bf238da3cd5e8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aac18b5e53c864fa9ac9a574e00518a4a"><td class="memItemLeft" align="right" valign="top">static llvm::Function * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aac18b5e53c864fa9ac9a574e00518a4a">CodegenWriteAlignedTuples</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *, <a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> *, llvm::Function *write_tuple_fn)</td></tr> |
| <tr class="separator:aac18b5e53c864fa9ac9a574e00518a4a"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pro-attribs"></a> |
| Protected Attributes</h2></td></tr> |
| <tr class="memitem:a0f2cf6306114cd79fcd2a4fa26084275"><td class="memItemLeft" align="right" valign="top">char * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a0f2cf6306114cd79fcd2a4fa26084275">byte_buffer_ptr_</a></td></tr> |
| <tr class="memdesc:a0f2cf6306114cd79fcd2a4fa26084275"><td class="mdescLeft"> </td><td class="mdescRight">Current position in byte buffer. <a href="#a0f2cf6306114cd79fcd2a4fa26084275">More...</a><br/></td></tr> |
| <tr class="separator:a0f2cf6306114cd79fcd2a4fa26084275"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a2c16649190b8816c77c0acf1a0a205ae"><td class="memItemLeft" align="right" valign="top">char * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a2c16649190b8816c77c0acf1a0a205ae">byte_buffer_end_</a></td></tr> |
| <tr class="memdesc:a2c16649190b8816c77c0acf1a0a205ae"><td class="mdescLeft"> </td><td class="mdescRight">Ending position of HDFS buffer. <a href="#a2c16649190b8816c77c0acf1a0a205ae">More...</a><br/></td></tr> |
| <tr class="separator:a2c16649190b8816c77c0acf1a0a205ae"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a4ec6d690e88c717ecfc947dea843ca47"><td class="memItemLeft" align="right" valign="top">int64_t </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a4ec6d690e88c717ecfc947dea843ca47">byte_buffer_read_size_</a></td></tr> |
| <tr class="memdesc:a4ec6d690e88c717ecfc947dea843ca47"><td class="mdescLeft"> </td><td class="mdescRight">Actual bytes received from last file read. <a href="#a4ec6d690e88c717ecfc947dea843ca47">More...</a><br/></td></tr> |
| <tr class="separator:a4ec6d690e88c717ecfc947dea843ca47"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a91b85de27fc7842c0b8fe55a59a0b766"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a91b85de27fc7842c0b8fe55a59a0b766">only_parsing_header_</a></td></tr> |
| <tr class="memdesc:a91b85de27fc7842c0b8fe55a59a0b766"><td class="mdescLeft"> </td><td class="mdescRight">True if we are parsing the header for this scanner. <a href="#a91b85de27fc7842c0b8fe55a59a0b766">More...</a><br/></td></tr> |
| <tr class="separator:a91b85de27fc7842c0b8fe55a59a0b766"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a1b361d5cc69ef26b606349a26c1a8415"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a1b361d5cc69ef26b606349a26c1a8415">scan_node_</a></td></tr> |
| <tr class="memdesc:a1b361d5cc69ef26b606349a26c1a8415"><td class="mdescLeft"> </td><td class="mdescRight">The scan node that started this scanner. <a href="#a1b361d5cc69ef26b606349a26c1a8415">More...</a><br/></td></tr> |
| <tr class="separator:a1b361d5cc69ef26b606349a26c1a8415"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8d1c8ea0d46e9378c6db2c24395a3f78"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8d1c8ea0d46e9378c6db2c24395a3f78">state_</a></td></tr> |
| <tr class="memdesc:a8d1c8ea0d46e9378c6db2c24395a3f78"><td class="mdescLeft"> </td><td class="mdescRight"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> for error reporting. <a href="#a8d1c8ea0d46e9378c6db2c24395a3f78">More...</a><br/></td></tr> |
| <tr class="separator:a8d1c8ea0d46e9378c6db2c24395a3f78"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a6c1f7899de6ac5baca6914640f4859f4"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a6c1f7899de6ac5baca6914640f4859f4">context_</a></td></tr> |
| <tr class="memdesc:a6c1f7899de6ac5baca6914640f4859f4"><td class="mdescLeft"> </td><td class="mdescRight">Context for this scanner. <a href="#a6c1f7899de6ac5baca6914640f4859f4">More...</a><br/></td></tr> |
| <tr class="separator:a6c1f7899de6ac5baca6914640f4859f4"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa539050780939fc85267c63e9f262e05"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1ScannerContext_1_1Stream.html">ScannerContext::Stream</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aa539050780939fc85267c63e9f262e05">stream_</a></td></tr> |
| <tr class="memdesc:aa539050780939fc85267c63e9f262e05"><td class="mdescLeft"> </td><td class="mdescRight">The first stream for context_. <a href="#aa539050780939fc85267c63e9f262e05">More...</a><br/></td></tr> |
| <tr class="separator:aa539050780939fc85267c63e9f262e05"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:afb3a3d0ceb454e1ea3aa549786e45fac"><td class="memItemLeft" align="right" valign="top">std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#afb3a3d0ceb454e1ea3aa549786e45fac">conjunct_ctxs_</a></td></tr> |
| <tr class="separator:afb3a3d0ceb454e1ea3aa549786e45fac"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af1088c4f13acd1a6374c2693041f372e"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#af1088c4f13acd1a6374c2693041f372e">template_tuple_</a></td></tr> |
| <tr class="separator:af1088c4f13acd1a6374c2693041f372e"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a80ed99fe7814a4047c9220d31fc57536"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a80ed99fe7814a4047c9220d31fc57536">tuple_byte_size_</a></td></tr> |
| <tr class="memdesc:a80ed99fe7814a4047c9220d31fc57536"><td class="mdescLeft"> </td><td class="mdescRight">Fixed size of each tuple, in bytes. <a href="#a80ed99fe7814a4047c9220d31fc57536">More...</a><br/></td></tr> |
| <tr class="separator:a80ed99fe7814a4047c9220d31fc57536"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a0f92b70b3120fbf0b5ed833d26176c39"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a0f92b70b3120fbf0b5ed833d26176c39">tuple_</a></td></tr> |
| <tr class="memdesc:a0f92b70b3120fbf0b5ed833d26176c39"><td class="mdescLeft"> </td><td class="mdescRight">Current tuple pointer into tuple_mem_. <a href="#a0f92b70b3120fbf0b5ed833d26176c39">More...</a><br/></td></tr> |
| <tr class="separator:a0f92b70b3120fbf0b5ed833d26176c39"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a5c6954a13e4a285d2e4b544574c03b76"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a5c6954a13e4a285d2e4b544574c03b76">batch_</a></td></tr> |
| <tr class="separator:a5c6954a13e4a285d2e4b544574c03b76"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8b0b36b42ab140cbb793aa986683c432"><td class="memItemLeft" align="right" valign="top">uint8_t * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8b0b36b42ab140cbb793aa986683c432">tuple_mem_</a></td></tr> |
| <tr class="memdesc:a8b0b36b42ab140cbb793aa986683c432"><td class="mdescLeft"> </td><td class="mdescRight">The tuple memory of batch_. <a href="#a8b0b36b42ab140cbb793aa986683c432">More...</a><br/></td></tr> |
| <tr class="separator:a8b0b36b42ab140cbb793aa986683c432"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a06581f363b634869efc5d09839bc3e25"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a06581f363b634869efc5d09839bc3e25">num_errors_in_file_</a></td></tr> |
| <tr class="memdesc:a06581f363b634869efc5d09839bc3e25"><td class="mdescLeft"> </td><td class="mdescRight">number of errors in current file <a href="#a06581f363b634869efc5d09839bc3e25">More...</a><br/></td></tr> |
| <tr class="separator:a06581f363b634869efc5d09839bc3e25"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aebc25f28c691efb3b377e2cf7bc728a8"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1TextConverter.html">TextConverter</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aebc25f28c691efb3b377e2cf7bc728a8">text_converter_</a></td></tr> |
| <tr class="memdesc:aebc25f28c691efb3b377e2cf7bc728a8"><td class="mdescLeft"> </td><td class="mdescRight">Helper class for converting text to other types;. <a href="#aebc25f28c691efb3b377e2cf7bc728a8">More...</a><br/></td></tr> |
| <tr class="separator:aebc25f28c691efb3b377e2cf7bc728a8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a922115776b26544df5d86ad82dff87ef"><td class="memItemLeft" align="right" valign="top">int32_t </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a922115776b26544df5d86ad82dff87ef">num_null_bytes_</a></td></tr> |
| <tr class="memdesc:a922115776b26544df5d86ad82dff87ef"><td class="mdescLeft"> </td><td class="mdescRight">Number of null bytes in the tuple. <a href="#a922115776b26544df5d86ad82dff87ef">More...</a><br/></td></tr> |
| <tr class="separator:a922115776b26544df5d86ad82dff87ef"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8196c5dc7e842622df7ab43b8cd68397"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8196c5dc7e842622df7ab43b8cd68397">parse_status_</a></td></tr> |
| <tr class="separator:a8196c5dc7e842622df7ab43b8cd68397"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a8d995c8326def52de037ecf074fc2654"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1Codec.html">Codec</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a8d995c8326def52de037ecf074fc2654">decompressor_</a></td></tr> |
| <tr class="memdesc:a8d995c8326def52de037ecf074fc2654"><td class="mdescLeft"> </td><td class="mdescRight">Decompressor class to use, if any. <a href="#a8d995c8326def52de037ecf074fc2654">More...</a><br/></td></tr> |
| <tr class="separator:a8d995c8326def52de037ecf074fc2654"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aba0c7e8797db0a61444e50f342c775c6"><td class="memItemLeft" align="right" valign="top">THdfsCompression::type </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aba0c7e8797db0a61444e50f342c775c6">decompression_type_</a></td></tr> |
| <tr class="memdesc:aba0c7e8797db0a61444e50f342c775c6"><td class="mdescLeft"> </td><td class="mdescRight">The most recently used decompression type. <a href="#aba0c7e8797db0a61444e50f342c775c6">More...</a><br/></td></tr> |
| <tr class="separator:aba0c7e8797db0a61444e50f342c775c6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ae86f3019c271de0d0593204003ee1653"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#ae86f3019c271de0d0593204003ee1653">data_buffer_pool_</a></td></tr> |
| <tr class="separator:ae86f3019c271de0d0593204003ee1653"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a7bd58a77c6acecf09cbfcc9b4e592618"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#a7bd58a77c6acecf09cbfcc9b4e592618">decompress_timer_</a></td></tr> |
| <tr class="memdesc:a7bd58a77c6acecf09cbfcc9b4e592618"><td class="mdescLeft"> </td><td class="mdescRight">Time spent decompressing bytes. <a href="#a7bd58a77c6acecf09cbfcc9b4e592618">More...</a><br/></td></tr> |
| <tr class="separator:a7bd58a77c6acecf09cbfcc9b4e592618"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aea086d8b6a0604df26216cacea3feeed"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1HdfsScanner.html#a1875e5409fedbadf979001ef761ae753">WriteTuplesFn</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsScanner.html#aea086d8b6a0604df26216cacea3feeed">write_tuples_fn_</a></td></tr> |
| <tr class="memdesc:aea086d8b6a0604df26216cacea3feeed"><td class="mdescLeft"> </td><td class="mdescRight">Jitted write tuples function pointer. Null if codegen is disabled. <a href="#aea086d8b6a0604df26216cacea3feeed">More...</a><br/></td></tr> |
| <tr class="separator:aea086d8b6a0604df26216cacea3feeed"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pri-methods"></a> |
| Private Member Functions</h2></td></tr> |
| <tr class="memitem:a4cebfa743efe38c408fc73b53422e851"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a4cebfa743efe38c408fc73b53422e851">InitNewRange</a> ()</td></tr> |
| <tr class="separator:a4cebfa743efe38c408fc73b53422e851"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a4c9ad94b3cf268c69e691b08d575cc71"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a4c9ad94b3cf268c69e691b08d575cc71">FindFirstTuple</a> (<a class="el" href="classbool.html">bool</a> *tuple_found)</td></tr> |
| <tr class="separator:a4c9ad94b3cf268c69e691b08d575cc71"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a615822ff5f2cf540f07f64cfbb9ae412"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a615822ff5f2cf540f07f64cfbb9ae412">ProcessRange</a> (int *num_tuples, <a class="el" href="classbool.html">bool</a> past_scan_range)</td></tr> |
| <tr class="separator:a615822ff5f2cf540f07f64cfbb9ae412"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a10766e88beda8fc0696eb27821658307"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a10766e88beda8fc0696eb27821658307">FinishScanRange</a> ()</td></tr> |
| <tr class="memdesc:a10766e88beda8fc0696eb27821658307"><td class="mdescLeft"> </td><td class="mdescRight">Reads past the end of the scan range for the next tuple end. <a href="#a10766e88beda8fc0696eb27821658307">More...</a><br/></td></tr> |
| <tr class="separator:a10766e88beda8fc0696eb27821658307"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a023ad955932e3854826b9f35b28e69a3"><td class="memItemLeft" align="right" valign="top">virtual <a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a023ad955932e3854826b9f35b28e69a3">FillByteBuffer</a> (<a class="el" href="classbool.html">bool</a> *eosr, int num_bytes=0)</td></tr> |
| <tr class="separator:a023ad955932e3854826b9f35b28e69a3"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab971b60e69d77383eb5a9935d0107247"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#ab971b60e69d77383eb5a9935d0107247">FillByteBufferCompressedFile</a> (<a class="el" href="classbool.html">bool</a> *eosr)</td></tr> |
| <tr class="separator:ab971b60e69d77383eb5a9935d0107247"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ab8a17657c224d4cdb37147b98945c0b6"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#ab8a17657c224d4cdb37147b98945c0b6">FillByteBufferGzip</a> (<a class="el" href="classbool.html">bool</a> *eosr)</td></tr> |
| <tr class="separator:ab8a17657c224d4cdb37147b98945c0b6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a4ed1c34d70614ea3c693ae64a7ea2cce"><td class="memItemLeft" align="right" valign="top">void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a4ed1c34d70614ea3c693ae64a7ea2cce">CopyBoundaryField</a> (<a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *data, <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *<a class="el" href="expr-benchmark_8cc.html#a3a5de7bd423fbc0afc4cf935c166ca6b">pool</a>)</td></tr> |
| <tr class="separator:a4ed1c34d70614ea3c693ae64a7ea2cce"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aacd8f4c470ed895a8edca42753249cc1"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#aacd8f4c470ed895a8edca42753249cc1">WriteFields</a> (<a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *tuple_row_mem, int num_fields, int num_tuples)</td></tr> |
| <tr class="separator:aacd8f4c470ed895a8edca42753249cc1"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a9450d62287c2c03d6ef7b1b7df9c604c"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a9450d62287c2c03d6ef7b1b7df9c604c">WritePartialTuple</a> (<a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *, int num_fields, <a class="el" href="classbool.html">bool</a> copy_strings)</td></tr> |
| <tr class="separator:a9450d62287c2c03d6ef7b1b7df9c604c"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:aa54a763cd29781bd631158d72bd269ad"><td class="memItemLeft" align="right" valign="top">virtual void </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#aa54a763cd29781bd631158d72bd269ad">LogRowParseError</a> (int row_idx, std::stringstream *)</td></tr> |
| <tr class="separator:aa54a763cd29781bd631158d72bd269ad"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pri-attribs"></a> |
| Private Attributes</h2></td></tr> |
| <tr class="memitem:a11e34b1126e3e884ff50c74e89ed79ff"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr< <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a11e34b1126e3e884ff50c74e89ed79ff">boundary_pool_</a></td></tr> |
| <tr class="memdesc:a11e34b1126e3e884ff50c74e89ed79ff"><td class="mdescLeft"> </td><td class="mdescRight">Mem pool for boundary_row_ and boundary_column_. <a href="#a11e34b1126e3e884ff50c74e89ed79ff">More...</a><br/></td></tr> |
| <tr class="separator:a11e34b1126e3e884ff50c74e89ed79ff"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a2c68a3e019f0700490a7e6ed5a05dc75"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1StringBuffer.html">StringBuffer</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a2c68a3e019f0700490a7e6ed5a05dc75">boundary_row_</a></td></tr> |
| <tr class="separator:a2c68a3e019f0700490a7e6ed5a05dc75"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a0fe0caecdbe8e8af0ee793213f226608"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1StringBuffer.html">StringBuffer</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a0fe0caecdbe8e8af0ee793213f226608">boundary_column_</a></td></tr> |
| <tr class="memdesc:a0fe0caecdbe8e8af0ee793213f226608"><td class="mdescLeft"> </td><td class="mdescRight">Helper string for dealing with columns that span file blocks. <a href="#a0fe0caecdbe8e8af0ee793213f226608">More...</a><br/></td></tr> |
| <tr class="separator:a0fe0caecdbe8e8af0ee793213f226608"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a5847845a7ec2240820018a5dfc64e75a"><td class="memItemLeft" align="right" valign="top">int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a5847845a7ec2240820018a5dfc64e75a">slot_idx_</a></td></tr> |
| <tr class="memdesc:a5847845a7ec2240820018a5dfc64e75a"><td class="mdescLeft"> </td><td class="mdescRight">Index into materialized_slots_ for the next slot to output for the current tuple. <a href="#a5847845a7ec2240820018a5dfc64e75a">More...</a><br/></td></tr> |
| <tr class="separator:a5847845a7ec2240820018a5dfc64e75a"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a35beeba40f0e22c352a00863db60a3f5"><td class="memItemLeft" align="right" valign="top">boost::scoped_ptr<br class="typebreak"/> |
| < <a class="el" href="classimpala_1_1DelimitedTextParser.html">DelimitedTextParser</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a35beeba40f0e22c352a00863db60a3f5">delimited_text_parser_</a></td></tr> |
| <tr class="memdesc:a35beeba40f0e22c352a00863db60a3f5"><td class="mdescLeft"> </td><td class="mdescRight">Helper class for picking fields and rows from delimited text. <a href="#a35beeba40f0e22c352a00863db60a3f5">More...</a><br/></td></tr> |
| <tr class="separator:a35beeba40f0e22c352a00863db60a3f5"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a293f2bde8183f6eee3402ec56248b9d8"><td class="memItemLeft" align="right" valign="top">std::vector< <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a293f2bde8183f6eee3402ec56248b9d8">field_locations_</a></td></tr> |
| <tr class="memdesc:a293f2bde8183f6eee3402ec56248b9d8"><td class="mdescLeft"> </td><td class="mdescRight">Return field locations from the Delimited Text Parser. <a href="#a293f2bde8183f6eee3402ec56248b9d8">More...</a><br/></td></tr> |
| <tr class="separator:a293f2bde8183f6eee3402ec56248b9d8"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:af5a1a7043e1742e1ac83b221b09a0342"><td class="memItemLeft" align="right" valign="top">std::vector< char * > </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#af5a1a7043e1742e1ac83b221b09a0342">row_end_locations_</a></td></tr> |
| <tr class="separator:af5a1a7043e1742e1ac83b221b09a0342"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a296803327e3e709428a6a3dc6c75ac64"><td class="memItemLeft" align="right" valign="top">char * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a296803327e3e709428a6a3dc6c75ac64">batch_start_ptr_</a></td></tr> |
| <tr class="separator:a296803327e3e709428a6a3dc6c75ac64"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ad7fa312f29fbe765c3c81729d94d7e80"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#ad7fa312f29fbe765c3c81729d94d7e80">error_in_row_</a></td></tr> |
| <tr class="separator:ad7fa312f29fbe765c3c81729d94d7e80"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:ad61234b712a93ddbe385c8040ea809d6"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#ad61234b712a93ddbe385c8040ea809d6">partial_tuple_</a></td></tr> |
| <tr class="separator:ad61234b712a93ddbe385c8040ea809d6"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a26e56274c0ad92d7b9a76010f15887a3"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a> </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a26e56274c0ad92d7b9a76010f15887a3">partial_tuple_empty_</a></td></tr> |
| <tr class="separator:a26e56274c0ad92d7b9a76010f15887a3"><td class="memSeparator" colspan="2"> </td></tr> |
| <tr class="memitem:a3d58a37c969e8483a458cb48d7ab40f9"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a> * </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#a3d58a37c969e8483a458cb48d7ab40f9">parse_delimiter_timer_</a></td></tr> |
| <tr class="memdesc:a3d58a37c969e8483a458cb48d7ab40f9"><td class="mdescLeft"> </td><td class="mdescRight">Time parsing text files. <a href="#a3d58a37c969e8483a458cb48d7ab40f9">More...</a><br/></td></tr> |
| <tr class="separator:a3d58a37c969e8483a458cb48d7ab40f9"><td class="memSeparator" colspan="2"> </td></tr> |
| </table><table class="memberdecls"> |
| <tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pri-static-attribs"></a> |
| Static Private Attributes</h2></td></tr> |
| <tr class="memitem:ab3ed30a9bc70a9354b39d7855a871249"><td class="memItemLeft" align="right" valign="top">static const int </td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1HdfsTextScanner.html#ab3ed30a9bc70a9354b39d7855a871249">NEXT_BLOCK_READ_SIZE</a> = 1024</td></tr> |
| <tr class="separator:ab3ed30a9bc70a9354b39d7855a871249"><td class="memSeparator" colspan="2"> </td></tr> |
| </table> |
| <a name="details" id="details"></a><h2 class="groupheader">Detailed Description</h2> |
| <div class="textblock"><p><a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> implementation that understands text-formatted records. Uses SSE instructions, if available, for performance. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00030">30</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| </div><h2 class="groupheader">Member Typedef Documentation</h2> |
| <a class="anchor" id="a1875e5409fedbadf979001ef761ae753"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">typedef int(* impala::HdfsScanner::WriteTuplesFn)(<a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> *, <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> *, <a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> *, int, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *, int, int, int, int)</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Matching typedef for WriteAlignedTuples for codegen. Refer to comments for that function. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00212">212</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Constructor & Destructor Documentation</h2> |
| <a class="anchor" id="a4d21be24ac5c7ee3d8803f6cec1e6d29"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">HdfsTextScanner::HdfsTextScanner </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname"><em>scan_node</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> * </td> |
| <td class="paramname"><em>state</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00051">51</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa0baa5f9785a91e8af28ada950689781"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">HdfsTextScanner::~HdfsTextScanner </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00064">64</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Member Function Documentation</h2> |
| <a class="anchor" id="a36e661fbaff23879b78f53cdc1a56552"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::AddFinalRowBatch </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Attach all remaining resources from context_ to batch_ and send batch_ to the scan node. This must be called after all rows have been committed and no further resources are needed from context_ (in practice this will happen in each scanner subclass's <a class="el" href="classimpala_1_1HdfsScanner.html#a61d4a073305f6befe9f12bad12f9de48">Close()</a> implementation). </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00145">145</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scan-node_8cc_source.html#l00688">impala::HdfsScanNode::AddMaterializedRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="scanner-context_8cc_source.html#l00045">impala::ScannerContext::ReleaseCompletedResources()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, and <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a94b437624601d6e7626e9b3d59cd87da"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void impala::HdfsScanner::AttachPool </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> </td> |
| <td class="paramname"><em>commit_batch</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Release all memory in 'pool' to batch_. If commit_batch is true, the row batch will be committed. commit_batch should be true if the attached pool is expected to be non-trivial (i.e. a decompression buffer) to minimize scanner mem usage. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00256">256</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="mem-pool_8cc_source.html#l00161">impala::MemPool::AcquireData()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, and <a class="el" href="row-batch_8h_source.html#l00148">impala::RowBatch::tuple_data_pool()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00532">impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00231">impala::HdfsRCFileScanner::ResetRowGroup()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a10b654ff3364a09cde00aec5dc64a21a"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsTextScanner::Close </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Release all resources the scanner has allocated. This is the last chance for the scanner to attach any resources to the <a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> object. </p> |
| |
| <p>Reimplemented from <a class="el" href="classimpala_1_1HdfsScanner.html#a61d4a073305f6befe9f12bad12f9de48">impala::HdfsScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">185</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8cc_source.html#l00145">impala::HdfsScanner::AddFinalRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00256">impala::HdfsScanner::AttachPool()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00137">boundary_pool_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00082">impala::HdfsScanner::Close()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00205">impala::HdfsScanner::data_buffer_pool_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">impala::HdfsScanner::decompressor_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00067">impala::HdfsFileDesc::file_compression</a>, <a class="el" href="scanner-context_8h_source.html#l00120">impala::ScannerContext::Stream::file_desc()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00068">only_parsing_header_</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00924">impala::HdfsScanNode::RangeComplete()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="abd5837c883c0e0490a228e7809ee5478"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Function * HdfsTextScanner::Codegen </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > & </td> |
| <td class="paramname"><em>conjunct_ctxs</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Codegen writing tuples and evaluating predicates. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00609">609</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00183">impala::RuntimeState::codegen_enabled()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00495">impala::HdfsScanner::CodegenWriteAlignedTuples()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00296">impala::HdfsScanner::CodegenWriteCompleteTuple()</a>, <a class="el" href="runtime-state_8cc_source.html#l00312">impala::RuntimeState::GetCodegen()</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00136">impala::HdfsScanNode::runtime_state()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scan-node_8cc_source.html#l00304">impala::HdfsScanNode::Prepare()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aac18b5e53c864fa9ac9a574e00518a4a"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Function * HdfsScanner::CodegenWriteAlignedTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">llvm::Function * </td> |
| <td class="paramname"><em>write_tuple_fn</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Codegen function to replace WriteAlignedTuples. WriteAlignedTuples is cross compiled to IR. This function loads the precompiled IR function, modifies it and returns the resulting function. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00495">495</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="llvm-codegen_8h_source.html#l00135">impala::LlvmCodeGen::codegen_timer()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00596">impala::LlvmCodeGen::FinalizeFunction()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00421">impala::LlvmCodeGen::GetFunction()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00489">impala::LlvmCodeGen::ReplaceCallSites()</a>, and <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00609">Codegen()</a>, and <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00050">impala::HdfsSequenceScanner::Codegen()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab0d4c09c4bacaf57bf5bf238da3cd5e8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">Function * HdfsScanner::CodegenWriteCompleteTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1LlvmCodeGen.html">LlvmCodeGen</a> * </td> |
| <td class="paramname">, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const std::vector< <a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * > & </td> |
| <td class="paramname"><em>conjunct_ctxs</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Codegen function to replace WriteCompleteTuple. Should behave identically to WriteCompleteTuple. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00296">296</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="llvm-codegen_8h_source.html#l00171">impala::LlvmCodeGen::FnPrototype::AddArgument()</a>, <a class="el" href="descriptors_8h_source.html#l00300">impala::TupleDescriptor::byte_size()</a>, <a class="el" href="llvm-codegen_8h_source.html#l00135">impala::LlvmCodeGen::codegen_timer()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00933">impala::LlvmCodeGen::CodegenMemcpy()</a>, <a class="el" href="text-converter_8cc_source.html#l00099">impala::TextConverter::CodegenWriteSlot()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00956">impala::HdfsScanNode::ComputeSlotMaterializationOrder()</a>, <a class="el" href="llvm-codegen_8h_source.html#l00214">impala::LlvmCodeGen::context()</a>, <a class="el" href="codegen-anyval_8cc_source.html#l00148">impala::CodegenAnyVal::CreateCallWrapped()</a>, <a class="el" href="llvm-codegen_8h_source.html#l00381">impala::LlvmCodeGen::false_value()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00596">impala::LlvmCodeGen::FinalizeFunction()</a>, <a class="el" href="descriptors_8cc_source.html#l00556">impala::TupleDescriptor::GenerateLlvmStruct()</a>, <a class="el" href="status_8cc_source.html#l00184">impala::Status::GetDetail()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00421">impala::LlvmCodeGen::GetFunction()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00371">impala::LlvmCodeGen::GetIntConstant()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00312">impala::LlvmCodeGen::GetType()</a>, <a class="el" href="codegen-anyval_8cc_source.html#l00258">impala::CodegenAnyVal::GetVal()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00134">impala::HdfsScanNode::hdfs_table()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00061">impala::FieldLocation::LLVM_CLASS_NAME</a>, <a class="el" href="tuple-row_8h_source.html#l00076">impala::TupleRow::LLVM_CLASS_NAME</a>, <a class="el" href="tuple_8h_source.html#l00134">impala::Tuple::LLVM_CLASS_NAME</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00137">impala::HdfsScanner::LLVM_CLASS_NAME</a>, <a class="el" href="mem-pool_8h_source.html#l00177">impala::MemPool::LLVM_CLASS_NAME</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="descriptors_8h_source.html#l00233">impala::HdfsTableDescriptor::null_column_value()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00130">impala::HdfsScanNode::num_materialized_partition_keys()</a>, <a class="el" href="descriptors_8h_source.html#l00301">impala::TupleDescriptor::num_null_bytes()</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="llvm-codegen_8cc_source.html#l00583">impala::LlvmCodeGen::OptimizeFunctionWithExprs()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00136">impala::HdfsScanNode::runtime_state()</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, <a class="el" href="llvm-codegen_8h_source.html#l00380">impala::LlvmCodeGen::true_value()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>, <a class="el" href="types_8h_source.html#l00060">impala::ColumnType::type</a>, <a class="el" href="descriptors_8h_source.html#l00078">impala::SlotDescriptor::type()</a>, <a class="el" href="types_8h_source.html#l00030">impala::TYPE_BOOLEAN</a>, <a class="el" href="types_8h_source.html#l00042">impala::TYPE_DECIMAL</a>, <a class="el" href="types_8h_source.html#l00033">impala::TYPE_INT</a>, <a class="el" href="types_8h_source.html#l00037">impala::TYPE_TIMESTAMP</a>, and <a class="el" href="types_8h_source.html#l00031">impala::TYPE_TINYINT</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00609">Codegen()</a>, and <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00050">impala::HdfsSequenceScanner::Codegen()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab72a9901f5c1c2cb0f053ce6ad45d9b6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::CommitRows </td> |
| <td>(</td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_rows</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Commit num_rows to the current row batch. If this completes, the row batch is enqueued with the scan node and <a class="el" href="classimpala_1_1HdfsScanner.html#a370d072e6ebd37891c012fc9da453133" title="Set batch_ to a new row batch and update tuple_mem_ accordingly. ">StartNewRowBatch()</a> is called. Returns <a class="el" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a> if the query is not cancelled and hasn't exceeded any mem limits. Scanner can call this with 0 rows to flush any pending resources (attached pools and io buffers) to minimize memory consumption. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00124">124</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scan-node_8cc_source.html#l00688">impala::HdfsScanNode::AddMaterializedRowBatch()</a>, <a class="el" href="row-batch_8h_source.html#l00120">impala::RowBatch::AtCapacity()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, <a class="el" href="descriptors_8h_source.html#l00300">impala::TupleDescriptor::byte_size()</a>, <a class="el" href="status_8h_source.html#l00088">impala::Status::CANCELLED</a>, <a class="el" href="scanner-context_8cc_source.html#l00282">impala::ScannerContext::cancelled()</a>, <a class="el" href="row-batch_8h_source.html#l00216">impala::RowBatch::capacity()</a>, <a class="el" href="runtime-state_8cc_source.html#l00286">impala::RuntimeState::CheckQueryState()</a>, <a class="el" href="row-batch_8h_source.html#l00102">impala::RowBatch::CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00154">impala::HdfsScanner::conjunct_ctxs_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="expr-context_8cc_source.html#l00109">impala::ExprContext::FreeLocalAllocations()</a>, <a class="el" href="scanner-context_8h_source.html#l00277">impala::ScannerContext::num_completed_io_buffers()</a>, <a class="el" href="row-batch_8h_source.html#l00215">impala::RowBatch::num_rows()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="scanner-context_8cc_source.html#l00045">impala::ScannerContext::ReleaseCompletedResources()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">impala::HdfsScanner::StartNewRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00180">impala::HdfsScanner::tuple_mem_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00256">impala::HdfsScanner::AttachPool()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, and <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a4ed1c34d70614ea3c693ae64a7ea2cce"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsTextScanner::CopyBoundaryField </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>data</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Prepends field data that was from the previous file buffer (This field straddled two file buffers). 'data' already contains the pointer/len from the current file buffer, boundary_column_ contains the beginning of the data from the previous file buffer. This function will allocate a new string from the tuple pool, concatenate the two pieces and update 'data' to contain the new pointer/len. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00757">757</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="mem-pool_8h_source.html#l00092">impala::MemPool::Allocate()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00145">boundary_column_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00059">impala::FieldLocation::len</a>, <a class="el" href="string-value_8h_source.html#l00037">impala::StringValue::ptr</a>, <a class="el" href="string-buffer_8h_source.html#l00081">impala::StringBuffer::Size()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00056">impala::FieldLocation::start</a>, and <a class="el" href="string-buffer_8h_source.html#l00086">impala::StringBuffer::str()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa329170edaabadb0f1ff94de121b3cec"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> <a class="el" href="impala-ir_8h.html#af2b653018219cda8c9a2d9efc0ce3f44">IR_ALWAYS_INLINE</a> impala::HdfsScanner::EvalConjuncts </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>row</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Convenience function for evaluating conjuncts using this scanner's ExprContexts. This must always be inlined so we can correctly replace the call to <a class="el" href="classimpala_1_1ExecNode.html#a48ec693b9eb748682b58479751812f08">ExecNode::EvalConjuncts()</a> during codegen. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00266">266</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00154">impala::HdfsScanner::conjunct_ctxs_</a>, and <a class="el" href="exec-node_8cc_source.html#l00393">impala::ExecNode::EvalConjuncts()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">impala::HdfsScanner::WriteCompleteTuple()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">impala::HdfsScanner::WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a023ad955932e3854826b9f35b28e69a3"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::FillByteBuffer </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> * </td> |
| <td class="paramname"><em>eosr</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_bytes</em> = <code>0</code> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Fills the next byte buffer from the context. This will block if there are no bytes ready. Updates byte_buffer_ptr_, byte_buffer_end_ and byte_buffer_read_size_. If num_bytes is 0, the scanner will read whatever is the io mgr buffer size, otherwise it will just read num_bytes. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">410</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-text-scanner_8h_source.html#l00062">byte_buffer_end_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">byte_buffer_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00065">byte_buffer_read_size_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00201">impala::HdfsScanner::decompression_type_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">impala::HdfsScanner::decompressor_</a>, <a class="el" href="scanner-context_8h_source.html#l00113">impala::ScannerContext::Stream::eosr()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="scanner-context_8cc_source.html#l00171">impala::ScannerContext::Stream::GetBuffer()</a>, <a class="el" href="scanner-context_8inline_8h_source.html#l00031">impala::ScannerContext::Stream::GetBytes()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab971b60e69d77383eb5a9935d0107247"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::FillByteBufferCompressedFile </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> * </td> |
| <td class="paramname"><em>eosr</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Fills the next byte buffer from the compressed data in stream_ by reading the entire file, decompressing it, and setting the byte_buffer_ptr_ to the decompressed buffer. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">530</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">byte_buffer_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00065">byte_buffer_read_size_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00208">impala::HdfsScanner::decompress_timer_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00201">impala::HdfsScanner::decompression_type_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">impala::HdfsScanner::decompressor_</a>, <a class="el" href="scanner-context_8h_source.html#l00113">impala::ScannerContext::Stream::eosr()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00062">impala::HdfsFileDesc::file_length</a>, <a class="el" href="scanner-context_8h_source.html#l00118">impala::ScannerContext::Stream::filename()</a>, <a class="el" href="scanner-context_8inline_8h_source.html#l00031">impala::ScannerContext::Stream::GetBytes()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00206">impala::HdfsScanNode::GetFileDesc()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="scanner-context_8cc_source.html#l00045">impala::ScannerContext::ReleaseCompletedResources()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, and <a class="el" href="logging_8h_source.html#l00058">VLOG_FILE</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab8a17657c224d4cdb37147b98945c0b6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::FillByteBufferGzip </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> * </td> |
| <td class="paramname"><em>eosr</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Fills the next byte buffer from the gzip compressed data in stream_. Unlike <a class="el" href="classimpala_1_1HdfsTextScanner.html#ab971b60e69d77383eb5a9935d0107247">FillByteBufferCompressedFile()</a>, the entire file does not need to be read at once. Buffers from stream_ are decompressed as they are read and byte_buffer_ptr_ is set to available decompressed data. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">437</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00256">impala::HdfsScanner::AttachPool()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">byte_buffer_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00065">byte_buffer_read_size_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00205">impala::HdfsScanner::data_buffer_pool_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00208">impala::HdfsScanner::decompress_timer_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">impala::HdfsScanner::decompressor_</a>, <a class="el" href="scanner-context_8h_source.html#l00113">impala::ScannerContext::Stream::eosr()</a>, <a class="el" href="scanner-context_8h_source.html#l00118">impala::ScannerContext::Stream::filename()</a>, <a class="el" href="scanner-context_8cc_source.html#l00171">impala::ScannerContext::Stream::GetBuffer()</a>, <a class="el" href="scanner-context_8inline_8h_source.html#l00031">impala::ScannerContext::Stream::GetBytes()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00049">GZIP_FIXED_READ_SIZE</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">impala::HdfsScanner::parse_status_</a>, <a class="el" href="scanner-context_8cc_source.html#l00045">impala::ScannerContext::ReleaseCompletedResources()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, <a class="el" href="scanner-context_8inline_8h_source.html#l00070">impala::ScannerContext::Stream::SkipBytes()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, and <a class="el" href="logging_8h_source.html#l00058">VLOG_FILE</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a4c9ad94b3cf268c69e691b08d575cc71"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::FindFirstTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> * </td> |
| <td class="paramname"><em>tuple_found</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Finds the start of the first tuple in this scan range and initializes byte_buffer_ptr to be the next character (the start of the first tuple). If there are no tuples starts in the entire range, *tuple_found is set to false and no more processing neesd to be done in this range (i.e. there are really large columns) </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">577</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">byte_buffer_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00065">byte_buffer_read_size_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00151">delimited_text_parser_</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="disk-io-mgr_8h_source.html#l00267">impala::DiskIoMgr::RequestRange::offset()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00180">parse_delimiter_timer_</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="scanner-context_8h_source.html#l00119">impala::ScannerContext::Stream::scan_range()</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a10766e88beda8fc0696eb27821658307"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::FinishScanRange </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Reads past the end of the scan range for the next tuple end. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">253</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00162">batch_start_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00145">boundary_column_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00142">boundary_row_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00065">byte_buffer_read_size_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="runtime-profile_8h_source.html#l00055">COUNTER_ADD</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">impala::HdfsScanner::decompressor_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00151">delimited_text_parser_</a>, <a class="el" href="string-buffer_8h_source.html#l00076">impala::StringBuffer::Empty()</a>, <a class="el" href="scanner-context_8h_source.html#l00116">impala::ScannerContext::Stream::eof()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00154">field_locations_</a>, <a class="el" href="scanner-context_8h_source.html#l00123">impala::ScannerContext::Stream::file_offset()</a>, <a class="el" href="scanner-context_8h_source.html#l00118">impala::ScannerContext::Stream::filename()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="status_8cc_source.html#l00184">impala::Status::GetDetail()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00115">impala::HdfsScanner::GetMemory()</a>, <a class="el" href="status_8h_source.html#l00174">impala::Status::IsCancelled()</a>, <a class="el" href="string-value_8h_source.html#l00038">impala::StringValue::len</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00071">NEXT_BLOCK_READ_SIZE</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00130">impala::HdfsScanNode::num_materialized_partition_keys()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00177">partial_tuple_empty_</a>, <a class="el" href="expr-benchmark_8cc_source.html#l00089">pool</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="string-value_8h_source.html#l00037">impala::StringValue::ptr</a>, <a class="el" href="exec-node_8h_source.html#l00159">impala::ExecNode::ReachedLimit()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00158">row_end_locations_</a>, <a class="el" href="scan-node_8h_source.html#l00096">impala::ScanNode::rows_read_counter()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="string-buffer_8h_source.html#l00081">impala::StringBuffer::Size()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, <a class="el" href="string-buffer_8h_source.html#l00086">impala::StringBuffer::str()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00170">impala::HdfsScanner::tuple_</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a77ca8ddf41e61984b20255881759ef99"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> * HdfsScanner::GetConjunctCtx </td> |
| <td>(</td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>idx</em></td><td>)</td> |
| <td> const</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Simple wrapper around conjunct_ctxs_. Used in the codegen'd version of <a class="el" href="classimpala_1_1HdfsScanner.html#ad6bf05f906b32b58d7df914cbd115492">WriteCompleteTuple()</a> because it's easier than writing IR to access conjunct_ctxs_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00079">79</a> of file <a class="el" href="hdfs-scanner-ir_8cc_source.html">hdfs-scanner-ir.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00154">impala::HdfsScanner::conjunct_ctxs_</a>, and <a class="el" href="gen__ir__descriptions_8py_source.html#l00215">gen_ir_descriptions::idx</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ac2a391565eba0ab7cf58a7a024032220"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::GetMemory </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> ** </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> ** </td> |
| <td class="paramname"><em>tuple_mem</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> ** </td> |
| <td class="paramname"><em>tuple_row_mem</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Gets memory for outputting tuples into batch_. *pool is the mem pool that should be used for memory allocated for those tuples. *tuple_mem should be the location to output tuples, and *tuple_row_mem for outputting tuple rows. Returns the maximum number of tuples/tuple rows that can be output (before the current row batch is complete and a new one is allocated). Memory returned from this call is invalidated after calling CommitRows. Callers must call GetMemory again after calling this function. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00115">115</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="row-batch_8h_source.html#l00100">impala::RowBatch::AddRow()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, <a class="el" href="row-batch_8h_source.html#l00216">impala::RowBatch::capacity()</a>, <a class="el" href="row-batch_8h_source.html#l00140">impala::RowBatch::GetRow()</a>, <a class="el" href="row-batch_8h_source.html#l00215">impala::RowBatch::num_rows()</a>, <a class="el" href="row-batch_8h_source.html#l00148">impala::RowBatch::tuple_data_pool()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00180">impala::HdfsScanner::tuple_mem_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="adab08ccc8e27d64a6473683eabaadb84"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::InitializeWriteTuplesFn </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a> * </td> |
| <td class="paramname"><em>partition</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">THdfsFileFormat::type </td> |
| <td class="paramname"><em>type</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const std::string & </td> |
| <td class="paramname"><em>scanner_name</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Initializes write_tuples_fn_ to the jitted function if codegen is possible.</p> |
| <ul> |
| <li>partition - partition descriptor for this scanner/scan range</li> |
| <li>type - type for this scanner</li> |
| <li>scanner_name - debug string name for this scanner (e.g. <a class="el" href="classimpala_1_1HdfsTextScanner.html">HdfsTextScanner</a>) </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00087">87</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="descriptors_8h_source.html#l00183">impala::HdfsPartitionDescriptor::escape_char()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00224">impala::HdfsScanNode::GetCodegenFn()</a>, <a class="el" href="exec-node_8h_source.html#l00154">impala::ExecNode::id()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00172">impala::HdfsScanNode::IncNumScannersCodegenDisabled()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00168">impala::HdfsScanNode::IncNumScannersCodegenEnabled()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="descriptors_8h_source.html#l00303">impala::TupleDescriptor::string_slots()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00215">impala::HdfsScanner::write_tuples_fn_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a4cebfa743efe38c408fc73b53422e851"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::InitNewRange </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Initializes this scanner for this context. The context maps to a single scan range. </p> |
| |
| <p>Implements <a class="el" href="classimpala_1_1HdfsScanner.html#a542176620ff5194866253070a539134a">impala::HdfsScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">202</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="descriptors_8h_source.html#l00182">impala::HdfsPartitionDescriptor::collection_delim()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00151">delimited_text_parser_</a>, <a class="el" href="descriptors_8h_source.html#l00183">impala::HdfsPartitionDescriptor::escape_char()</a>, <a class="el" href="descriptors_8h_source.html#l00181">impala::HdfsPartitionDescriptor::field_delim()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00067">impala::HdfsFileDesc::file_compression</a>, <a class="el" href="scanner-context_8h_source.html#l00120">impala::ScannerContext::Stream::file_desc()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00134">impala::HdfsScanNode::hdfs_table()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00160">impala::HdfsScanNode::is_materialized_col()</a>, <a class="el" href="descriptors_8h_source.html#l00180">impala::HdfsPartitionDescriptor::line_delim()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="descriptors_8h_source.html#l00233">impala::HdfsTableDescriptor::null_column_value()</a>, <a class="el" href="descriptors_8h_source.html#l00152">impala::TableDescriptor::num_cols()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00127">impala::HdfsScanNode::num_partition_keys()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="scanner-context_8h_source.html#l00278">impala::ScannerContext::partition_descriptor()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="scanner-context_8h_source.html#l00097">impala::ScannerContext::Stream::set_contains_tuple_data()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00186">impala::HdfsScanner::text_converter_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="accd7ea935f503dfe61c1570d1b9c07d8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void impala::HdfsScanner::InitTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>template_tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>tuple</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Initialize a tuple. TODO: only copy over non-null slots. TODO: InitTuple is called frequently, avoid the if, perhaps via templatization. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00355">355</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00189">impala::HdfsScanner::num_null_bytes_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00167">impala::HdfsScanner::tuple_byte_size_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">impala::HdfsScanner::WriteCompleteTuple()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af79bef6374f38e24303a824b0952dfc8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::IssueInitialRanges </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> * </td> |
| <td class="paramname"><em>scan_node</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const std::vector< <a class="el" href="structimpala_1_1HdfsFileDesc.html">HdfsFileDesc</a> * > & </td> |
| <td class="paramname"><em>files</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Issue io manager byte ranges for 'files'. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00067">67</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="classimpala_1_1HdfsScanNode.html#afe9bc42ad3846bde9f7113e5f1c133dc">impala::HdfsScanNode::AddDiskIoRanges()</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00183">impala::HdfsScanNode::AllocateScanRange()</a>, <a class="el" href="disk-io-mgr_8h_source.html#l00269">impala::DiskIoMgr::RequestRange::disk_id()</a>, <a class="el" href="disk-io-mgr_8h_source.html#l00314">impala::DiskIoMgr::ScanRange::expected_local()</a>, <a class="el" href="hdfs-lzo-text-scanner_8cc_source.html#l00060">impala::HdfsLzoTextScanner::IssueInitialRanges()</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00049">LZO_INDEX_SUFFIX</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00140">impala::HdfsScanNode::max_compressed_text_file_length()</a>, <a class="el" href="disk-io-mgr_8h_source.html#l00312">impala::DiskIoMgr::ScanRange::meta_data()</a>, <a class="el" href="disk-io-mgr_8h_source.html#l00267">impala::DiskIoMgr::RequestRange::offset()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00080">impala::ScanRangeMetadata::partition_id</a>, <a class="el" href="hdfs-scan-node_8cc_source.html#l00924">impala::HdfsScanNode::RangeComplete()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00136">impala::HdfsScanNode::runtime_state()</a>, <a class="el" href="runtime-profile_8h_source.html#l00148">impala::RuntimeProfile::HighWaterMarkCounter::Set()</a>, and <a class="el" href="disk-io-mgr_8h_source.html#l00313">impala::DiskIoMgr::ScanRange::try_cache()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scan-node_8cc_source.html#l00111">impala::HdfsScanNode::GetNext()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa54a763cd29781bd631158d72bd269ad"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsTextScanner::LogRowParseError </td> |
| <td>(</td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_idx</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">std::stringstream * </td> |
| <td class="paramname"> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Appends the current file and line to the <a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a>'s error log. row_idx is 0-based (in current batch) where the parse error occured. </p> |
| |
| <p>Reimplemented from <a class="el" href="classimpala_1_1HdfsScanner.html#adfe2965c538acda676b3a3627ff614ee">impala::HdfsScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00635">635</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-text-scanner_8h_source.html#l00162">batch_start_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00142">boundary_row_</a>, <a class="el" href="string-buffer_8h_source.html#l00076">impala::StringBuffer::Empty()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00158">row_end_locations_</a>, and <a class="el" href="string-buffer_8h_source.html#l00086">impala::StringBuffer::str()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae2cf3a0764b67bb1a1733daaa772e045"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a>* impala::HdfsScanner::next_row </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>r</em></td><td>)</td> |
| <td> const</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00368">368</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, and <a class="el" href="row-batch_8h_source.html#l00147">impala::RowBatch::row_byte_size()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00195">impala::HdfsScanner::WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a2ea24bc3f271386fe3e24336c7956ca7"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsScanner::next_tuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>t</em></td><td>)</td> |
| <td> const</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00363">363</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00167">impala::HdfsScanner::tuple_byte_size_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a370ef8b83aa2f583182c0939d9ab1e10"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::Prepare </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> * </td> |
| <td class="paramname"><em>context</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Implementation of <a class="el" href="classimpala_1_1HdfsScanner.html">HdfsScanner</a> interface. </p> |
| |
| <p>Reimplemented from <a class="el" href="classimpala_1_1HdfsScanner.html#a99165297c6e6546edbee741a54dfd194">impala::HdfsScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">620</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-profile_8h_source.html#l00051">ADD_CHILD_TIMER</a>, <a class="el" href="runtime-state_8h_source.html#l00098">impala::RuntimeState::batch_size()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00154">field_locations_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00180">parse_delimiter_timer_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00158">row_end_locations_</a>, <a class="el" href="exec-node_8h_source.html#l00161">impala::ExecNode::runtime_profile()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="scan-node_8h_source.html#l00131">impala::ScanNode::SCANNER_THREAD_TOTAL_WALLCLOCK_TIME</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a615822ff5f2cf540f07f64cfbb9ae412"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::ProcessRange </td> |
| <td>(</td> |
| <td class="paramtype">int * </td> |
| <td class="paramname"><em>num_tuples</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> </td> |
| <td class="paramname"><em>past_scan_range</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Process the entire scan range, reading bytes from context and appending materialized row batches to the scan node. *num_tuples returns the number of tuples parsed. past_scan_range is true if this is processing beyond the end of the scan range and this function should stop after finding one tuple. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">325</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="string-buffer_8h_source.html#l00044">impala::StringBuffer::Append()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00162">batch_start_ptr_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00145">boundary_column_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00142">boundary_row_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00062">byte_buffer_end_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">byte_buffer_ptr_</a>, <a class="el" href="string-buffer_8h_source.html#l00065">impala::StringBuffer::Clear()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00757">CopyBoundaryField()</a>, <a class="el" href="runtime-profile_8h_source.html#l00055">COUNTER_ADD</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00151">delimited_text_parser_</a>, <a class="el" href="string-buffer_8h_source.html#l00076">impala::StringBuffer::Empty()</a>, <a class="el" href="scanner-context_8h_source.html#l00113">impala::ScannerContext::Stream::eosr()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00154">field_locations_</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00115">impala::HdfsScanner::GetMemory()</a>, <a class="el" href="scan-node_8h_source.html#l00104">impala::ScanNode::materialize_tuple_timer()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00180">parse_delimiter_timer_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">impala::HdfsScanner::parse_status_</a>, <a class="el" href="expr-benchmark_8cc_source.html#l00089">pool</a>, <a class="el" href="exec-node_8h_source.html#l00159">impala::ExecNode::ReachedLimit()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00158">row_end_locations_</a>, <a class="el" href="scan-node_8h_source.html#l00096">impala::ScanNode::rows_read_counter()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00170">impala::HdfsScanner::tuple_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">impala::HdfsScanner::WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ad22f51e3ac67fce825aab6480477d839"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::ProcessSplit </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">virtual</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Process an entire split, reading bytes from the context's streams. Context is initialized with the split data (e.g. template tuple, partition descriptor, etc). This function should only return on error or end of scan range. </p> |
| |
| <p>Implements <a class="el" href="classimpala_1_1HdfsScanner.html#a2f5d1478537643952a179471165bf1b5">impala::HdfsScanner</a>.</p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">156</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scan-node_8h_source.html#l00067">impala::HdfsFileDesc::file_compression</a>, <a class="el" href="scanner-context_8h_source.html#l00120">impala::ScannerContext::Stream::file_desc()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00513">impala::HdfsScanner::UpdateDecompressor()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae05ee8b2f0b254228afcf6abf540f8f4"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::ReportColumnParseError </td> |
| <td>(</td> |
| <td class="paramtype">const <a class="el" href="classimpala_1_1SlotDescriptor.html">SlotDescriptor</a> * </td> |
| <td class="paramname"><em>desc</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">const char * </td> |
| <td class="paramname"><em>data</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>len</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Report parse error for column @ desc. If abort_on_error is true, sets parse_status_ to the error message. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00577">577</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="descriptors_8h_source.html#l00084">impala::SlotDescriptor::col_pos()</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00127">impala::HdfsScanNode::num_partition_keys()</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">impala::HdfsScanner::parse_status_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, and <a class="el" href="descriptors_8h_source.html#l00078">impala::SlotDescriptor::type()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="afd378fe0f2500a3a63012c7b33be4c90"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> HdfsScanner::ReportTupleParseError </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">uint8_t * </td> |
| <td class="paramname"><em>errors</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_idx</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Utility function to report parse errors for each field. If errors[i] is nonzero, fields[i] had a parse error. row_idx is the idx of the row in the current batch that had the parse error Returns false if parsing should be aborted. In this case parse_status_ is set to the error. This is called from WriteAlignedTuples. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00546">546</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="scanner-context_8h_source.html#l00118">impala::ScannerContext::Stream::filename()</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00572">impala::HdfsScanner::LogRowParseError()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00183">impala::HdfsScanner::num_errors_in_file_</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">impala::HdfsScanner::parse_status_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">impala::HdfsScanner::ReportColumnParseError()</a>, <a class="el" href="runtime-state_8cc_source.html#l00219">impala::RuntimeState::ReportFileErrors()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">impala::HdfsScanner::WriteAlignedTuples()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a3078e6f9306e5cbd2aa7cd0032e69ce8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsTextScanner::ResetScanner </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Reset the scanner. This clears any partial state that needs to be cleared when starting or when restarting after an error. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">228</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-text-scanner_8h_source.html#l00145">boundary_column_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00137">boundary_pool_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00142">boundary_row_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00062">byte_buffer_end_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">byte_buffer_ptr_</a>, <a class="el" href="descriptors_8h_source.html#l00300">impala::TupleDescriptor::byte_size()</a>, <a class="el" href="string-buffer_8h_source.html#l00065">impala::StringBuffer::Clear()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00147">impala::HdfsScanner::context_</a>, <a class="el" href="tuple_8h_source.html#l00051">impala::Tuple::Create()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00151">delimited_text_parser_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00168">error_in_row_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00087">impala::HdfsScanner::InitializeWriteTuplesFn()</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00173">partial_tuple_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00177">partial_tuple_empty_</a>, <a class="el" href="scanner-context_8h_source.html#l00278">impala::ScannerContext::partition_descriptor()</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00148">slot_idx_</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a370d072e6ebd37891c012fc9da453133"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">void HdfsScanner::StartNewRowBatch </td> |
| <td>(</td> |
| <td class="paramname"></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Set batch_ to a new row batch and update tuple_mem_ accordingly. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00108">108</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="mem-pool_8h_source.html#l00092">impala::MemPool::Allocate()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, <a class="el" href="runtime-state_8h_source.html#l00098">impala::RuntimeState::batch_size()</a>, <a class="el" href="exec-node_8h_source.html#l00162">impala::ExecNode::mem_tracker()</a>, <a class="el" href="exec-node_8h_source.html#l00156">impala::ExecNode::row_desc()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00167">impala::HdfsScanner::tuple_byte_size_</a>, <a class="el" href="row-batch_8h_source.html#l00148">impala::RowBatch::tuple_data_pool()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00180">impala::HdfsScanner::tuple_mem_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab7c78e78c88f81f7ff9d87ed12eece4e"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> HdfsScanner::UpdateDecompressor </td> |
| <td>(</td> |
| <td class="paramtype">const THdfsCompression::type & </td> |
| <td class="paramname"><em>compression</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Update the decompressor_ object given a compression type or codec name. Depending on the old compression type and the new one, it may close the old decompressor and/or create a new one of different type. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00513">513</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="classimpala_1_1Codec.html#a0d8e74ac086a91c1cc7dbd7fe166f307">impala::Codec::CreateDecompressor()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00205">impala::HdfsScanner::data_buffer_pool_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00201">impala::HdfsScanner::decompression_type_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00198">impala::HdfsScanner::decompressor_</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="status_8h_source.html#l00242">RETURN_IF_ERROR</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="descriptors_8h_source.html#l00303">impala::TupleDescriptor::string_slots()</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00493">impala::HdfsAvroScanner::InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">ProcessSplit()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aace11af0dd18b9d9dbf7afbc74dab1ba"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> impala::HdfsScanner::UpdateDecompressor </td> |
| <td>(</td> |
| <td class="paramtype">const std::string & </td> |
| <td class="paramname"><em>codec</em></td><td>)</td> |
| <td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ac95a9d24d8da6d038c93b177c9c48e6b"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::WriteAlignedTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row_mem</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_size</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>max_added_tuples</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>slots_per_tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>row_start_indx</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Processes batches of fields and writes them out to tuple_row_mem.</p> |
| <ul> |
| <li>'pool' mempool to allocate from for auxiliary tuple memory</li> |
| <li>'tuple_row_mem' preallocated tuple_row memory this function must use.</li> |
| <li>'fields' must start at the beginning of a tuple.</li> |
| <li>'num_tuples' number of tuples to process</li> |
| <li>'max_added_tuples' the maximum number of tuples that should be added to the batch.</li> |
| <li>'row_start_index' is the number of rows that have already been processed as part of WritePartialTuple. Returns the number of tuples added to the row batch. This can be less than num_tuples/tuples_till_limit because of failed conjuncts. Returns -1 if parsing should be aborted due to parse errors. </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">33</a> of file <a class="el" href="hdfs-scanner-ir_8cc_source.html">hdfs-scanner-ir.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">impala::HdfsScanner::template_tuple_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00170">impala::HdfsScanner::tuple_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00167">impala::HdfsScanner::tuple_byte_size_</a>, <a class="el" href="compiler-util_8h_source.html#l00033">UNLIKELY</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00217">impala::HdfsScanner::WriteCompleteTuple()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ad6bf05f906b32b58d7df914cbd115492"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> HdfsScanner::WriteCompleteTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a> * </td> |
| <td class="paramname"><em>template_tuple</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">uint8_t * </td> |
| <td class="paramname"><em>error_fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">uint8_t * </td> |
| <td class="paramname"><em>error_in_row</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Writes out all slots for 'tuple' from 'fields'. 'fields' must be aligned to the start of the tuple (e.g. fields[0] maps to slots[0]). After writing the tuple, it will be evaluated against the conjuncts.</p> |
| <ul> |
| <li>error_fields is an out array. error_fields[i] will be set to true if the ith field had a parse error</li> |
| <li>error_in_row is an out bool. It is set to true if any field had parse errors Returns whether the resulting tuplerow passed the conjuncts. The parsing of the fields and evaluating against conjuncts is combined in this function. This is done so it can be possible to evaluate conjuncts as slots are materialized (on partial tuples). This function is replaced by a codegen'd function at runtime. This is the reason that the out error parameters are typed uint8_t instead of bool. We need to be able to match this function's signature identically for the codegen'd function. Bool's as out parameters can get converted to bytes by the compiler and rather than implicitly depending on that to happen, we will explicitly type them to bytes. TODO: revisit this </li> |
| </ul> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00217">217</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00266">impala::HdfsScanner::EvalConjuncts()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00355">impala::HdfsScanner::InitTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00059">impala::FieldLocation::len</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00186">impala::HdfsScanner::text_converter_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>, and <a class="el" href="compiler-util_8h_source.html#l00033">UNLIKELY</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">impala::HdfsScanner::WriteAlignedTuples()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af5a25b22a0fb0b5bea62602fe224b021"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::WriteEmptyTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a> * </td> |
| <td class="paramname"><em>row_batch</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Utility method to write out tuples when there are no materialized fields (e.g. select count(*) or only partition keys). num_tuples - Total number of tuples to write out. Returns the number of tuples added to the row batch. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00157">157</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="row-batch_8h_source.html#l00100">impala::RowBatch::AddRow()</a>, <a class="el" href="row-batch_8h_source.html#l00094">impala::RowBatch::AddRows()</a>, <a class="el" href="row-batch_8h_source.html#l00120">impala::RowBatch::AtCapacity()</a>, <a class="el" href="row-batch_8h_source.html#l00216">impala::RowBatch::capacity()</a>, <a class="el" href="row-batch_8h_source.html#l00109">impala::RowBatch::CommitLastRow()</a>, <a class="el" href="row-batch_8h_source.html#l00102">impala::RowBatch::CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00266">impala::HdfsScanner::EvalConjuncts()</a>, <a class="el" href="row-batch_8h_source.html#l00140">impala::RowBatch::GetRow()</a>, <a class="el" href="row-batch_8h_source.html#l00087">impala::RowBatch::INVALID_ROW_INDEX</a>, <a class="el" href="row-batch_8h_source.html#l00215">impala::RowBatch::num_rows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">impala::HdfsScanner::template_tuple_</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae748aa64d908ff8a1fdcf3ae942ee51c"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsScanner::WriteEmptyTuples </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a> * </td> |
| <td class="paramname"><em>context</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Write empty tuples and commit them to the context object. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8cc_source.html#l00195">195</a> of file <a class="el" href="hdfs-scanner_8cc_source.html">hdfs-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00266">impala::HdfsScanner::EvalConjuncts()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00368">impala::HdfsScanner::next_row()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">impala::HdfsScanner::template_tuple_</a>, and <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aacd8f4c470ed895a8edca42753249cc1"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsTextScanner::WriteFields </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1MemPool.html">MemPool</a> * </td> |
| <td class="paramname"><em>pool</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classimpala_1_1TupleRow.html">TupleRow</a> * </td> |
| <td class="paramname"><em>tuple_row_mem</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_tuples</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Writes the intermediate parsed data into slots, outputting tuples to row_batch as they complete. Input Parameters: mempool: <a class="el" href="classimpala_1_1MemPool.html">MemPool</a> to allocate from for field data num_fields: Total number of fields contained in parsed_data_ num_tuples: Number of tuples in parsed_data_. This includes the potential partial tuple at the beginning of 'field_locations_'. Returns the number of tuples added to the row batch. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">663</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="runtime-state_8h_source.html#l00099">impala::RuntimeState::abort_on_error()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00177">impala::HdfsScanner::batch_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00142">boundary_row_</a>, <a class="el" href="descriptors_8h_source.html#l00300">impala::TupleDescriptor::byte_size()</a>, <a class="el" href="string-buffer_8h_source.html#l00065">impala::StringBuffer::Clear()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00168">error_in_row_</a>, <a class="el" href="runtime-state_8cc_source.html#l00203">impala::RuntimeState::ErrorLog()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00266">impala::HdfsScanner::EvalConjuncts()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00154">field_locations_</a>, <a class="el" href="scanner-context_8h_source.html#l00118">impala::ScannerContext::Stream::filename()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00355">impala::HdfsScanner::InitTuple()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00117">impala::HdfsScanNode::limit()</a>, <a class="el" href="runtime-state_8cc_source.html#l00224">impala::RuntimeState::LogError()</a>, <a class="el" href="runtime-state_8h_source.html#l00211">impala::RuntimeState::LogHasSpace()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00635">LogRowParseError()</a>, <a class="el" href="scan-node_8h_source.html#l00104">impala::ScanNode::materialize_tuple_timer()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00368">impala::HdfsScanner::next_row()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00363">impala::HdfsScanner::next_tuple()</a>, <a class="el" href="status_8h_source.html#l00172">impala::Status::ok()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00195">impala::HdfsScanner::parse_status_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00173">partial_tuple_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00177">partial_tuple_empty_</a>, <a class="el" href="row-batch_8h_source.html#l00147">impala::RowBatch::row_byte_size()</a>, <a class="el" href="exec-node_8h_source.html#l00157">impala::ExecNode::rows_returned()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="runtime-profile_8h_source.html#l00053">SCOPED_TIMER</a>, <a class="el" href="tuple-row_8h_source.html#l00034">impala::TupleRow::SetTuple()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00148">slot_idx_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00144">impala::HdfsScanner::state_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00150">impala::HdfsScanner::stream_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00164">impala::HdfsScanner::template_tuple_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00170">impala::HdfsScanner::tuple_</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00132">impala::HdfsScanNode::tuple_desc()</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00124">impala::HdfsScanNode::tuple_idx()</a>, <a class="el" href="compiler-util_8h_source.html#l00033">UNLIKELY</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00215">impala::HdfsScanner::write_tuples_fn_</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">impala::HdfsScanner::WriteAlignedTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a9450d62287c2c03d6ef7b1b7df9c604c"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int HdfsTextScanner::WritePartialTuple </td> |
| <td>(</td> |
| <td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> * </td> |
| <td class="paramname"><em>fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype">int </td> |
| <td class="paramname"><em>num_fields</em>, </td> |
| </tr> |
| <tr> |
| <td class="paramkey"></td> |
| <td></td> |
| <td class="paramtype"><a class="el" href="classbool.html">bool</a> </td> |
| <td class="paramname"><em>copy_strings</em> </td> |
| </tr> |
| <tr> |
| <td></td> |
| <td>)</td> |
| <td></td><td></td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Utility function to write out 'num_fields' to 'tuple_'. This is used to parse partial tuples. Returns bytes processed. If copy_strings is true, strings from fields will be copied into the boundary pool. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">768</a> of file <a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a>.</p> |
| |
| <p>References <a class="el" href="hdfs-scanner_8h_source.html#l00205">impala::HdfsScanner::data_buffer_pool_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00168">error_in_row_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00059">impala::FieldLocation::len</a>, <a class="el" href="hdfs-scan-node_8h_source.html#l00119">impala::HdfsScanNode::materialized_slots()</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00173">partial_tuple_</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">impala::HdfsScanner::ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00141">impala::HdfsScanner::scan_node_</a>, <a class="el" href="hdfs-text-scanner_8h_source.html#l00148">slot_idx_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00056">impala::FieldLocation::start</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00186">impala::HdfsScanner::text_converter_</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <h2 class="groupheader">Member Data Documentation</h2> |
| <a class="anchor" id="a5c6954a13e4a285d2e4b544574c03b76"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RowBatch.html">RowBatch</a>* impala::HdfsScanner::batch_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>The current row batch being populated. Creating new row batches, attaching context resources, and handing off to the scan node is handled by this class in <a class="el" href="classimpala_1_1HdfsScanner.html#ab72a9901f5c1c2cb0f053ce6ad45d9b6">CommitRows()</a>, but <a class="el" href="classimpala_1_1HdfsScanner.html#a94b437624601d6e7626e9b3d59cd87da">AttachPool()</a> must be called by scanner subclasses to attach any memory allocated by that subclass. All row batches created by this class are transferred to the scan node (i.e., all batches are ultimately owned by the scan node). </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00177">177</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00145">impala::HdfsScanner::AddFinalRowBatch()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00256">impala::HdfsScanner::AttachPool()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00115">impala::HdfsScanner::GetMemory()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00368">impala::HdfsScanner::next_row()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">impala::HdfsScanner::StartNewRowBatch()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00067">impala::HdfsScanner::~HdfsScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a296803327e3e709428a6a3dc6c75ac64"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">char* impala::HdfsTextScanner::batch_start_ptr_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Pointer into byte_buffer that is the start of the current batch being processed. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00162">162</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00635">LogRowParseError()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a0fe0caecdbe8e8af0ee793213f226608"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1StringBuffer.html">StringBuffer</a> impala::HdfsTextScanner::boundary_column_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Helper string for dealing with columns that span file blocks. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00145">145</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00757">CopyBoundaryField()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a11e34b1126e3e884ff50c74e89ed79ff"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1MemPool.html">MemPool</a>> impala::HdfsTextScanner::boundary_pool_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Mem pool for boundary_row_ and boundary_column_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00137">137</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a2c68a3e019f0700490a7e6ed5a05dc75"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1StringBuffer.html">StringBuffer</a> impala::HdfsTextScanner::boundary_row_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Helper string for dealing with input rows that span file blocks. We keep track of a whole line that spans file blocks to be able to report the line as erroneous in case of parsing errors. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00142">142</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00635">LogRowParseError()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a2c16649190b8816c77c0acf1a0a205ae"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">char* impala::HdfsTextScanner::byte_buffer_end_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Ending position of HDFS buffer. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00062">62</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a0f2cf6306114cd79fcd2a4fa26084275"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">char* impala::HdfsTextScanner::byte_buffer_ptr_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Current position in byte buffer. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00059">59</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a4ec6d690e88c717ecfc947dea843ca47"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int64_t impala::HdfsTextScanner::byte_buffer_read_size_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Actual bytes received from last file read. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00065">65</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="afb3a3d0ceb454e1ea3aa549786e45fac"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">std::vector<<a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a>*> impala::HdfsScanner::conjunct_ctxs_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p><a class="el" href="classimpala_1_1ExprContext.html">ExprContext</a> for each conjunct. Each scanner has its own ExprContexts so the conjuncts can be safely evaluated in parallel. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00154">154</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00082">impala::HdfsScanner::Close()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00266">impala::HdfsScanner::EvalConjuncts()</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00079">impala::HdfsScanner::GetConjunctCtx()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a6c1f7899de6ac5baca6914640f4859f4"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1ScannerContext.html">ScannerContext</a>* impala::HdfsScanner::context_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Context for this scanner. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00147">147</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00145">impala::HdfsScanner::AddFinalRowBatch()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01093">impala::HdfsParquetScanner::InitColumns()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ae86f3019c271de0d0593204003ee1653"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1MemPool.html">MemPool</a>> impala::HdfsScanner::data_buffer_pool_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Pool to allocate per data block memory. This should be used with the decompressor and any other per data block allocations. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00205">205</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00254">impala::HdfsRCFileScanner::ReadRowGroup()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00231">impala::HdfsRCFileScanner::ResetRowGroup()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00513">impala::HdfsScanner::UpdateDecompressor()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a7bd58a77c6acecf09cbfcc9b4e592618"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a>* impala::HdfsScanner::decompress_timer_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Time spent decompressing bytes. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00208">208</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00532">impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage()</a>, and <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aba0c7e8797db0a61444e50f342c775c6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">THdfsCompression::type impala::HdfsScanner::decompression_type_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The most recently used decompression type. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00201">201</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00513">impala::HdfsScanner::UpdateDecompressor()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8d995c8326def52de037ecf074fc2654"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1Codec.html">Codec</a>> impala::HdfsScanner::decompressor_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Decompressor class to use, if any. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00198">198</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00082">impala::HdfsScanner::Close()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00068">impala::HdfsRCFileScanner::InitNewRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00513">impala::HdfsScanner::UpdateDecompressor()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a35beeba40f0e22c352a00863db60a3f5"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1DelimitedTextParser.html">DelimitedTextParser</a>> impala::HdfsTextScanner::delimited_text_parser_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Helper class for picking fields and rows from delimited text. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00151">151</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ad7fa312f29fbe765c3c81729d94d7e80"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> impala::HdfsTextScanner::error_in_row_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Whether or not there was a parse error in the current row. Used for counting the number of errors per file. Once the error log is full, error_in_row will still be set, in order to be able to record the errors per file, even if the details are not logged. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00168">168</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a293f2bde8183f6eee3402ec56248b9d8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">std::vector<<a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a>> impala::HdfsTextScanner::field_locations_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Return field locations from the Delimited Text Parser. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00154">154</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">Prepare()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a0df2c19265535245917d3bb3b6095ba7"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int impala::HdfsScanner::FILE_BLOCK_SIZE = 4096</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Assumed size of an OS file block. Used mostly when reading file format headers, etc. This probably ought to be a derived number from the environment. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00095">95</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a65fed1adb4fde1c960b9ae522ba20aaa"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const char * HdfsTextScanner::LLVM_CLASS_NAME = "class.impala::HdfsTextScanner"</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00051">51</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a4e11f799fcf9fd729da778e8167bf74a"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const string HdfsTextScanner::LZO_INDEX_SUFFIX = ".index"</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Suffix for lzo index files. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00049">49</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00067">IssueInitialRanges()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ab3ed30a9bc70a9354b39d7855a871249"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">const int impala::HdfsTextScanner::NEXT_BLOCK_READ_SIZE = 1024</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">static</span><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00071">71</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a06581f363b634869efc5d09839bc3e25"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int impala::HdfsScanner::num_errors_in_file_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>number of errors in current file </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00183">183</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a922115776b26544df5d86ad82dff87ef"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int32_t impala::HdfsScanner::num_null_bytes_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Number of null bytes in the tuple. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00189">189</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8h_source.html#l00355">impala::HdfsScanner::InitTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a91b85de27fc7842c0b8fe55a59a0b766"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> impala::HdfsTextScanner::only_parsing_header_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>True if we are parsing the header for this scanner. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00068">68</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a3d58a37c969e8483a458cb48d7ab40f9"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a>* impala::HdfsTextScanner::parse_delimiter_timer_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Time parsing text files. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00180">180</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">Prepare()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8196c5dc7e842622df7ab43b8cd68397"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> impala::HdfsScanner::parse_status_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Contains current parse status to minimize the number of <a class="el" href="classimpala_1_1Status.html">Status</a> objects returned. This significantly minimizes the cross compile dependencies for llvm since status objects inline a bunch of string functions. Also, status objects aren't extremely cheap to create and destroy. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00195">195</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00125">impala::HdfsAvroScanner::ParseMetadata()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00419">impala::HdfsSequenceScanner::ReadBlockHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00099">impala::HdfsAvroScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00345">impala::HdfsSequenceScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00107">impala::HdfsRCFileScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00197">impala::HdfsRCFileScanner::ReadNumColumnsMetadata()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00278">impala::HdfsRCFileScanner::ReadRowGroupHeader()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00170">impala::BaseSequenceScanner::ReadSync()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">impala::HdfsScanner::ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00212">impala::BaseSequenceScanner::SkipToSync()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="ad61234b712a93ddbe385c8040ea809d6"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsTextScanner::partial_tuple_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Memory to store partial tuples split across buffers. Memory comes from boundary_pool_. There is only one tuple allocated for this object and reused for boundary tuples. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00173">173</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a26e56274c0ad92d7b9a76010f15887a3"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classbool.html">bool</a> impala::HdfsTextScanner::partial_tuple_empty_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>If false, there is a tuple that is partially materialized (i.e. partial_tuple_ contains data) </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00177">177</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af5a1a7043e1742e1ac83b221b09a0342"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">std::vector<char*> impala::HdfsTextScanner::row_end_locations_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>Pointers into 'byte_buffer_' for the end ptr locations for each row processed in the current batch. Used to report row errors. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00158">158</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00635">LogRowParseError()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">Prepare()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a1b361d5cc69ef26b606349a26c1a8415"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a>* impala::HdfsScanner::scan_node_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The scan node that started this scanner. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00141">141</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00145">impala::HdfsScanner::AddFinalRowBatch()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00240">impala::HdfsParquetScanner::BaseColumnReader::BaseColumnReader()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00449">impala::HdfsParquetScanner::Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00274">impala::BaseSequenceScanner::CloseFileRanges()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01050">impala::HdfsParquetScanner::CreateColumnReaders()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00472">impala::HdfsParquetScanner::CreateReader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00564">impala::HdfsRCFileScanner::DebugString()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01093">impala::HdfsParquetScanner::InitColumns()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00087">impala::HdfsScanner::InitializeWriteTuplesFn()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00493">impala::HdfsAvroScanner::InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00068">impala::HdfsRCFileScanner::InitNewRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00125">impala::HdfsAvroScanner::ParseMetadata()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">Prepare()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00074">impala::BaseSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00440">impala::HdfsParquetScanner::Prepare()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00088">impala::HdfsSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00060">impala::HdfsRCFileScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00532">impala::HdfsParquetScanner::BaseColumnReader::ReadDataPage()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00254">impala::HdfsRCFileScanner::ReadRowGroup()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">impala::HdfsScanner::ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00220">impala::HdfsAvroScanner::ResolveSchemas()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">impala::HdfsScanner::StartNewRowBatch()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00513">impala::HdfsScanner::UpdateDecompressor()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00426">impala::HdfsAvroScanner::VerifyTypesMatch()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">impala::HdfsScanner::WriteCompleteTuple()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">impala::HdfsScanner::WriteEmptyTuples()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a5847845a7ec2240820018a5dfc64e75a"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int impala::HdfsTextScanner::slot_idx_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">private</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Index into materialized_slots_ for the next slot to output for the current tuple. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-text-scanner_8h_source.html#l00148">148</a> of file <a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00228">ResetScanner()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8d1c8ea0d46e9378c6db2c24395a3f78"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* impala::HdfsScanner::state_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p><a class="el" href="classimpala_1_1RuntimeState.html">RuntimeState</a> for error reporting. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00144">144</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00082">impala::HdfsScanner::Close()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00620">Prepare()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00088">impala::HdfsSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00284">impala::BaseSequenceScanner::ReadPastSize()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00254">impala::HdfsRCFileScanner::ReadRowGroup()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00577">impala::HdfsScanner::ReportColumnParseError()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00220">impala::HdfsAvroScanner::ResolveSchemas()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">impala::HdfsScanner::StartNewRowBatch()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01290">impala::HdfsParquetScanner::ValidateColumn()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aa539050780939fc85267c63e9f262e05"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1ScannerContext_1_1Stream.html">ScannerContext::Stream</a>* impala::HdfsScanner::stream_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The first stream for context_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00150">150</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00185">Close()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00082">impala::BaseSequenceScanner::Close()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01050">impala::HdfsParquetScanner::CreateColumnReaders()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00564">impala::HdfsRCFileScanner::DebugString()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00410">FillByteBuffer()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00530">FillByteBufferCompressedFile()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00437">FillByteBufferGzip()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00577">FindFirstTuple()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00103">impala::HdfsSequenceScanner::GetRecord()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00068">impala::HdfsRCFileScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00368">impala::HdfsRCFileScanner::NextField()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00125">impala::HdfsAvroScanner::ParseMetadata()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00074">impala::BaseSequenceScanner::Prepare()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00161">impala::HdfsSequenceScanner::ProcessBlockCompressedScanRange()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00916">impala::HdfsParquetScanner::ProcessFooter()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00517">impala::HdfsAvroScanner::ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00156">ProcessSplit()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00100">impala::BaseSequenceScanner::ProcessSplit()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00769">impala::HdfsParquetScanner::ProcessSplit()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00419">impala::HdfsSequenceScanner::ReadBlockHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00413">impala::HdfsRCFileScanner::ReadColumnBuffers()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00441">impala::HdfsSequenceScanner::ReadCompressedBlock()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00099">impala::HdfsAvroScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00345">impala::HdfsSequenceScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00107">impala::HdfsRCFileScanner::ReadFileHeader()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00308">impala::HdfsRCFileScanner::ReadKeyBuffers()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00197">impala::HdfsRCFileScanner::ReadNumColumnsMetadata()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00278">impala::HdfsRCFileScanner::ReadRowGroupHeader()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00170">impala::BaseSequenceScanner::ReadSync()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00546">impala::HdfsScanner::ReportTupleParseError()</a>, <a class="el" href="base-sequence-scanner_8cc_source.html#l00212">impala::BaseSequenceScanner::SkipToSync()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01263">impala::HdfsParquetScanner::ValidateFileMetadata()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00426">impala::HdfsAvroScanner::VerifyTypesMatch()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="af1088c4f13acd1a6374c2693041f372e"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsScanner::template_tuple_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| <p>A partially materialized tuple with only partition key slots set. The non-partition key slots are set to NULL. The template tuple must be copied into tuple_ before any of the other slots are materialized. Pointer is NULL if there are no partition key slots. This template tuple is computed once for each file and valid for the duration of that file. It is owned by the HDFS scan node. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00164">164</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00093">impala::HdfsAvroScanner::AllocateFileHeader()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l01050">impala::HdfsParquetScanner::CreateColumnReaders()</a>, <a class="el" href="hdfs-avro-scanner-ir_8cc_source.html#l00023">impala::HdfsAvroScanner::DecodeAvroData()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00493">impala::HdfsAvroScanner::InitNewRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00071">impala::HdfsScanner::Prepare()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-avro-scanner_8cc_source.html#l00220">impala::HdfsAvroScanner::ResolveSchemas()</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">impala::HdfsScanner::WriteAlignedTuples()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00157">impala::HdfsScanner::WriteEmptyTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aebc25f28c691efb3b377e2cf7bc728a8"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">boost::scoped_ptr<<a class="el" href="classimpala_1_1TextConverter.html">TextConverter</a>> impala::HdfsScanner::text_converter_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Helper class for converting text to other types;. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00186">186</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00202">InitNewRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00061">impala::HdfsSequenceScanner::InitNewRange()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00060">impala::HdfsRCFileScanner::Prepare()</a>, <a class="el" href="hdfs-rcfile-scanner_8cc_source.html#l00451">impala::HdfsRCFileScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00217">impala::HdfsScanner::WriteCompleteTuple()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00768">WritePartialTuple()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a0f92b70b3120fbf0b5ed833d26176c39"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1Tuple.html">Tuple</a>* impala::HdfsScanner::tuple_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Current tuple pointer into tuple_mem_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00170">170</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-text-scanner_8cc_source.html#l00253">FinishScanRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, <a class="el" href="hdfs-text-scanner_8cc_source.html#l00325">ProcessRange()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00271">impala::HdfsSequenceScanner::ProcessRange()</a>, <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">impala::HdfsScanner::WriteAlignedTuples()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a80ed99fe7814a4047c9220d31fc57536"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">int impala::HdfsScanner::tuple_byte_size_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Fixed size of each tuple, in bytes. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00167">167</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-parquet-scanner_8cc_source.html#l00805">impala::HdfsParquetScanner::AssembleRows()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00355">impala::HdfsScanner::InitTuple()</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00363">impala::HdfsScanner::next_tuple()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00108">impala::HdfsScanner::StartNewRowBatch()</a>, and <a class="el" href="hdfs-scanner-ir_8cc_source.html#l00033">impala::HdfsScanner::WriteAlignedTuples()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="a8b0b36b42ab140cbb793aa986683c432"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname">uint8_t* impala::HdfsScanner::tuple_mem_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>The tuple memory of batch_. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00180">180</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00124">impala::HdfsScanner::CommitRows()</a>, <a class="el" href="hdfs-scanner_8cc_source.html#l00115">impala::HdfsScanner::GetMemory()</a>, and <a class="el" href="hdfs-scanner_8cc_source.html#l00108">impala::HdfsScanner::StartNewRowBatch()</a>.</p> |
| |
| </div> |
| </div> |
| <a class="anchor" id="aea086d8b6a0604df26216cacea3feeed"></a> |
| <div class="memitem"> |
| <div class="memproto"> |
| <table class="mlabels"> |
| <tr> |
| <td class="mlabels-left"> |
| <table class="memname"> |
| <tr> |
| <td class="memname"><a class="el" href="classimpala_1_1HdfsScanner.html#a1875e5409fedbadf979001ef761ae753">WriteTuplesFn</a> impala::HdfsScanner::write_tuples_fn_</td> |
| </tr> |
| </table> |
| </td> |
| <td class="mlabels-right"> |
| <span class="mlabels"><span class="mlabel">protected</span><span class="mlabel">inherited</span></span> </td> |
| </tr> |
| </table> |
| </div><div class="memdoc"> |
| |
| <p>Jitted write tuples function pointer. Null if codegen is disabled. </p> |
| |
| <p>Definition at line <a class="el" href="hdfs-scanner_8h_source.html#l00215">215</a> of file <a class="el" href="hdfs-scanner_8h_source.html">hdfs-scanner.h</a>.</p> |
| |
| <p>Referenced by <a class="el" href="hdfs-scanner_8cc_source.html#l00087">impala::HdfsScanner::InitializeWriteTuplesFn()</a>, <a class="el" href="hdfs-sequence-scanner_8cc_source.html#l00198">impala::HdfsSequenceScanner::ProcessDecompressedBlock()</a>, and <a class="el" href="hdfs-text-scanner_8cc_source.html#l00663">WriteFields()</a>.</p> |
| |
| </div> |
| </div> |
| <hr/>The documentation for this class was generated from the following files:<ul> |
| <li>be/src/exec/<a class="el" href="hdfs-text-scanner_8h_source.html">hdfs-text-scanner.h</a></li> |
| <li>be/src/exec/<a class="el" href="hdfs-text-scanner_8cc_source.html">hdfs-text-scanner.cc</a></li> |
| </ul> |
| </div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="namespaceimpala.html">impala</a></li><li class="navelem"><a class="el" href="classimpala_1_1HdfsTextScanner.html">HdfsTextScanner</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:47 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
