Google Git
Sign in
apache / impala / refs/heads/asf-site / . / doc / html / classimpala_1_1DelimitedTextParser.html
blob: ee145bf97180dca1aa00d19bb1f5ae63f00e8253 [file] [log] [blame]
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.6"/>
<title>Impala: impala::DelimitedTextParser Class Reference</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript">
$(document).ready(initResizable);
$(window).load(resizeHeight);
</script>
<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
$(document).ready(function() { searchBox.OnSelectItem(0); });
</script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td style="padding-left: 0.5em;">
<div id="projectname">Impala
</div>
<div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div>
</td>
</tr>
</tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.8.6 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
<div id="navrow1" class="tabs">
<ul class="tablist">
<li><a href="index.html"><span>Main&#160;Page</span></a></li>
<li><a href="namespaces.html"><span>Namespaces</span></a></li>
<li class="current"><a href="annotated.html"><span>Classes</span></a></li>
<li><a href="files.html"><span>Files</span></a></li>
<li>
<div id="MSearchBox" class="MSearchBoxInactive">
<span class="left">
<img id="MSearchSelect" src="search/mag_sel.png"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
alt=""/>
<input type="text" id="MSearchField" value="Search" accesskey="S"
onfocus="searchBox.OnSearchFieldFocus(true)"
onblur="searchBox.OnSearchFieldFocus(false)"
onkeyup="searchBox.OnSearchFieldChange(event)"/>
</span><span class="right">
<a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
</span>
</div>
</li>
</ul>
</div>
<div id="navrow2" class="tabs2">
<ul class="tablist">
<li><a href="annotated.html"><span>Class&#160;List</span></a></li>
<li><a href="classes.html"><span>Class&#160;Index</span></a></li>
<li><a href="inherits.html"><span>Class&#160;Hierarchy</span></a></li>
<li><a href="functions.html"><span>Class&#160;Members</span></a></li>
</ul>
</div>
</div><!-- top -->
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
<div id="nav-sync" class="sync"></div>
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
$(document).ready(function(){initNavTree('classimpala_1_1DelimitedTextParser.html','');});
</script>
<div id="doc-content">
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Macros</a></div>
<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0"
name="MSearchResults" id="MSearchResults">
</iframe>
</div>
<div class="header">
<div class="summary">
<a href="#pub-methods">Public Member Functions</a> &#124;
<a href="#pri-methods">Private Member Functions</a> &#124;
<a href="#pri-attribs">Private Attributes</a> &#124;
<a href="classimpala_1_1DelimitedTextParser-members.html">List of all members</a> </div>
<div class="headertitle">
<div class="title">impala::DelimitedTextParser Class Reference</div> </div>
</div><!--header-->
<div class="contents">
<p><code>#include &lt;<a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>&gt;</code></p>
<div class="dynheader">
Collaboration diagram for impala::DelimitedTextParser:</div>
<div class="dyncontent">
<div class="center"><div class="zoom"><iframe scrolling="no" frameborder="0" src="classimpala_1_1DelimitedTextParser__coll__graph.svg" width="100%" height="600"><p><b>This browser is not able to show SVG: try Firefox, Chrome, Safari, or Opera instead.</b></p></iframe></div></div>
</div>
<table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pub-methods"></a>
Public Member Functions</h2></td></tr>
<tr class="memitem:a0dc08dae91fdf873d66e546019e07395"><td class="memItemLeft" align="right" valign="top">&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a0dc08dae91fdf873d66e546019e07395">DelimitedTextParser</a> (int num_cols, int num_partition_keys, const <a class="el" href="classbool.html">bool</a> *is_materialized_col, char tuple_delim, char <a class="el" href="classimpala_1_1DelimitedTextParser.html#a3d1863cc77840f2210b18b2856bdd079">field_delim_</a>= '\0', char collection_item_delim= '^', char <a class="el" href="classimpala_1_1DelimitedTextParser.html#a92cc229ac4039828ee0eaf1fef278ae5">escape_char</a>= '\0')</td></tr>
<tr class="memdesc:a0dc08dae91fdf873d66e546019e07395"><td class="mdescLeft">&#160;</td><td class="mdescRight">num_cols is the total number of columns including partition keys. <a href="#a0dc08dae91fdf873d66e546019e07395">More...</a><br/></td></tr>
<tr class="separator:a0dc08dae91fdf873d66e546019e07395"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:af1b586109f27c8284758f7c99151b93b"><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#af1b586109f27c8284758f7c99151b93b">ParserReset</a> ()</td></tr>
<tr class="memdesc:af1b586109f27c8284758f7c99151b93b"><td class="mdescLeft">&#160;</td><td class="mdescRight">Called to initialize parser at beginning of scan range. <a href="#af1b586109f27c8284758f7c99151b93b">More...</a><br/></td></tr>
<tr class="separator:af1b586109f27c8284758f7c99151b93b"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ad0894acfbfbda82d27f36c52ba49b0ed"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#ad0894acfbfbda82d27f36c52ba49b0ed">AtTupleStart</a> ()</td></tr>
<tr class="memdesc:ad0894acfbfbda82d27f36c52ba49b0ed"><td class="mdescLeft">&#160;</td><td class="mdescRight">Check if we are at the start of a tuple. <a href="#ad0894acfbfbda82d27f36c52ba49b0ed">More...</a><br/></td></tr>
<tr class="separator:ad0894acfbfbda82d27f36c52ba49b0ed"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a92cc229ac4039828ee0eaf1fef278ae5"><td class="memItemLeft" align="right" valign="top">char&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a92cc229ac4039828ee0eaf1fef278ae5">escape_char</a> () const </td></tr>
<tr class="separator:a92cc229ac4039828ee0eaf1fef278ae5"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a5c0e2f5d09e4871280dbfd3bdf823718"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classimpala_1_1Status.html">Status</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a5c0e2f5d09e4871280dbfd3bdf823718">ParseFieldLocations</a> (int max_tuples, int64_t remaining_len, char **byte_buffer_ptr, char **row_end_locations, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *field_locations, int *num_tuples, int *num_fields, char **next_column_start)</td></tr>
<tr class="separator:a5c0e2f5d09e4871280dbfd3bdf823718"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:aec738aa24e13292397979b1ad44d9db4"><td class="memTemplParams" colspan="2">template&lt;bool process_escapes&gt; </td></tr>
<tr class="memitem:aec738aa24e13292397979b1ad44d9db4"><td class="memTemplItemLeft" align="right" valign="top">void&#160;</td><td class="memTemplItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#aec738aa24e13292397979b1ad44d9db4">ParseSingleTuple</a> (int64_t len, char *buffer, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *field_locations, int *num_fields)</td></tr>
<tr class="memdesc:aec738aa24e13292397979b1ad44d9db4"><td class="mdescLeft">&#160;</td><td class="mdescRight">Simplified version of ParseSSE which does not handle tuple delimiters. <a href="#aec738aa24e13292397979b1ad44d9db4">More...</a><br/></td></tr>
<tr class="separator:aec738aa24e13292397979b1ad44d9db4"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a46a04d3e77bd8430cb14647922d0da97"><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a46a04d3e77bd8430cb14647922d0da97">FindFirstInstance</a> (const char *buffer, int len)</td></tr>
<tr class="separator:a46a04d3e77bd8430cb14647922d0da97"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ade41cc665ee376667a0152cec0a10a33"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#ade41cc665ee376667a0152cec0a10a33">ReturnCurrentColumn</a> () const </td></tr>
<tr class="separator:ade41cc665ee376667a0152cec0a10a33"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:aa8fc6153c6c7e09ff2743fa489d5def3"><td class="memTemplParams" colspan="2">template&lt;bool process_escapes&gt; </td></tr>
<tr class="memitem:aa8fc6153c6c7e09ff2743fa489d5def3"><td class="memTemplItemLeft" align="right" valign="top">void&#160;</td><td class="memTemplItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#aa8fc6153c6c7e09ff2743fa489d5def3">FillColumns</a> (int len, char **last_column, int *num_fields, <a class="el" href="structimpala_1_1FieldLocation.html">impala::FieldLocation</a> *field_locations)</td></tr>
<tr class="separator:aa8fc6153c6c7e09ff2743fa489d5def3"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ab55c8d3ff0a0d9fdb52fc9ae0449f479"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#ab55c8d3ff0a0d9fdb52fc9ae0449f479">HasUnfinishedTuple</a> ()</td></tr>
<tr class="separator:ab55c8d3ff0a0d9fdb52fc9ae0449f479"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table><table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pri-methods"></a>
Private Member Functions</h2></td></tr>
<tr class="memitem:aea7161ba2d6299ffaf4a0c7ab8fb31c3"><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#aea7161ba2d6299ffaf4a0c7ab8fb31c3">ParserInit</a> (<a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *scan_node)</td></tr>
<tr class="memdesc:aea7161ba2d6299ffaf4a0c7ab8fb31c3"><td class="mdescLeft">&#160;</td><td class="mdescRight">Initialize the parser state. <a href="#aea7161ba2d6299ffaf4a0c7ab8fb31c3">More...</a><br/></td></tr>
<tr class="separator:aea7161ba2d6299ffaf4a0c7ab8fb31c3"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a31ac7a574c187eeb5089dcabc004b3bc"><td class="memTemplParams" colspan="2">template&lt;bool process_escapes&gt; </td></tr>
<tr class="memitem:a31ac7a574c187eeb5089dcabc004b3bc"><td class="memTemplItemLeft" align="right" valign="top">void&#160;</td><td class="memTemplItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a31ac7a574c187eeb5089dcabc004b3bc">AddColumn</a> (int len, char **next_column_start, int *num_fields, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *field_locations)</td></tr>
<tr class="separator:a31ac7a574c187eeb5089dcabc004b3bc"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a6e2bf22f515e313004002279cf436677"><td class="memTemplParams" colspan="2">template&lt;bool process_escapes&gt; </td></tr>
<tr class="memitem:a6e2bf22f515e313004002279cf436677"><td class="memTemplItemLeft" align="right" valign="top">void&#160;</td><td class="memTemplItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a6e2bf22f515e313004002279cf436677">ParseSse</a> (int max_tuples, int64_t *remaining_len, char **byte_buffer_ptr, char **row_end_locations_, <a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *field_locations, int *num_tuples, int *num_fields, char **next_column_start)</td></tr>
<tr class="separator:a6e2bf22f515e313004002279cf436677"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table><table class="memberdecls">
<tr class="heading"><td colspan="2"><h2 class="groupheader"><a name="pri-attribs"></a>
Private Attributes</h2></td></tr>
<tr class="memitem:a0936a25431098faa81f5ed29ca9a0dd0"><td class="memItemLeft" align="right" valign="top">__m128i&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a0936a25431098faa81f5ed29ca9a0dd0">xmm_tuple_search_</a></td></tr>
<tr class="memdesc:a0936a25431098faa81f5ed29ca9a0dd0"><td class="mdescLeft">&#160;</td><td class="mdescRight">SSE(xmm) register containing the tuple search character. <a href="#a0936a25431098faa81f5ed29ca9a0dd0">More...</a><br/></td></tr>
<tr class="separator:a0936a25431098faa81f5ed29ca9a0dd0"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:af57576143f76df3152dc58b34cc2aee5"><td class="memItemLeft" align="right" valign="top">__m128i&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#af57576143f76df3152dc58b34cc2aee5">xmm_delim_search_</a></td></tr>
<tr class="memdesc:af57576143f76df3152dc58b34cc2aee5"><td class="mdescLeft">&#160;</td><td class="mdescRight">SSE(xmm) register containing the delimiter search character. <a href="#af57576143f76df3152dc58b34cc2aee5">More...</a><br/></td></tr>
<tr class="separator:af57576143f76df3152dc58b34cc2aee5"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ab76d3428a7bb9ad6aa1850b80f243d28"><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#ab76d3428a7bb9ad6aa1850b80f243d28">num_delims_</a></td></tr>
<tr class="memdesc:ab76d3428a7bb9ad6aa1850b80f243d28"><td class="mdescLeft">&#160;</td><td class="mdescRight">The number of delimiters contained in xmm_delim_search_, i.e. its length. <a href="#ab76d3428a7bb9ad6aa1850b80f243d28">More...</a><br/></td></tr>
<tr class="separator:ab76d3428a7bb9ad6aa1850b80f243d28"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a49637d59453e03a3ed19420b0bd65c3a"><td class="memItemLeft" align="right" valign="top">__m128i&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a49637d59453e03a3ed19420b0bd65c3a">xmm_escape_search_</a></td></tr>
<tr class="memdesc:a49637d59453e03a3ed19420b0bd65c3a"><td class="mdescLeft">&#160;</td><td class="mdescRight">SSE(xmm) register containing the escape search character. <a href="#a49637d59453e03a3ed19420b0bd65c3a">More...</a><br/></td></tr>
<tr class="separator:a49637d59453e03a3ed19420b0bd65c3a"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a3d1863cc77840f2210b18b2856bdd079"><td class="memItemLeft" align="right" valign="top">char&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a3d1863cc77840f2210b18b2856bdd079">field_delim_</a></td></tr>
<tr class="memdesc:a3d1863cc77840f2210b18b2856bdd079"><td class="mdescLeft">&#160;</td><td class="mdescRight">Character delimiting fields (to become slots). <a href="#a3d1863cc77840f2210b18b2856bdd079">More...</a><br/></td></tr>
<tr class="separator:a3d1863cc77840f2210b18b2856bdd079"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a90b891102dc9bc8e5465831ae382a5cb"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a90b891102dc9bc8e5465831ae382a5cb">process_escapes_</a></td></tr>
<tr class="memdesc:a90b891102dc9bc8e5465831ae382a5cb"><td class="mdescLeft">&#160;</td><td class="mdescRight">True if this parser should handle escape characters. <a href="#a90b891102dc9bc8e5465831ae382a5cb">More...</a><br/></td></tr>
<tr class="separator:a90b891102dc9bc8e5465831ae382a5cb"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:ae249b2c21125183c7d6144a0286ac9ef"><td class="memItemLeft" align="right" valign="top">char&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#ae249b2c21125183c7d6144a0286ac9ef">escape_char_</a></td></tr>
<tr class="memdesc:ae249b2c21125183c7d6144a0286ac9ef"><td class="mdescLeft">&#160;</td><td class="mdescRight">Escape character. Only used if process_escapes_ is true. <a href="#ae249b2c21125183c7d6144a0286ac9ef">More...</a><br/></td></tr>
<tr class="separator:ae249b2c21125183c7d6144a0286ac9ef"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a536f4958ff143714c86aa36f9fd0eafc"><td class="memItemLeft" align="right" valign="top">char&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a536f4958ff143714c86aa36f9fd0eafc">collection_item_delim_</a></td></tr>
<tr class="memdesc:a536f4958ff143714c86aa36f9fd0eafc"><td class="mdescLeft">&#160;</td><td class="mdescRight">Character delimiting collection items (to become slots). <a href="#a536f4958ff143714c86aa36f9fd0eafc">More...</a><br/></td></tr>
<tr class="separator:a536f4958ff143714c86aa36f9fd0eafc"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a5e6f5a82b0c51692fbc51b428597d0d4"><td class="memItemLeft" align="right" valign="top">char&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a5e6f5a82b0c51692fbc51b428597d0d4">tuple_delim_</a></td></tr>
<tr class="memdesc:a5e6f5a82b0c51692fbc51b428597d0d4"><td class="mdescLeft">&#160;</td><td class="mdescRight">Character delimiting tuples. <a href="#a5e6f5a82b0c51692fbc51b428597d0d4">More...</a><br/></td></tr>
<tr class="separator:a5e6f5a82b0c51692fbc51b428597d0d4"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:acacd7799b6a245c30ee25dcb34ee9810"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#acacd7799b6a245c30ee25dcb34ee9810">current_column_has_escape_</a></td></tr>
<tr class="separator:acacd7799b6a245c30ee25dcb34ee9810"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a9083e73358436888ad756c74e216a928"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a9083e73358436888ad756c74e216a928">last_char_is_escape_</a></td></tr>
<tr class="memdesc:a9083e73358436888ad756c74e216a928"><td class="mdescLeft">&#160;</td><td class="mdescRight">Whether or not the previous character was the escape character. <a href="#a9083e73358436888ad756c74e216a928">More...</a><br/></td></tr>
<tr class="separator:a9083e73358436888ad756c74e216a928"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a893aaa8252b433080c1b0415298c1ba2"><td class="memItemLeft" align="right" valign="top">int32_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a893aaa8252b433080c1b0415298c1ba2">last_row_delim_offset_</a></td></tr>
<tr class="separator:a893aaa8252b433080c1b0415298c1ba2"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a3b47697ec0f5ee137f48b4b40ffbef06"><td class="memItemLeft" align="right" valign="top">uint16_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a3b47697ec0f5ee137f48b4b40ffbef06">low_mask_</a> [16]</td></tr>
<tr class="memdesc:a3b47697ec0f5ee137f48b4b40ffbef06"><td class="mdescLeft">&#160;</td><td class="mdescRight">Precomputed masks to process escape characters. <a href="#a3b47697ec0f5ee137f48b4b40ffbef06">More...</a><br/></td></tr>
<tr class="separator:a3b47697ec0f5ee137f48b4b40ffbef06"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a7dbdea45da5c530f066969ed5dbe8f55"><td class="memItemLeft" align="right" valign="top">uint16_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a7dbdea45da5c530f066969ed5dbe8f55">high_mask_</a> [16]</td></tr>
<tr class="separator:a7dbdea45da5c530f066969ed5dbe8f55"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:aa9220fc54558ed3eec36a4958bbc8a57"><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#aa9220fc54558ed3eec36a4958bbc8a57">num_cols_</a></td></tr>
<tr class="memdesc:aa9220fc54558ed3eec36a4958bbc8a57"><td class="mdescLeft">&#160;</td><td class="mdescRight">Number of columns in the table (including partition columns) <a href="#aa9220fc54558ed3eec36a4958bbc8a57">More...</a><br/></td></tr>
<tr class="separator:aa9220fc54558ed3eec36a4958bbc8a57"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a474d30148a664e0398635f46151be2a3"><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a474d30148a664e0398635f46151be2a3">num_partition_keys_</a></td></tr>
<tr class="memdesc:a474d30148a664e0398635f46151be2a3"><td class="mdescLeft">&#160;</td><td class="mdescRight">Number of partition columns in the table. <a href="#a474d30148a664e0398635f46151be2a3">More...</a><br/></td></tr>
<tr class="separator:a474d30148a664e0398635f46151be2a3"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a46e65ae301846622a37038d7ef2b3190"><td class="memItemLeft" align="right" valign="top">const <a class="el" href="classbool.html">bool</a> *&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a46e65ae301846622a37038d7ef2b3190">is_materialized_col_</a></td></tr>
<tr class="separator:a46e65ae301846622a37038d7ef2b3190"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a7dec5b1a3046f87e59b491f84f0f69fe"><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a7dec5b1a3046f87e59b491f84f0f69fe">column_idx_</a></td></tr>
<tr class="memdesc:a7dec5b1a3046f87e59b491f84f0f69fe"><td class="mdescLeft">&#160;</td><td class="mdescRight">Index to keep track of the current column in the current file. <a href="#a7dec5b1a3046f87e59b491f84f0f69fe">More...</a><br/></td></tr>
<tr class="separator:a7dec5b1a3046f87e59b491f84f0f69fe"><td class="memSeparator" colspan="2">&#160;</td></tr>
<tr class="memitem:a163201ab3b9a50974f1fcd1b0fb9d3b5"><td class="memItemLeft" align="right" valign="top"><a class="el" href="classbool.html">bool</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classimpala_1_1DelimitedTextParser.html#a163201ab3b9a50974f1fcd1b0fb9d3b5">unfinished_tuple_</a></td></tr>
<tr class="memdesc:a163201ab3b9a50974f1fcd1b0fb9d3b5"><td class="mdescLeft">&#160;</td><td class="mdescRight">True if the last tuple is unfinished (not ended with tuple delimiter). <a href="#a163201ab3b9a50974f1fcd1b0fb9d3b5">More...</a><br/></td></tr>
<tr class="separator:a163201ab3b9a50974f1fcd1b0fb9d3b5"><td class="memSeparator" colspan="2">&#160;</td></tr>
</table>
<a name="details" id="details"></a><h2 class="groupheader">Detailed Description</h2>
<div class="textblock">
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00025">25</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
</div><h2 class="groupheader">Constructor &amp; Destructor Documentation</h2>
<a class="anchor" id="a0dc08dae91fdf873d66e546019e07395"></a>
<div class="memitem">
<div class="memproto">
<table class="memname">
<tr>
<td class="memname">DelimitedTextParser::DelimitedTextParser </td>
<td>(</td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>num_cols</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>num_partition_keys</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">const <a class="el" href="classbool.html">bool</a> *&#160;</td>
<td class="paramname"><em>is_materialized_col</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char&#160;</td>
<td class="paramname"><em>tuple_delim</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char&#160;</td>
<td class="paramname"><em>field_delim_</em> = <code>'\0'</code>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char&#160;</td>
<td class="paramname"><em>collection_item_delim</em> = <code>'^'</code>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char&#160;</td>
<td class="paramname"><em>escape_char</em> = <code>'\0'</code>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</div><div class="memdoc">
<p>num_cols is the total number of columns including partition keys. </p>
<p>The Delimited Text Parser parses text rows that are delimited by specific characters: tuple_delim: delimits tuples field_delim: delimits fields collection_item_delim: delimits collection items escape_char: escape delimiters, make them part of the data.is_materialized_col should be initialized to an array of length 'num_cols', with is_materialized_col[i] = &lt;true if column i should be materialized, false otherwise&gt; Owned by caller. The main method is ParseData which fills in a vector of pointers and lengths to the fields. It also can handle an escape character which masks a tuple or field delimiter that occurs in the data. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8cc_source.html#l00024">24</a> of file <a class="el" href="delimited-text-parser_8cc_source.html">delimited-text-parser.cc</a>.</p>
<p>References <a class="el" href="sse-util_8h_source.html#l00028">impala::SSEUtil::CHARS_PER_128_BIT_REGISTER</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00175">collection_item_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00172">escape_char_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00166">field_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00198">high_mask_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00197">low_mask_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00160">num_delims_</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00090">ParserReset()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00169">process_escapes_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00178">tuple_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00157">xmm_delim_search_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00163">xmm_escape_search_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00154">xmm_tuple_search_</a>.</p>
</div>
</div>
<h2 class="groupheader">Member Function Documentation</h2>
<a class="anchor" id="a31ac7a574c187eeb5089dcabc004b3bc"></a>
<div class="memitem">
<div class="memproto">
<div class="memtemplate">
template&lt;bool process_escapes&gt; </div>
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">void impala::DelimitedTextParser::AddColumn </td>
<td>(</td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>len</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>next_column_start</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_fields</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *&#160;</td>
<td class="paramname"><em>field_locations</em>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Helper routine to add a column to the field_locations vector. Template parameter: process_escapes &ndash; if true the the column may have escape characters and the negative of the len will be stored. len: lenght of the current column. Input/Output: next_column_start: Start of the current column, moved to the start of the next. num_fields: current number of fields processed, updated to next field. Output: field_locations: updated with start and length of current field. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00053">53</a> of file <a class="el" href="delimited-text-parser_8inline_8h_source.html">delimited-text-parser.inline.h</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00182">current_column_has_escape_</a>, <a class="el" href="hdfs-scanner_8h_source.html#l00059">impala::FieldLocation::len</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00102">ReturnCurrentColumn()</a>, and <a class="el" href="hdfs-scanner_8h_source.html#l00056">impala::FieldLocation::start</a>.</p>
</div>
</div>
<a class="anchor" id="ad0894acfbfbda82d27f36c52ba49b0ed"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::AtTupleStart </td>
<td>(</td>
<td class="paramname"></td><td>)</td>
<td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Check if we are at the start of a tuple. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00053">53</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00204">num_partition_keys_</a>.</p>
</div>
</div>
<a class="anchor" id="a92cc229ac4039828ee0eaf1fef278ae5"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">char impala::DelimitedTextParser::escape_char </td>
<td>(</td>
<td class="paramname"></td><td>)</td>
<td> const</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00055">55</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00172">escape_char_</a>.</p>
</div>
</div>
<a class="anchor" id="aa8fc6153c6c7e09ff2743fa489d5def3"></a>
<div class="memitem">
<div class="memproto">
<div class="memtemplate">
template&lt;bool process_escapes&gt; </div>
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">void impala::DelimitedTextParser::FillColumns </td>
<td>(</td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>len</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>last_column</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_fields</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">impala::FieldLocation</a> *&#160;</td>
<td class="paramname"><em>field_locations</em>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Fill in columns missing at the end of the tuple. len and last_column may contain the length and the pointer to the last column on which the file ended without a delimiter. Fills in the offsets and lengths in field_locations. If parsing stopped on a delimiter and there is no last column then len will be 0. Other columns beyond that are filled with 0 length fields. num_fields points to an initialized count of fields and will incremented by the number fields added. field_locations will be updated with the start and length of the fields. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00071">71</a> of file <a class="el" href="delimited-text-parser_8inline_8h_source.html">delimited-text-parser.inline.h</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00201">num_cols_</a>.</p>
</div>
</div>
<a class="anchor" id="a46a04d3e77bd8430cb14647922d0da97"></a>
<div class="memitem">
<div class="memproto">
<table class="memname">
<tr>
<td class="memname">int DelimitedTextParser::FindFirstInstance </td>
<td>(</td>
<td class="paramtype">const char *&#160;</td>
<td class="paramname"><em>buffer</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>len</em>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</div><div class="memdoc">
<p>FindFirstInstance returns the position after the first non-escaped tuple delimiter from the starting offset. Used to find the start of a tuple if jumping into the middle of a text file. Also used to find the sync marker for Sequenced and RC files. If no tuple delimiter is found within the buffer, return -1; </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8cc_source.html#l00194">194</a> of file <a class="el" href="delimited-text-parser_8cc_source.html">delimited-text-parser.cc</a>.</p>
<p>References <a class="el" href="sse-util_8h_source.html#l00028">impala::SSEUtil::CHARS_PER_128_BIT_REGISTER</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00172">escape_char_</a>, <a class="el" href="cpu-info_8h_source.html#l00058">impala::CpuInfo::IsSupported()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00194">last_row_delim_offset_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00169">process_escapes_</a>, <a class="el" href="cpu-info_8h_source.html#l00034">impala::CpuInfo::SSE4_2</a>, <a class="el" href="sse-util_8h_source.html#l00089">impala::SSE4_cmpestrm()</a>, <a class="el" href="sse-util_8h_source.html#l00048">impala::SSEUtil::SSE_BITMASK</a>, <a class="el" href="sse-util_8h_source.html#l00040">impala::SSEUtil::STRCHR_MODE</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00178">tuple_delim_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00154">xmm_tuple_search_</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser-test_8cc_source.html#l00025">impala::Validate()</a>.</p>
</div>
</div>
<a class="anchor" id="ab55c8d3ff0a0d9fdb52fc9ae0449f479"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::HasUnfinishedTuple </td>
<td>(</td>
<td class="paramname"></td><td>)</td>
<td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Return true if we have not seen a tuple delimiter for the current tuple being parsed (i.e., the last byte read was not a tuple delimiter). </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00121">121</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00214">unfinished_tuple_</a>.</p>
</div>
</div>
<a class="anchor" id="a5c0e2f5d09e4871280dbfd3bdf823718"></a>
<div class="memitem">
<div class="memproto">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classimpala_1_1Status.html">Status</a> DelimitedTextParser::ParseFieldLocations </td>
<td>(</td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>max_tuples</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int64_t&#160;</td>
<td class="paramname"><em>remaining_len</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>byte_buffer_ptr</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>row_end_locations</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *&#160;</td>
<td class="paramname"><em>field_locations</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_tuples</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_fields</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>next_column_start</em>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</div><div class="memdoc">
<p>Parses a byte buffer for the field and tuple breaks. This function will write the field start &amp; len to field_locations which can then be written out to tuples. This function uses SSE ("Intel x86 instruction set extension 'Streaming Simd Extension') if the hardware supports SSE4.2 instructions. SSE4.2 added string processing instructions that allow for processing 16 characters at a time. Otherwise, this function walks the file_buffer_ character by character. Input Parameters: max_tuples: The maximum number of tuples that should be parsed. This is used to control how the batching works. remaining_len: Length of data remaining in the byte_buffer_pointer. byte_buffer_pointer: Pointer to the buffer containing the data to be parsed. Output Parameters: field_locations: array of pointers to data fields and their lengths num_tuples: Number of tuples parsed num_fields: Number of materialized fields parsed next_column_start: pointer within file_buffer_ where the next field starts after the return from the call to ParseData </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8cc_source.html#l00098">98</a> of file <a class="el" href="delimited-text-parser_8cc_source.html">delimited-text-parser.cc</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00175">collection_item_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00182">current_column_has_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00172">escape_char_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00166">field_delim_</a>, <a class="el" href="cpu-info_8h_source.html#l00058">impala::CpuInfo::IsSupported()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00185">last_char_is_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00194">last_row_delim_offset_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00204">num_partition_keys_</a>, <a class="el" href="status_8h_source.html#l00087">impala::Status::OK</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00169">process_escapes_</a>, <a class="el" href="cpu-info_8h_source.html#l00034">impala::CpuInfo::SSE4_2</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00178">tuple_delim_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00214">unfinished_tuple_</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser-test_8cc_source.html#l00025">impala::Validate()</a>.</p>
</div>
</div>
<a class="anchor" id="aea7161ba2d6299ffaf4a0c7ab8fb31c3"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">void impala::DelimitedTextParser::ParserInit </td>
<td>(</td>
<td class="paramtype"><a class="el" href="classimpala_1_1HdfsScanNode.html">HdfsScanNode</a> *&#160;</td>
<td class="paramname"><em>scan_node</em></td><td>)</td>
<td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Initialize the parser state. </p>
</div>
</div>
<a class="anchor" id="af1b586109f27c8284758f7c99151b93b"></a>
<div class="memitem">
<div class="memproto">
<table class="memname">
<tr>
<td class="memname">void DelimitedTextParser::ParserReset </td>
<td>(</td>
<td class="paramname"></td><td>)</td>
<td></td>
</tr>
</table>
</div><div class="memdoc">
<p>Called to initialize parser at beginning of scan range. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8cc_source.html#l00090">90</a> of file <a class="el" href="delimited-text-parser_8cc_source.html">delimited-text-parser.cc</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00182">current_column_has_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00185">last_char_is_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00194">last_row_delim_offset_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00204">num_partition_keys_</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, and <a class="el" href="delimited-text-parser-test_8cc_source.html#l00025">impala::Validate()</a>.</p>
</div>
</div>
<a class="anchor" id="aec738aa24e13292397979b1ad44d9db4"></a>
<div class="memitem">
<div class="memproto">
<div class="memtemplate">
template&lt;bool process_escapes&gt; </div>
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">void impala::DelimitedTextParser::ParseSingleTuple </td>
<td>(</td>
<td class="paramtype">int64_t&#160;</td>
<td class="paramname"><em>len</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char *&#160;</td>
<td class="paramname"><em>buffer</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *&#160;</td>
<td class="paramname"><em>field_locations</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_fields</em>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Simplified version of ParseSSE which does not handle tuple delimiters. </p>
<p>Parse a single tuple from buffer.</p>
<ul>
<li>buffer/len are input parameters for the entire record.</li>
<li>on return field_locations will contain the start/len for each materialized col.</li>
<li>*num_fields returns the number of fields processed. This function is used to parse sequence file records which do not need to parse for tuple delimiters. </li>
</ul>
<p>Definition at line <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">221</a> of file <a class="el" href="delimited-text-parser_8inline_8h_source.html">delimited-text-parser.inline.h</a>.</p>
<p>References <a class="el" href="sse-util_8h_source.html#l00028">impala::SSEUtil::CHARS_PER_128_BIT_REGISTER</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00175">collection_item_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00182">current_column_has_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00172">escape_char_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00166">field_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00198">high_mask_</a>, <a class="el" href="cpu-info_8h_source.html#l00058">impala::CpuInfo::IsSupported()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00185">last_char_is_escape_</a>, <a class="el" href="compiler-util_8h_source.html#l00032">LIKELY</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00197">low_mask_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00160">num_delims_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00204">num_partition_keys_</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00028">impala::ProcessEscapeMask()</a>, <a class="el" href="cpu-info_8h_source.html#l00034">impala::CpuInfo::SSE4_2</a>, <a class="el" href="sse-util_8h_source.html#l00089">impala::SSE4_cmpestrm()</a>, <a class="el" href="sse-util_8h_source.html#l00048">impala::SSEUtil::SSE_BITMASK</a>, <a class="el" href="sse-util_8h_source.html#l00040">impala::SSEUtil::STRCHR_MODE</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00157">xmm_delim_search_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00163">xmm_escape_search_</a>.</p>
</div>
</div>
<a class="anchor" id="a6e2bf22f515e313004002279cf436677"></a>
<div class="memitem">
<div class="memproto">
<div class="memtemplate">
template&lt;bool process_escapes&gt; </div>
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">void impala::DelimitedTextParser::ParseSse </td>
<td>(</td>
<td class="paramtype">int&#160;</td>
<td class="paramname"><em>max_tuples</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int64_t *&#160;</td>
<td class="paramname"><em>remaining_len</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>byte_buffer_ptr</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>row_end_locations</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype"><a class="el" href="structimpala_1_1FieldLocation.html">FieldLocation</a> *&#160;</td>
<td class="paramname"><em>field_locations</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_tuples</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">int *&#160;</td>
<td class="paramname"><em>num_fields</em>, </td>
</tr>
<tr>
<td class="paramkey"></td>
<td></td>
<td class="paramtype">char **&#160;</td>
<td class="paramname"><em>next_column_start</em>&#160;</td>
</tr>
<tr>
<td></td>
<td>)</td>
<td></td><td></td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Helper routine to parse delimited text using SSE instructions. Identical arguments as ParseFieldLocations. If the template argument, 'process_escapes' is true, this function will handle escapes, otherwise, it will assume the text is unescaped. By using templates, we can special case the un-escaped path for better performance. The unescaped path is optimized away by the compiler.</p>
<p>SSE optimized raw text file parsing. SSE4_2 added an instruction (with 3 modes) for text processing. The modes mimic strchr, strstr and strcmp. For text parsing, we can leverage the strchr functionality. The instruction operates on two sse registers:</p>
<ul>
<li>the needle (what you are searching for)</li>
<li>the haystack (where you are searching in) Both registers can contain up to 16 characters. The result is a 16-bit mask with a bit set for each character in the haystack that matched any character in the needle. For example: Needle = 'abcd000000000000' (we're searching for any a's, b's, c's or d's) Haystack = 'asdfghjklhjbdwwc' (the raw string) Result = '1010000000011001' </li>
</ul>
<p>Definition at line <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">98</a> of file <a class="el" href="delimited-text-parser_8inline_8h_source.html">delimited-text-parser.inline.h</a>.</p>
<p>References <a class="el" href="sse-util_8h_source.html#l00028">impala::SSEUtil::CHARS_PER_128_BIT_REGISTER</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00175">collection_item_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00182">current_column_has_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00172">escape_char_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00166">field_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00198">high_mask_</a>, <a class="el" href="cpu-info_8h_source.html#l00058">impala::CpuInfo::IsSupported()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00185">last_char_is_escape_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00194">last_row_delim_offset_</a>, <a class="el" href="compiler-util_8h_source.html#l00032">LIKELY</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00197">low_mask_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00160">num_delims_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00204">num_partition_keys_</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00028">impala::ProcessEscapeMask()</a>, <a class="el" href="cpu-info_8h_source.html#l00034">impala::CpuInfo::SSE4_2</a>, <a class="el" href="sse-util_8h_source.html#l00089">impala::SSE4_cmpestrm()</a>, <a class="el" href="sse-util_8h_source.html#l00048">impala::SSEUtil::SSE_BITMASK</a>, <a class="el" href="sse-util_8h_source.html#l00040">impala::SSEUtil::STRCHR_MODE</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00178">tuple_delim_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00214">unfinished_tuple_</a>, <a class="el" href="compiler-util_8h_source.html#l00033">UNLIKELY</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00157">xmm_delim_search_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00163">xmm_escape_search_</a>.</p>
</div>
</div>
<a class="anchor" id="ade41cc665ee376667a0152cec0a10a33"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::ReturnCurrentColumn </td>
<td>(</td>
<td class="paramname"></td><td>)</td>
<td> const</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">inline</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Will we return the current column to the query? Hive allows cols at the end of the table that are not in the schema. We'll just ignore those columns </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00102">102</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>References <a class="el" href="delimited-text-parser_8h_source.html#l00211">column_idx_</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00208">is_materialized_col_</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00201">num_cols_</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00053">AddColumn()</a>.</p>
</div>
</div>
<h2 class="groupheader">Member Data Documentation</h2>
<a class="anchor" id="a536f4958ff143714c86aa36f9fd0eafc"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">char impala::DelimitedTextParser::collection_item_delim_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Character delimiting collection items (to become slots). </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00175">175</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a7dec5b1a3046f87e59b491f84f0f69fe"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">int impala::DelimitedTextParser::column_idx_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Index to keep track of the current column in the current file. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00211">211</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00053">AddColumn()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00053">AtTupleStart()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00071">FillColumns()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00090">ParserReset()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00102">ReturnCurrentColumn()</a>.</p>
</div>
</div>
<a class="anchor" id="acacd7799b6a245c30ee25dcb34ee9810"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::current_column_has_escape_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Whether or not the current column has an escape character in it (and needs to be unescaped) </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00182">182</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00053">AddColumn()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00090">ParserReset()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="ae249b2c21125183c7d6144a0286ac9ef"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">char impala::DelimitedTextParser::escape_char_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Escape character. Only used if process_escapes_ is true. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00172">172</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8h_source.html#l00055">escape_char()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00194">FindFirstInstance()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a3d1863cc77840f2210b18b2856bdd079"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">char impala::DelimitedTextParser::field_delim_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Character delimiting fields (to become slots). </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00166">166</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a7dbdea45da5c530f066969ed5dbe8f55"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">uint16_t impala::DelimitedTextParser::high_mask_[16]</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00198">198</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a46e65ae301846622a37038d7ef2b3190"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">const <a class="el" href="classbool.html">bool</a>* impala::DelimitedTextParser::is_materialized_col_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>For each col index [0, num_cols_), true if the column should be materialized. Not owned. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00208">208</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8h_source.html#l00102">ReturnCurrentColumn()</a>.</p>
</div>
</div>
<a class="anchor" id="a9083e73358436888ad756c74e216a928"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::last_char_is_escape_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Whether or not the previous character was the escape character. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00185">185</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00090">ParserReset()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a893aaa8252b433080c1b0415298c1ba2"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">int32_t impala::DelimitedTextParser::last_row_delim_offset_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Used for special processing of . This will be the offset of the last instance of from the end of the current buffer being searched unless the last row delimiter was not a in which case it will be -1. If the last character in a buffer is then the value will be 0. At the start of processing a new buffer if last_row_delim_offset_ is 0 then it is set to be one more than the size of the buffer so that if the buffer starts with <br/>
it is processed as <br/>
. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00194">194</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00194">FindFirstInstance()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00090">ParserReset()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a3b47697ec0f5ee137f48b4b40ffbef06"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">uint16_t impala::DelimitedTextParser::low_mask_[16]</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Precomputed masks to process escape characters. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00197">197</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="aa9220fc54558ed3eec36a4958bbc8a57"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">int impala::DelimitedTextParser::num_cols_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Number of columns in the table (including partition columns) </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00201">201</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00071">FillColumns()</a>, and <a class="el" href="delimited-text-parser_8h_source.html#l00102">ReturnCurrentColumn()</a>.</p>
</div>
</div>
<a class="anchor" id="ab76d3428a7bb9ad6aa1850b80f243d28"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">int impala::DelimitedTextParser::num_delims_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>The number of delimiters contained in xmm_delim_search_, i.e. its length. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00160">160</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a474d30148a664e0398635f46151be2a3"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">int impala::DelimitedTextParser::num_partition_keys_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Number of partition columns in the table. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00204">204</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8h_source.html#l00053">AtTupleStart()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00090">ParserReset()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a90b891102dc9bc8e5465831ae382a5cb"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::process_escapes_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>True if this parser should handle escape characters. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00169">169</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00194">FindFirstInstance()</a>, and <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>.</p>
</div>
</div>
<a class="anchor" id="a5e6f5a82b0c51692fbc51b428597d0d4"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">char impala::DelimitedTextParser::tuple_delim_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>Character delimiting tuples. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00178">178</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00194">FindFirstInstance()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a163201ab3b9a50974f1fcd1b0fb9d3b5"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname"><a class="el" href="classbool.html">bool</a> impala::DelimitedTextParser::unfinished_tuple_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>True if the last tuple is unfinished (not ended with tuple delimiter). </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00214">214</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8h_source.html#l00121">HasUnfinishedTuple()</a>, <a class="el" href="delimited-text-parser_8cc_source.html#l00098">ParseFieldLocations()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="af57576143f76df3152dc58b34cc2aee5"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">__m128i impala::DelimitedTextParser::xmm_delim_search_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>SSE(xmm) register containing the delimiter search character. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00157">157</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a49637d59453e03a3ed19420b0bd65c3a"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">__m128i impala::DelimitedTextParser::xmm_escape_search_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>SSE(xmm) register containing the escape search character. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00163">163</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00221">ParseSingleTuple()</a>, and <a class="el" href="delimited-text-parser_8inline_8h_source.html#l00098">ParseSse()</a>.</p>
</div>
</div>
<a class="anchor" id="a0936a25431098faa81f5ed29ca9a0dd0"></a>
<div class="memitem">
<div class="memproto">
<table class="mlabels">
<tr>
<td class="mlabels-left">
<table class="memname">
<tr>
<td class="memname">__m128i impala::DelimitedTextParser::xmm_tuple_search_</td>
</tr>
</table>
</td>
<td class="mlabels-right">
<span class="mlabels"><span class="mlabel">private</span></span> </td>
</tr>
</table>
</div><div class="memdoc">
<p>SSE(xmm) register containing the tuple search character. </p>
<p>Definition at line <a class="el" href="delimited-text-parser_8h_source.html#l00154">154</a> of file <a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a>.</p>
<p>Referenced by <a class="el" href="delimited-text-parser_8cc_source.html#l00024">DelimitedTextParser()</a>, and <a class="el" href="delimited-text-parser_8cc_source.html#l00194">FindFirstInstance()</a>.</p>
</div>
</div>
<hr/>The documentation for this class was generated from the following files:<ul>
<li>be/src/exec/<a class="el" href="delimited-text-parser_8h_source.html">delimited-text-parser.h</a></li>
<li>be/src/exec/<a class="el" href="delimited-text-parser_8cc_source.html">delimited-text-parser.cc</a></li>
<li>be/src/exec/<a class="el" href="delimited-text-parser_8inline_8h_source.html">delimited-text-parser.inline.h</a></li>
</ul>
</div><!-- contents -->
</div><!-- doc-content -->
<!-- start footer part -->
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
<ul>
<li class="navelem"><a class="el" href="namespaceimpala.html">impala</a></li><li class="navelem"><a class="el" href="classimpala_1_1DelimitedTextParser.html">DelimitedTextParser</a></li>
<li class="footer">Generated on Thu May 7 2015 16:10:47 for Impala by
<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li>
</ul>
</div>
</body>
</html>