| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: be/src/exec/hdfs-parquet-table-writer.cc Source File</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('hdfs-parquet-table-writer_8cc_source.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">hdfs-parquet-table-writer.cc</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| <a href="hdfs-parquet-table-writer_8cc.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">// Copyright 2012 Cloudera Inc.</span></div> |
| <div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment">// Licensed under the Apache License, Version 2.0 (the "License");</span></div> |
| <div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment">// you may not use this file except in compliance with the License.</span></div> |
| <div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment">// You may obtain a copy of the License at</span></div> |
| <div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment">// http://www.apache.org/licenses/LICENSE-2.0</span></div> |
| <div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment">// Unless required by applicable law or agreed to in writing, software</span></div> |
| <div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment">// distributed under the License is distributed on an "AS IS" BASIS,</span></div> |
| <div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment">// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div> |
| <div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment">// See the License for the specific language governing permissions and</span></div> |
| <div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment">// limitations under the License.</span></div> |
| <div class="line"><a name="l00014"></a><span class="lineno"> 14</span> </div> |
| <div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="preprocessor">#include "<a class="code" href="hdfs-parquet-table-writer_8h.html">exec/hdfs-parquet-table-writer.h</a>"</span></div> |
| <div class="line"><a name="l00016"></a><span class="lineno"> 16</span> </div> |
| <div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="preprocessor">#include "<a class="code" href="version_8h.html">common/version.h</a>"</span></div> |
| <div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="preprocessor">#include "<a class="code" href="expr_8h.html">exprs/expr.h</a>"</span></div> |
| <div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="preprocessor">#include "<a class="code" href="expr-context_8h.html">exprs/expr-context.h</a>"</span></div> |
| <div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="preprocessor">#include "<a class="code" href="decimal-value_8h.html">runtime/decimal-value.h</a>"</span></div> |
| <div class="line"><a name="l00021"></a><span class="lineno"> 21</span> <span class="preprocessor">#include "<a class="code" href="raw-value_8h.html">runtime/raw-value.h</a>"</span></div> |
| <div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="preprocessor">#include "<a class="code" href="row-batch_8h.html">runtime/row-batch.h</a>"</span></div> |
| <div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="preprocessor">#include "<a class="code" href="runtime-state_8h.html">runtime/runtime-state.h</a>"</span></div> |
| <div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="preprocessor">#include "<a class="code" href="string-value_8inline_8h.html">runtime/string-value.inline.h</a>"</span></div> |
| <div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="preprocessor">#include "<a class="code" href="bit-stream-utils_8h.html">util/bit-stream-utils.h</a>"</span></div> |
| <div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="preprocessor">#include "<a class="code" href="bit-util_8h.html">util/bit-util.h</a>"</span></div> |
| <div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="preprocessor">#include "<a class="code" href="buffer-builder_8h.html">util/buffer-builder.h</a>"</span></div> |
| <div class="line"><a name="l00028"></a><span class="lineno"> 28</span> <span class="preprocessor">#include "<a class="code" href="compress_8h.html">util/compress.h</a>"</span></div> |
| <div class="line"><a name="l00029"></a><span class="lineno"> 29</span> <span class="preprocessor">#include "<a class="code" href="debug-util_8h.html">util/debug-util.h</a>"</span></div> |
| <div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="preprocessor">#include "<a class="code" href="dict-encoding_8h.html">util/dict-encoding.h</a>"</span></div> |
| <div class="line"><a name="l00031"></a><span class="lineno"> 31</span> <span class="preprocessor">#include "<a class="code" href="hdfs-util_8h.html">util/hdfs-util.h</a>"</span></div> |
| <div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="preprocessor">#include "<a class="code" href="rle-encoding_8h.html">util/rle-encoding.h</a>"</span></div> |
| <div class="line"><a name="l00033"></a><span class="lineno"> 33</span> <span class="preprocessor">#include "<a class="code" href="thrift-util_8h.html">rpc/thrift-util.h</a>"</span></div> |
| <div class="line"><a name="l00034"></a><span class="lineno"> 34</span> </div> |
| <div class="line"><a name="l00035"></a><span class="lineno"> 35</span> <span class="preprocessor">#include <sstream></span></div> |
| <div class="line"><a name="l00036"></a><span class="lineno"> 36</span> </div> |
| <div class="line"><a name="l00037"></a><span class="lineno"> 37</span> <span class="preprocessor">#include "gen-cpp/ImpalaService_types.h"</span></div> |
| <div class="line"><a name="l00038"></a><span class="lineno"> 38</span> </div> |
| <div class="line"><a name="l00039"></a><span class="lineno"> 39</span> <span class="preprocessor">#include "<a class="code" href="names_8h.html">common/names.h</a>"</span></div> |
| <div class="line"><a name="l00040"></a><span class="lineno"> 40</span> <span class="keyword">using namespace </span>impala;</div> |
| <div class="line"><a name="l00041"></a><span class="lineno"> 41</span> <span class="keyword">using namespace </span>parquet;</div> |
| <div class="line"><a name="l00042"></a><span class="lineno"> 42</span> <span class="keyword">using namespace </span>apache::thrift;</div> |
| <div class="line"><a name="l00043"></a><span class="lineno"> 43</span> </div> |
| <div class="line"><a name="l00044"></a><span class="lineno"> 44</span> <span class="comment">// Managing file sizes: We need to estimate how big the files being buffered</span></div> |
| <div class="line"><a name="l00045"></a><span class="lineno"> 45</span> <span class="comment">// are in order to split them correctly in HDFS. Having a file that is too big</span></div> |
| <div class="line"><a name="l00046"></a><span class="lineno"> 46</span> <span class="comment">// will cause remote reads (parquet files are non-splittable).</span></div> |
| <div class="line"><a name="l00047"></a><span class="lineno"> 47</span> <span class="comment">// It's too expensive to compute the exact file sizes as the rows are buffered</span></div> |
| <div class="line"><a name="l00048"></a><span class="lineno"> 48</span> <span class="comment">// since the values in the current pages are only encoded/compressed when the page</span></div> |
| <div class="line"><a name="l00049"></a><span class="lineno"> 49</span> <span class="comment">// is full. Once the page is full, we encode and compress it, at which point we know</span></div> |
| <div class="line"><a name="l00050"></a><span class="lineno"> 50</span> <span class="comment">// the exact on file size.</span></div> |
| <div class="line"><a name="l00051"></a><span class="lineno"> 51</span> <span class="comment">// The current buffered pages (one for each column) can have a very poor estimate.</span></div> |
| <div class="line"><a name="l00052"></a><span class="lineno"> 52</span> <span class="comment">// To adjust for this, we aim for a slightly smaller file size than the ideal.</span></div> |
| <div class="line"><a name="l00053"></a><span class="lineno"> 53</span> </div> |
| <div class="line"><a name="l00054"></a><span class="lineno"> 54</span> <span class="comment">// The maximum entries in the dictionary before giving up and switching to</span></div> |
| <div class="line"><a name="l00055"></a><span class="lineno"> 55</span> <span class="comment">// plain encoding.</span></div> |
| <div class="line"><a name="l00056"></a><span class="lineno"> 56</span> <span class="comment">// TODO: more complicated heuristic?</span></div> |
| <div class="line"><a name="l00057"></a><span class="lineno"><a class="line" href="hdfs-parquet-table-writer_8cc.html#ac7c634c7e5c4f05df3731ec71156d191"> 57</a></span> <span class="keyword">static</span> <span class="keyword">const</span> <span class="keywordtype">int</span> <a class="code" href="hdfs-parquet-table-writer_8cc.html#ac7c634c7e5c4f05df3731ec71156d191">MAX_DICTIONARY_ENTRIES</a> = (1 << 16) - 1;</div> |
| <div class="line"><a name="l00058"></a><span class="lineno"> 58</span> </div> |
| <div class="line"><a name="l00059"></a><span class="lineno"> 59</span> <span class="comment">// Class that encapsulates all the state for writing a single column. This contains</span></div> |
| <div class="line"><a name="l00060"></a><span class="lineno"> 60</span> <span class="comment">// all the buffered pages as well as the metadata (e.g. byte sizes, num values, etc).</span></div> |
| <div class="line"><a name="l00061"></a><span class="lineno"> 61</span> <span class="comment">// This is intended to be created once per writer per column and reused across</span></div> |
| <div class="line"><a name="l00062"></a><span class="lineno"> 62</span> <span class="comment">// row groups.</span></div> |
| <div class="line"><a name="l00063"></a><span class="lineno"> 63</span> <span class="comment">// We currently accumulate all the data pages for an entire row group per column</span></div> |
| <div class="line"><a name="l00064"></a><span class="lineno"> 64</span> <span class="comment">// before flushing them. This can be pretty large (hundreds of MB) but we can't</span></div> |
| <div class="line"><a name="l00065"></a><span class="lineno"> 65</span> <span class="comment">// fix this without collocated files in HDFS. With collocated files, the minimum</span></div> |
| <div class="line"><a name="l00066"></a><span class="lineno"> 66</span> <span class="comment">// we'd need to buffer is 1 page per column so on the order of 1MB (although we might</span></div> |
| <div class="line"><a name="l00067"></a><span class="lineno"> 67</span> <span class="comment">// decide to buffer a few pages for better HDFS write performance).</span></div> |
| <div class="line"><a name="l00068"></a><span class="lineno"> 68</span> <span class="comment">// Pages are reused between flushes. They are created on demand as necessary and</span></div> |
| <div class="line"><a name="l00069"></a><span class="lineno"> 69</span> <span class="comment">// recycled after a flush.</span></div> |
| <div class="line"><a name="l00070"></a><span class="lineno"> 70</span> <span class="comment">// As rows come in, we accumulate the encoded values into the values_ and def_levels_</span></div> |
| <div class="line"><a name="l00071"></a><span class="lineno"> 71</span> <span class="comment">// buffers. When we've accumulated a page worth's of data, we combine values_ and</span></div> |
| <div class="line"><a name="l00072"></a><span class="lineno"> 72</span> <span class="comment">// def_levels_ into a single buffer that would be the exact bytes (with no gaps) in</span></div> |
| <div class="line"><a name="l00073"></a><span class="lineno"> 73</span> <span class="comment">// the file. The combined buffer is compressed if compression is enabled and we</span></div> |
| <div class="line"><a name="l00074"></a><span class="lineno"> 74</span> <span class="comment">// keep the combined/compressed buffer until we need to flush the file. The</span></div> |
| <div class="line"><a name="l00075"></a><span class="lineno"> 75</span> <span class="comment">// values_ and def_levels_ are then reused for the next page.</span></div> |
| <div class="line"><a name="l00076"></a><span class="lineno"> 76</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00077"></a><span class="lineno"> 77</span> <span class="comment">// TODO: For codegen, we would codegen the AppendRow() function for each column.</span></div> |
| <div class="line"><a name="l00078"></a><span class="lineno"> 78</span> <span class="comment">// This codegen is specific to the column expr (and type) and encoding. The</span></div> |
| <div class="line"><a name="l00079"></a><span class="lineno"> 79</span> <span class="comment">// parent writer object would combine all the generated AppendRow from all</span></div> |
| <div class="line"><a name="l00080"></a><span class="lineno"> 80</span> <span class="comment">// the columns and run that function over row batches.</span></div> |
| <div class="line"><a name="l00081"></a><span class="lineno"> 81</span> <span class="comment">// TODO: we need to pass in the compression from the FE/metadata</span></div> |
| <div class="line"><a name="l00082"></a><span class="lineno"> 82</span> </div> |
| <div class="line"><a name="l00083"></a><span class="lineno"> 83</span> <span class="keyword">namespace </span>impala {</div> |
| <div class="line"><a name="l00084"></a><span class="lineno"> 84</span> </div> |
| <div class="line"><a name="l00085"></a><span class="lineno"> 85</span> <span class="comment">// Base class for column writers. This contains most of the logic except for</span></div> |
| <div class="line"><a name="l00086"></a><span class="lineno"> 86</span> <span class="comment">// the type specific functions which are implemented in the subclasses.</span></div> |
| <div class="line"><a name="l00087"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html"> 87</a></span> <span class="keyword">class </span><a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">HdfsParquetTableWriter::BaseColumnWriter</a> {</div> |
| <div class="line"><a name="l00088"></a><span class="lineno"> 88</span>  <span class="keyword">public</span>:</div> |
| <div class="line"><a name="l00089"></a><span class="lineno"> 89</span>  <span class="comment">// expr - the expression to generate output values for this column.</span></div> |
| <div class="line"><a name="l00090"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a54959e4d1e9a68988a6050493d9efc46"> 90</a></span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a54959e4d1e9a68988a6050493d9efc46">BaseColumnWriter</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html">HdfsParquetTableWriter</a>* parent, <a class="code" href="classimpala_1_1ExprContext.html">ExprContext</a>* expr_ctx,</div> |
| <div class="line"><a name="l00091"></a><span class="lineno"> 91</span>  <span class="keyword">const</span> THdfsCompression::type& codec)</div> |
| <div class="line"><a name="l00092"></a><span class="lineno"> 92</span>  : parent_(parent), expr_ctx_(expr_ctx), codec_(codec),</div> |
| <div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  page_size_(DEFAULT_DATA_PAGE_SIZE), current_page_(NULL), num_values_(0),</div> |
| <div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  total_compressed_byte_size_(0),</div> |
| <div class="line"><a name="l00095"></a><span class="lineno"> 95</span>  total_uncompressed_byte_size_(0),</div> |
| <div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  dict_encoder_base_(NULL),</div> |
| <div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  def_levels_(NULL),</div> |
| <div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  values_buffer_len_(DEFAULT_DATA_PAGE_SIZE) {</div> |
| <div class="line"><a name="l00099"></a><span class="lineno"> 99</span>  <a class="code" href="classimpala_1_1Codec.html#a18854cd2e08266b09600a6e559c7c6e1">Codec::CreateCompressor</a>(NULL, <span class="keyword">false</span>, codec, &compressor_);</div> |
| <div class="line"><a name="l00100"></a><span class="lineno"> 100</span> </div> |
| <div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  def_levels_ = parent_->state_->obj_pool()->Add(</div> |
| <div class="line"><a name="l00102"></a><span class="lineno"> 102</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1RleEncoder.html">RleEncoder</a>(parent_->reusable_col_mem_pool_->Allocate(DEFAULT_DATA_PAGE_SIZE),</div> |
| <div class="line"><a name="l00103"></a><span class="lineno"> 103</span>  DEFAULT_DATA_PAGE_SIZE, 1));</div> |
| <div class="line"><a name="l00104"></a><span class="lineno"> 104</span>  values_buffer_ = parent_->reusable_col_mem_pool_->Allocate(values_buffer_len_);</div> |
| <div class="line"><a name="l00105"></a><span class="lineno"> 105</span>  }</div> |
| <div class="line"><a name="l00106"></a><span class="lineno"> 106</span> </div> |
| <div class="line"><a name="l00107"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a303f0997b8e8b3eed05c5bdd21325f82"> 107</a></span>  <span class="keyword">virtual</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a303f0997b8e8b3eed05c5bdd21325f82">~BaseColumnWriter</a>() {}</div> |
| <div class="line"><a name="l00108"></a><span class="lineno"> 108</span> </div> |
| <div class="line"><a name="l00109"></a><span class="lineno"> 109</span>  <span class="comment">// Appends the row to this column. This buffers the value into a data page. Returns</span></div> |
| <div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  <span class="comment">// error if the space needed for the encoded value is larger than the data page size.</span></div> |
| <div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <span class="comment">// TODO: this needs to be batch based, instead of row based for better performance.</span></div> |
| <div class="line"><a name="l00112"></a><span class="lineno"> 112</span>  <span class="comment">// This is a bit trickier to handle the case where only a partial row batch can be</span></div> |
| <div class="line"><a name="l00113"></a><span class="lineno"> 113</span>  <span class="comment">// output to the current file because it reaches the max file size. Enabling codegen</span></div> |
| <div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  <span class="comment">// would also solve this problem.</span></div> |
| <div class="line"><a name="l00115"></a><span class="lineno"> 115</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> AppendRow(<a class="code" href="classimpala_1_1TupleRow.html">TupleRow</a>* row);</div> |
| <div class="line"><a name="l00116"></a><span class="lineno"> 116</span> </div> |
| <div class="line"><a name="l00117"></a><span class="lineno"> 117</span>  <span class="comment">// Flushes all buffered data pages to the file.</span></div> |
| <div class="line"><a name="l00118"></a><span class="lineno"> 118</span>  <span class="comment">// *file_pos is an output parameter and will be incremented by</span></div> |
| <div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  <span class="comment">// the number of bytes needed to write all the data pages for this column.</span></div> |
| <div class="line"><a name="l00120"></a><span class="lineno"> 120</span>  <span class="comment">// first_data_page and first_dictionary_page are also out parameters and</span></div> |
| <div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  <span class="comment">// will contain the byte offset for the data page and dictionary page. They</span></div> |
| <div class="line"><a name="l00122"></a><span class="lineno"> 122</span>  <span class="comment">// will be set to -1 if the column does not contain that type of page.</span></div> |
| <div class="line"><a name="l00123"></a><span class="lineno"> 123</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> Flush(int64_t* file_pos, int64_t* first_data_page,</div> |
| <div class="line"><a name="l00124"></a><span class="lineno"> 124</span>  int64_t* first_dictionary_page);</div> |
| <div class="line"><a name="l00125"></a><span class="lineno"> 125</span> </div> |
| <div class="line"><a name="l00126"></a><span class="lineno"> 126</span>  <span class="comment">// Resets all the data accumulated for this column. Memory can now be reused for</span></div> |
| <div class="line"><a name="l00127"></a><span class="lineno"> 127</span>  <span class="comment">// the next row group</span></div> |
| <div class="line"><a name="l00128"></a><span class="lineno"> 128</span>  <span class="comment">// Any data for previous row groups must be reset (e.g. dictionaries).</span></div> |
| <div class="line"><a name="l00129"></a><span class="lineno"> 129</span>  <span class="comment">// Subclasses must call this if they override this function.</span></div> |
| <div class="line"><a name="l00130"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#aaa09b09c831fa3d0f1f29e1b67169eae"> 130</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#aaa09b09c831fa3d0f1f29e1b67169eae">Reset</a>() {</div> |
| <div class="line"><a name="l00131"></a><span class="lineno"> 131</span>  num_data_pages_ = 0;</div> |
| <div class="line"><a name="l00132"></a><span class="lineno"> 132</span>  current_page_ = NULL;</div> |
| <div class="line"><a name="l00133"></a><span class="lineno"> 133</span>  num_values_ = 0;</div> |
| <div class="line"><a name="l00134"></a><span class="lineno"> 134</span>  total_compressed_byte_size_ = 0;</div> |
| <div class="line"><a name="l00135"></a><span class="lineno"> 135</span>  current_encoding_ = <a class="code" href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">Encoding::PLAIN</a>;</div> |
| <div class="line"><a name="l00136"></a><span class="lineno"> 136</span>  }</div> |
| <div class="line"><a name="l00137"></a><span class="lineno"> 137</span> </div> |
| <div class="line"><a name="l00138"></a><span class="lineno"> 138</span>  <span class="comment">// Close this writer. This is only called after Flush() and no more rows will</span></div> |
| <div class="line"><a name="l00139"></a><span class="lineno"> 139</span>  <span class="comment">// be added.</span></div> |
| <div class="line"><a name="l00140"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#af9f34726a2605d7d0a13881089343746"> 140</a></span>  <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#af9f34726a2605d7d0a13881089343746">Close</a>() {</div> |
| <div class="line"><a name="l00141"></a><span class="lineno"> 141</span>  <span class="keywordflow">if</span> (compressor_.get() != NULL) compressor_->Close();</div> |
| <div class="line"><a name="l00142"></a><span class="lineno"> 142</span>  <span class="keywordflow">if</span> (dict_encoder_base_ != NULL) dict_encoder_base_->ClearIndices();</div> |
| <div class="line"><a name="l00143"></a><span class="lineno"> 143</span>  }</div> |
| <div class="line"><a name="l00144"></a><span class="lineno"> 144</span> </div> |
| <div class="line"><a name="l00145"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a7ea48982ea52afc394148b5813afe9b7"> 145</a></span>  <span class="keyword">const</span> <a class="code" href="structimpala_1_1ColumnType.html">ColumnType</a>& <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a7ea48982ea52afc394148b5813afe9b7">type</a>()<span class="keyword"> const </span>{ <span class="keywordflow">return</span> expr_ctx_->root()-><a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a>(); }</div> |
| <div class="line"><a name="l00146"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae401238cb881d9277416cc0db68bbdfc"> 146</a></span>  <a class="code" href="classuint64__t.html">uint64_t</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae401238cb881d9277416cc0db68bbdfc">num_values</a>()<span class="keyword"> const </span>{ <span class="keywordflow">return</span> num_values_; }</div> |
| <div class="line"><a name="l00147"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a76ba4a28fb2cad5241eb48f32fdc2941"> 147</a></span>  <a class="code" href="classuint64__t.html">uint64_t</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a76ba4a28fb2cad5241eb48f32fdc2941">total_compressed_size</a>()<span class="keyword"> const </span>{ <span class="keywordflow">return</span> total_compressed_byte_size_; }</div> |
| <div class="line"><a name="l00148"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8621a48438ae5353ff6b864294a8cf4e"> 148</a></span>  <a class="code" href="classuint64__t.html">uint64_t</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8621a48438ae5353ff6b864294a8cf4e">total_uncompressed_size</a>()<span class="keyword"> const </span>{ <span class="keywordflow">return</span> total_uncompressed_byte_size_; }</div> |
| <div class="line"><a name="l00149"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a69016580aa437551ed8126fa652a75dc"> 149</a></span>  parquet::CompressionCodec::type <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a69016580aa437551ed8126fa652a75dc">codec</a>()<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l00150"></a><span class="lineno"> 150</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#ad7f28231c208b04ff9bedf46f1673f4b">IMPALA_TO_PARQUET_CODEC</a>[codec_];</div> |
| <div class="line"><a name="l00151"></a><span class="lineno"> 151</span>  }</div> |
| <div class="line"><a name="l00152"></a><span class="lineno"> 152</span> </div> |
| <div class="line"><a name="l00153"></a><span class="lineno"> 153</span>  <span class="keyword">protected</span>:</div> |
| <div class="line"><a name="l00154"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a77ef6e36e19dcb34d602d4536276ad9f"> 154</a></span>  <span class="keyword">friend</span> <span class="keyword">class </span><a class="code" href="classimpala_1_1HdfsParquetTableWriter.html">HdfsParquetTableWriter</a>;</div> |
| <div class="line"><a name="l00155"></a><span class="lineno"> 155</span> </div> |
| <div class="line"><a name="l00156"></a><span class="lineno"> 156</span>  <span class="comment">// Encode value into the current page output buffer. Returns true if the value fits</span></div> |
| <div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  <span class="comment">// on the current page. If this function returned false, the caller should create a</span></div> |
| <div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  <span class="comment">// new page and try again with the same value.</span></div> |
| <div class="line"><a name="l00159"></a><span class="lineno"> 159</span>  <span class="comment">// *bytes_needed will contain the (estimated) number of bytes needed to successfully</span></div> |
| <div class="line"><a name="l00160"></a><span class="lineno"> 160</span>  <span class="comment">// encode the value in the page.</span></div> |
| <div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <span class="comment">// Implemented in the subclass.</span></div> |
| <div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  <span class="keyword">virtual</span> <span class="keywordtype">bool</span> EncodeValue(<span class="keywordtype">void</span>* value, int64_t* bytes_needed) = 0;</div> |
| <div class="line"><a name="l00163"></a><span class="lineno"> 163</span> </div> |
| <div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  <span class="comment">// Encodes out all data for the current page and updates the metadata.</span></div> |
| <div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  <span class="keyword">virtual</span> <span class="keywordtype">void</span> FinalizeCurrentPage();</div> |
| <div class="line"><a name="l00166"></a><span class="lineno"> 166</span> </div> |
| <div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  <span class="comment">// Update current_page_ to a new page, reusing pages allocated if possible.</span></div> |
| <div class="line"><a name="l00168"></a><span class="lineno"> 168</span>  <span class="keywordtype">void</span> NewPage();</div> |
| <div class="line"><a name="l00169"></a><span class="lineno"> 169</span> </div> |
| <div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  <span class="comment">// Writes out the dictionary encoded data buffered in dict_encoder_.</span></div> |
| <div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  <span class="keywordtype">void</span> WriteDictDataPage();</div> |
| <div class="line"><a name="l00172"></a><span class="lineno"> 172</span> </div> |
| <div class="line"><a name="l00173"></a><span class="lineno"><a class="line" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html"> 173</a></span>  <span class="keyword">struct </span><a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html">DataPage</a> {</div> |
| <div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  <span class="comment">// Page header. This is a union of all page types.</span></div> |
| <div class="line"><a name="l00175"></a><span class="lineno"><a class="line" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55"> 175</a></span>  PageHeader <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">header</a>;</div> |
| <div class="line"><a name="l00176"></a><span class="lineno"> 176</span> </div> |
| <div class="line"><a name="l00177"></a><span class="lineno"> 177</span>  <span class="comment">// Number of bytes needed to store definition levels.</span></div> |
| <div class="line"><a name="l00178"></a><span class="lineno"><a class="line" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a35b49574b69acd7b34ded9083c9f2ef8"> 178</a></span>  <span class="keywordtype">int</span> <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a35b49574b69acd7b34ded9083c9f2ef8">num_def_bytes</a>;</div> |
| <div class="line"><a name="l00179"></a><span class="lineno"> 179</span> </div> |
| <div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  <span class="comment">// This is the payload for the data page. This includes the definition/repetition</span></div> |
| <div class="line"><a name="l00181"></a><span class="lineno"> 181</span>  <span class="comment">// levels data and the encoded values. If compression is enabled, this is the</span></div> |
| <div class="line"><a name="l00182"></a><span class="lineno"> 182</span>  <span class="comment">// compressed data.</span></div> |
| <div class="line"><a name="l00183"></a><span class="lineno"><a class="line" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a4b0955451fd2bbcc8ef33f631033c627"> 183</a></span>  uint8_t* <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a4b0955451fd2bbcc8ef33f631033c627">data</a>;</div> |
| <div class="line"><a name="l00184"></a><span class="lineno"> 184</span> </div> |
| <div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  <span class="comment">// If true, this data page has been finalized. All sizes are computed, header is</span></div> |
| <div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  <span class="comment">// fully populated and any compression is done.</span></div> |
| <div class="line"><a name="l00187"></a><span class="lineno"><a class="line" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab25f53e961fe5b3871feded8ea9b97b3"> 187</a></span>  <span class="keywordtype">bool</span> <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab25f53e961fe5b3871feded8ea9b97b3">finalized</a>;</div> |
| <div class="line"><a name="l00188"></a><span class="lineno"> 188</span> </div> |
| <div class="line"><a name="l00189"></a><span class="lineno"> 189</span>  <span class="comment">// Number of non-null values</span></div> |
| <div class="line"><a name="l00190"></a><span class="lineno"><a class="line" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a570d69e9568d8b34483d2746cd90daac"> 190</a></span>  <span class="keywordtype">int</span> <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a570d69e9568d8b34483d2746cd90daac">num_non_null</a>;</div> |
| <div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  };</div> |
| <div class="line"><a name="l00192"></a><span class="lineno"> 192</span> </div> |
| <div class="line"><a name="l00193"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#adc18fee9bd07043ebc621d91bb8f57f2"> 193</a></span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html">HdfsParquetTableWriter</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#adc18fee9bd07043ebc621d91bb8f57f2">parent_</a>;</div> |
| <div class="line"><a name="l00194"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#af147ac530796b3aa05000035596861e0"> 194</a></span>  <a class="code" href="classimpala_1_1ExprContext.html">ExprContext</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#af147ac530796b3aa05000035596861e0">expr_ctx_</a>;</div> |
| <div class="line"><a name="l00195"></a><span class="lineno"> 195</span> </div> |
| <div class="line"><a name="l00196"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a3726f7987a677970a9a193a585cd14dc"> 196</a></span>  THdfsCompression::type <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a3726f7987a677970a9a193a585cd14dc">codec_</a>;</div> |
| <div class="line"><a name="l00197"></a><span class="lineno"> 197</span> </div> |
| <div class="line"><a name="l00198"></a><span class="lineno"> 198</span>  <span class="comment">// Compression codec for this column. If NULL, this column is will not be compressed.</span></div> |
| <div class="line"><a name="l00199"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a27b116b49ec5a6965bf03d7df33019bf"> 199</a></span>  scoped_ptr<Codec> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a27b116b49ec5a6965bf03d7df33019bf">compressor_</a>;</div> |
| <div class="line"><a name="l00200"></a><span class="lineno"> 200</span> </div> |
| <div class="line"><a name="l00201"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a6c65b99c47cbd2ad8e475fc5bac00e3a"> 201</a></span>  vector<DataPage> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a6c65b99c47cbd2ad8e475fc5bac00e3a">pages_</a>;</div> |
| <div class="line"><a name="l00202"></a><span class="lineno"> 202</span> </div> |
| <div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  <span class="comment">// Number of pages in 'pages_' that are used. 'pages_' is reused between flushes</span></div> |
| <div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  <span class="comment">// so this number can be less than pages_.size()</span></div> |
| <div class="line"><a name="l00205"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a32b5ddbc74fd317fb16ebdb859c7e167"> 205</a></span>  <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a32b5ddbc74fd317fb16ebdb859c7e167">num_data_pages_</a>;</div> |
| <div class="line"><a name="l00206"></a><span class="lineno"> 206</span> </div> |
| <div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  <span class="comment">// Size of newly created pages. Defaults to DEFAULT_DATA_PAGE_SIZE and is increased</span></div> |
| <div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  <span class="comment">// when pages are not big enough. This only happens when there are enough unique values</span></div> |
| <div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  <span class="comment">// such that we switch from PLAIN_DICTIONARY to PLAIN encoding and then have very</span></div> |
| <div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  <span class="comment">// large values (i.e. greater than DEFAULT_DATA_PAGE_SIZE).</span></div> |
| <div class="line"><a name="l00211"></a><span class="lineno"> 211</span>  <span class="comment">// TODO: Consider removing and only creating a single large page as necessary.</span></div> |
| <div class="line"><a name="l00212"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a076783ce2c4d6603675879867334fe7d"> 212</a></span>  int64_t <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a076783ce2c4d6603675879867334fe7d">page_size_</a>;</div> |
| <div class="line"><a name="l00213"></a><span class="lineno"> 213</span> </div> |
| <div class="line"><a name="l00214"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ad3d393e5df5ebc0d0a3def6d1b9ef6ca"> 214</a></span>  <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html">DataPage</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ad3d393e5df5ebc0d0a3def6d1b9ef6ca">current_page_</a>;</div> |
| <div class="line"><a name="l00215"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a58fbe85de5770a41f8936ed57e47f6a7"> 215</a></span>  int64_t <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a58fbe85de5770a41f8936ed57e47f6a7">num_values_</a>; <span class="comment">// Total number of values across all pages, including NULLs.</span></div> |
| <div class="line"><a name="l00216"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae4efff2f9c9b600b8bb51e24d51fa74a"> 216</a></span>  int64_t <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae4efff2f9c9b600b8bb51e24d51fa74a">total_compressed_byte_size_</a>;</div> |
| <div class="line"><a name="l00217"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a17dd47f44b1374d094451e3cb1f5801e"> 217</a></span>  int64_t <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a17dd47f44b1374d094451e3cb1f5801e">total_uncompressed_byte_size_</a>;</div> |
| <div class="line"><a name="l00218"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ab3c85c3219e6cc628543c2ab08b1b204"> 218</a></span>  Encoding::type <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ab3c85c3219e6cc628543c2ab08b1b204">current_encoding_</a>;</div> |
| <div class="line"><a name="l00219"></a><span class="lineno"> 219</span> </div> |
| <div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="comment">// Created and set by the base class.</span></div> |
| <div class="line"><a name="l00221"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ab74915a4a656b8cb4cce2cf71f31036a"> 221</a></span>  <a class="code" href="classimpala_1_1DictEncoderBase.html">DictEncoderBase</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ab74915a4a656b8cb4cce2cf71f31036a">dict_encoder_base_</a>;</div> |
| <div class="line"><a name="l00222"></a><span class="lineno"> 222</span> </div> |
| <div class="line"><a name="l00223"></a><span class="lineno"> 223</span>  <span class="comment">// Rle encoder object for storing definition levels. For non-nested schemas,</span></div> |
| <div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="comment">// this always uses 1 bit per row.</span></div> |
| <div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  <span class="comment">// This is reused across pages since the underlying buffer is copied out when</span></div> |
| <div class="line"><a name="l00226"></a><span class="lineno"> 226</span>  <span class="comment">// the page is finalized.</span></div> |
| <div class="line"><a name="l00227"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a16639b10ee6d377ac527c80a882f07ed"> 227</a></span>  <a class="code" href="classimpala_1_1RleEncoder.html">RleEncoder</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a16639b10ee6d377ac527c80a882f07ed">def_levels_</a>;</div> |
| <div class="line"><a name="l00228"></a><span class="lineno"> 228</span> </div> |
| <div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  <span class="comment">// Data for buffered values. This is reused across pages.</span></div> |
| <div class="line"><a name="l00230"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a226ca85a62a609e56a3bca74f7dc90d6"> 230</a></span>  uint8_t* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a226ca85a62a609e56a3bca74f7dc90d6">values_buffer_</a>;</div> |
| <div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  <span class="comment">// The size of values_buffer_.</span></div> |
| <div class="line"><a name="l00232"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae8afd359ab02b0c78aeb28807da48f32"> 232</a></span>  <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae8afd359ab02b0c78aeb28807da48f32">values_buffer_len_</a>;</div> |
| <div class="line"><a name="l00233"></a><span class="lineno"> 233</span> };</div> |
| <div class="line"><a name="l00234"></a><span class="lineno"> 234</span> </div> |
| <div class="line"><a name="l00235"></a><span class="lineno"> 235</span> <span class="comment">// Per type column writer.</span></div> |
| <div class="line"><a name="l00236"></a><span class="lineno"> 236</span> <span class="keyword">template</span><<span class="keyword">typename</span> T></div> |
| <div class="line"><a name="l00237"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html"> 237</a></span> <span class="keyword">class </span><a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">HdfsParquetTableWriter::ColumnWriter</a> :</div> |
| <div class="line"><a name="l00238"></a><span class="lineno"> 238</span>  <span class="keyword">public</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">HdfsParquetTableWriter::BaseColumnWriter</a> {</div> |
| <div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  <span class="keyword">public</span>:</div> |
| <div class="line"><a name="l00240"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#aa0ef7d2984567dd8995251198975113b"> 240</a></span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#aa0ef7d2984567dd8995251198975113b">ColumnWriter</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html">HdfsParquetTableWriter</a>* parent, <a class="code" href="classimpala_1_1ExprContext.html">ExprContext</a>* ctx,</div> |
| <div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  <span class="keyword">const</span> THdfsCompression::type& codec) : <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">BaseColumnWriter</a>(parent, ctx, codec),</div> |
| <div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  num_values_since_dict_size_check_(0) {</div> |
| <div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  DCHECK_NE(ctx-><a class="code" href="classimpala_1_1ExprContext.html#a2e652a852e99d5cd375dce3f91520aee">root</a>()-><a class="code" href="classimpala_1_1Expr.html#a742827844080d45d514719742cb3e7f5">type</a>().<a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a>, <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a936286363e052834631db5c19be3187d">TYPE_BOOLEAN</a>);</div> |
| <div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  encoded_value_size_ = <a class="code" href="classimpala_1_1ParquetPlainEncoder.html#a42f57c98268a410f74f3cbefde238c28">ParquetPlainEncoder::ByteSize</a>(ctx-><a class="code" href="classimpala_1_1ExprContext.html#a2e652a852e99d5cd375dce3f91520aee">root</a>()-><a class="code" href="classimpala_1_1Expr.html#a742827844080d45d514719742cb3e7f5">type</a>());</div> |
| <div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  }</div> |
| <div class="line"><a name="l00246"></a><span class="lineno"> 246</span> </div> |
| <div class="line"><a name="l00247"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a753b558d1afe680335aad9b5772772ab"> 247</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a753b558d1afe680335aad9b5772772ab">Reset</a>() {</div> |
| <div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  BaseColumnWriter::Reset();</div> |
| <div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  <span class="comment">// Default to dictionary encoding. If the cardinality ends up being too high,</span></div> |
| <div class="line"><a name="l00250"></a><span class="lineno"> 250</span>  <span class="comment">// it will fall back to plain.</span></div> |
| <div class="line"><a name="l00251"></a><span class="lineno"> 251</span>  current_encoding_ = Encoding::PLAIN_DICTIONARY;</div> |
| <div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  dict_encoder_.reset(</div> |
| <div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1DictEncoder.html">DictEncoder<T></a>(parent_->per_file_mem_pool_.get(), encoded_value_size_));</div> |
| <div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  dict_encoder_base_ = dict_encoder_.get();</div> |
| <div class="line"><a name="l00255"></a><span class="lineno"> 255</span>  }</div> |
| <div class="line"><a name="l00256"></a><span class="lineno"> 256</span> </div> |
| <div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  <span class="keyword">protected</span>:</div> |
| <div class="line"><a name="l00258"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a934d462020552f342270418e27135dbd"> 258</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">bool</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a934d462020552f342270418e27135dbd">EncodeValue</a>(<span class="keywordtype">void</span>* value, int64_t* bytes_needed) {</div> |
| <div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  <span class="keywordflow">if</span> (current_encoding_ == Encoding::PLAIN_DICTIONARY) {</div> |
| <div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  <span class="keywordflow">if</span> (<a class="code" href="compiler-util_8h.html#a9acc330d508b9a3b775cfdf7ce405e7d">UNLIKELY</a>(num_values_since_dict_size_check_ >=</div> |
| <div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  DICTIONARY_DATA_PAGE_SIZE_CHECK_PERIOD)) {</div> |
| <div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  num_values_since_dict_size_check_ = 0;</div> |
| <div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  <span class="keywordflow">if</span> (dict_encoder_->EstimatedDataEncodedSize() >= page_size_) <span class="keywordflow">return</span> <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00264"></a><span class="lineno"> 264</span>  }</div> |
| <div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  ++num_values_since_dict_size_check_;</div> |
| <div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  *bytes_needed = dict_encoder_->Put(*CastValue(value));</div> |
| <div class="line"><a name="l00267"></a><span class="lineno"> 267</span>  <span class="comment">// If the dictionary contains the maximum number of values, switch to plain</span></div> |
| <div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  <span class="comment">// encoding. The current dictionary encoded page is written out.</span></div> |
| <div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  <span class="keywordflow">if</span> (<a class="code" href="compiler-util_8h.html#a9acc330d508b9a3b775cfdf7ce405e7d">UNLIKELY</a>(*bytes_needed < 0)) {</div> |
| <div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  FinalizeCurrentPage();</div> |
| <div class="line"><a name="l00271"></a><span class="lineno"> 271</span>  current_encoding_ = <a class="code" href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">Encoding::PLAIN</a>;</div> |
| <div class="line"><a name="l00272"></a><span class="lineno"> 272</span>  <span class="keywordflow">return</span> <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00273"></a><span class="lineno"> 273</span>  }</div> |
| <div class="line"><a name="l00274"></a><span class="lineno"> 274</span>  parent_->file_size_estimate_ += *bytes_needed;</div> |
| <div class="line"><a name="l00275"></a><span class="lineno"> 275</span>  } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (current_encoding_ == <a class="code" href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">Encoding::PLAIN</a>) {</div> |
| <div class="line"><a name="l00276"></a><span class="lineno"> 276</span>  T* v = CastValue(value);</div> |
| <div class="line"><a name="l00277"></a><span class="lineno"> 277</span>  *bytes_needed = encoded_value_size_ < 0 ?</div> |
| <div class="line"><a name="l00278"></a><span class="lineno"> 278</span>  ParquetPlainEncoder::ByteSize<T>(*v) : encoded_value_size_;</div> |
| <div class="line"><a name="l00279"></a><span class="lineno"> 279</span>  <span class="keywordflow">if</span> (current_page_->header.uncompressed_page_size + *bytes_needed > page_size_) {</div> |
| <div class="line"><a name="l00280"></a><span class="lineno"> 280</span>  <span class="keywordflow">return</span> <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00281"></a><span class="lineno"> 281</span>  }</div> |
| <div class="line"><a name="l00282"></a><span class="lineno"> 282</span>  uint8_t* dst_ptr = values_buffer_ + current_page_->header.uncompressed_page_size;</div> |
| <div class="line"><a name="l00283"></a><span class="lineno"> 283</span>  int64_t written_len =</div> |
| <div class="line"><a name="l00284"></a><span class="lineno"> 284</span>  <a class="code" href="classimpala_1_1ParquetPlainEncoder.html#af05564bd80bdc32ae9fcf9a39240add7">ParquetPlainEncoder::Encode</a>(dst_ptr, encoded_value_size_, *v);</div> |
| <div class="line"><a name="l00285"></a><span class="lineno"> 285</span>  DCHECK_EQ(*bytes_needed, written_len);</div> |
| <div class="line"><a name="l00286"></a><span class="lineno"> 286</span>  current_page_->header.uncompressed_page_size += written_len;</div> |
| <div class="line"><a name="l00287"></a><span class="lineno"> 287</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  <span class="comment">// TODO: support other encodings here</span></div> |
| <div class="line"><a name="l00289"></a><span class="lineno"> 289</span>  DCHECK(<span class="keyword">false</span>);</div> |
| <div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  }</div> |
| <div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  <span class="keywordflow">return</span> <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00292"></a><span class="lineno"> 292</span>  }</div> |
| <div class="line"><a name="l00293"></a><span class="lineno"> 293</span> </div> |
| <div class="line"><a name="l00294"></a><span class="lineno"> 294</span>  <span class="keyword">private</span>:</div> |
| <div class="line"><a name="l00295"></a><span class="lineno"> 295</span>  <span class="comment">// The period, in # of rows, to check the estimated dictionary page size against</span></div> |
| <div class="line"><a name="l00296"></a><span class="lineno"> 296</span>  <span class="comment">// the data page size. We want to start a new data page when the estimated size</span></div> |
| <div class="line"><a name="l00297"></a><span class="lineno"> 297</span>  <span class="comment">// is at least that big. The estimated size computation is not very cheap and</span></div> |
| <div class="line"><a name="l00298"></a><span class="lineno"> 298</span>  <span class="comment">// we can tolerate going over the data page size by some amount.</span></div> |
| <div class="line"><a name="l00299"></a><span class="lineno"> 299</span>  <span class="comment">// The expected byte size per dictionary value is < 1B and at most 2 bytes so the</span></div> |
| <div class="line"><a name="l00300"></a><span class="lineno"> 300</span>  <span class="comment">// error is pretty low.</span></div> |
| <div class="line"><a name="l00301"></a><span class="lineno"> 301</span>  <span class="comment">// TODO: is there a better way?</span></div> |
| <div class="line"><a name="l00302"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#ae62a040b4017c0ccee8046989187039c"> 302</a></span>  <span class="keyword">static</span> <span class="keyword">const</span> <span class="keywordtype">int</span> DICTIONARY_DATA_PAGE_SIZE_CHECK_PERIOD = 100;</div> |
| <div class="line"><a name="l00303"></a><span class="lineno"> 303</span> </div> |
| <div class="line"><a name="l00304"></a><span class="lineno"> 304</span>  <span class="comment">// Encoder for dictionary encoding for different columns. Only one is set.</span></div> |
| <div class="line"><a name="l00305"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a5b4fb69116d191d1f70569ae77231460"> 305</a></span>  scoped_ptr<DictEncoder<T> > <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a5b4fb69116d191d1f70569ae77231460">dict_encoder_</a>;</div> |
| <div class="line"><a name="l00306"></a><span class="lineno"> 306</span> </div> |
| <div class="line"><a name="l00307"></a><span class="lineno"> 307</span>  <span class="comment">// The number of values added since we last checked the dictionary.</span></div> |
| <div class="line"><a name="l00308"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#ae9a4f4f1916abea5c92001bf43d5d436"> 308</a></span>  <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#ae9a4f4f1916abea5c92001bf43d5d436">num_values_since_dict_size_check_</a>;</div> |
| <div class="line"><a name="l00309"></a><span class="lineno"> 309</span> </div> |
| <div class="line"><a name="l00310"></a><span class="lineno"> 310</span>  <span class="comment">// Size of each encoded value. -1 if the size is type is variable-length.</span></div> |
| <div class="line"><a name="l00311"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a2c4a2e702c8776b1e5e3493a157bc4d2"> 311</a></span>  int64_t <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a2c4a2e702c8776b1e5e3493a157bc4d2">encoded_value_size_</a>;</div> |
| <div class="line"><a name="l00312"></a><span class="lineno"> 312</span> </div> |
| <div class="line"><a name="l00313"></a><span class="lineno"> 313</span>  <span class="comment">// Temporary string value to hold CHAR(N)</span></div> |
| <div class="line"><a name="l00314"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a7530ca781087774aa969e2f03600e234"> 314</a></span>  <a class="code" href="structimpala_1_1StringValue.html">StringValue</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a7530ca781087774aa969e2f03600e234">temp_</a>;</div> |
| <div class="line"><a name="l00315"></a><span class="lineno"> 315</span> </div> |
| <div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  <span class="comment">// Converts a slot pointer to a raw value suitable for encoding</span></div> |
| <div class="line"><a name="l00317"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a84b7339c7e5ebb25ea9e6e3907b41b3b"> 317</a></span>  <span class="keyword">inline</span> T* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a84b7339c7e5ebb25ea9e6e3907b41b3b">CastValue</a>(<span class="keywordtype">void</span>* value) {</div> |
| <div class="line"><a name="l00318"></a><span class="lineno"> 318</span>  <span class="keywordflow">return</span> <span class="keyword">reinterpret_cast<</span>T*<span class="keyword">></span>(value);</div> |
| <div class="line"><a name="l00319"></a><span class="lineno"> 319</span>  }</div> |
| <div class="line"><a name="l00320"></a><span class="lineno"> 320</span> };</div> |
| <div class="line"><a name="l00321"></a><span class="lineno"> 321</span> </div> |
| <div class="line"><a name="l00322"></a><span class="lineno"> 322</span> <span class="keyword">template</span><></div> |
| <div class="line"><a name="l00323"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#ab8445287491cd4ec2e5458b1f09c8b6c"> 323</a></span> <span class="keyword">inline</span> <a class="code" href="structimpala_1_1StringValue.html">StringValue</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a84b7339c7e5ebb25ea9e6e3907b41b3b">HdfsParquetTableWriter::ColumnWriter<StringValue>::CastValue</a>(</div> |
| <div class="line"><a name="l00324"></a><span class="lineno"> 324</span>  <span class="keywordtype">void</span>* value) {</div> |
| <div class="line"><a name="l00325"></a><span class="lineno"> 325</span>  <span class="keywordflow">if</span> (type().type == <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ae5ec6e333587a27f8b83c5bd9c815a2a">TYPE_CHAR</a>) {</div> |
| <div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  temp_.ptr = <a class="code" href="structimpala_1_1StringValue.html#a60791f03f5efcddd11a3afb9ddc833e4">StringValue::CharSlotToPtr</a>(value, type());</div> |
| <div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  temp_.len = <a class="code" href="structimpala_1_1StringValue.html#abed17df66bd5a82e95f5c1fcf9c0da1c">StringValue::UnpaddedCharLength</a>(temp_.ptr, type().len);</div> |
| <div class="line"><a name="l00328"></a><span class="lineno"> 328</span>  <span class="keywordflow">return</span> &temp_;</div> |
| <div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  }</div> |
| <div class="line"><a name="l00330"></a><span class="lineno"> 330</span>  <span class="keywordflow">return</span> <span class="keyword">reinterpret_cast<</span><a class="code" href="structimpala_1_1StringValue.html">StringValue</a>*<span class="keyword">></span>(value);</div> |
| <div class="line"><a name="l00331"></a><span class="lineno"> 331</span> }</div> |
| <div class="line"><a name="l00332"></a><span class="lineno"> 332</span> </div> |
| <div class="line"><a name="l00333"></a><span class="lineno"> 333</span> <span class="comment">// Bools are encoded a bit differently so subclass it explicitly.</span></div> |
| <div class="line"><a name="l00334"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html"> 334</a></span> <span class="keyword">class </span><a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html">HdfsParquetTableWriter::BoolColumnWriter</a> :</div> |
| <div class="line"><a name="l00335"></a><span class="lineno"> 335</span>  <span class="keyword">public</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">HdfsParquetTableWriter::BaseColumnWriter</a> {</div> |
| <div class="line"><a name="l00336"></a><span class="lineno"> 336</span>  <span class="keyword">public</span>:</div> |
| <div class="line"><a name="l00337"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#acf6e2e3b8c295c2a485c619fbf5596d9"> 337</a></span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#acf6e2e3b8c295c2a485c619fbf5596d9">BoolColumnWriter</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html">HdfsParquetTableWriter</a>* parent, <a class="code" href="classimpala_1_1ExprContext.html">ExprContext</a>* ctx,</div> |
| <div class="line"><a name="l00338"></a><span class="lineno"> 338</span>  <span class="keyword">const</span> THdfsCompression::type& codec) : <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">BaseColumnWriter</a>(parent, ctx, codec) {</div> |
| <div class="line"><a name="l00339"></a><span class="lineno"> 339</span>  DCHECK_EQ(ctx-><a class="code" href="classimpala_1_1ExprContext.html#a2e652a852e99d5cd375dce3f91520aee">root</a>()-><a class="code" href="classimpala_1_1Expr.html#a742827844080d45d514719742cb3e7f5">type</a>().<a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a>, <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a936286363e052834631db5c19be3187d">TYPE_BOOLEAN</a>);</div> |
| <div class="line"><a name="l00340"></a><span class="lineno"> 340</span>  bool_values_ = parent_->state_->obj_pool()->Add(</div> |
| <div class="line"><a name="l00341"></a><span class="lineno"> 341</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1BitWriter.html">BitWriter</a>(values_buffer_, values_buffer_len_));</div> |
| <div class="line"><a name="l00342"></a><span class="lineno"> 342</span>  <span class="comment">// Dictionary encoding doesn't make sense for bools and is not allowed by</span></div> |
| <div class="line"><a name="l00343"></a><span class="lineno"> 343</span>  <span class="comment">// the format.</span></div> |
| <div class="line"><a name="l00344"></a><span class="lineno"> 344</span>  current_encoding_ = <a class="code" href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">Encoding::PLAIN</a>;</div> |
| <div class="line"><a name="l00345"></a><span class="lineno"> 345</span>  dict_encoder_base_ = NULL;</div> |
| <div class="line"><a name="l00346"></a><span class="lineno"> 346</span>  }</div> |
| <div class="line"><a name="l00347"></a><span class="lineno"> 347</span> </div> |
| <div class="line"><a name="l00348"></a><span class="lineno"> 348</span>  <span class="keyword">protected</span>:</div> |
| <div class="line"><a name="l00349"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a59be46b62d8150d518cfe5c8bd36ea8d"> 349</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">bool</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a59be46b62d8150d518cfe5c8bd36ea8d">EncodeValue</a>(<span class="keywordtype">void</span>* value, int64_t* bytes_needed) {</div> |
| <div class="line"><a name="l00350"></a><span class="lineno"> 350</span>  <span class="keywordflow">return</span> bool_values_->PutValue(*reinterpret_cast<bool*>(value), 1);</div> |
| <div class="line"><a name="l00351"></a><span class="lineno"> 351</span>  }</div> |
| <div class="line"><a name="l00352"></a><span class="lineno"> 352</span> </div> |
| <div class="line"><a name="l00353"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a7df1d2c3919af5601f56c2aed1c21d74"> 353</a></span>  <span class="keyword">virtual</span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a7df1d2c3919af5601f56c2aed1c21d74">FinalizeCurrentPage</a>() {</div> |
| <div class="line"><a name="l00354"></a><span class="lineno"> 354</span>  DCHECK(current_page_ != NULL);</div> |
| <div class="line"><a name="l00355"></a><span class="lineno"> 355</span>  <span class="keywordflow">if</span> (current_page_->finalized) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00356"></a><span class="lineno"> 356</span>  bool_values_->Flush();</div> |
| <div class="line"><a name="l00357"></a><span class="lineno"> 357</span>  <span class="keywordtype">int</span> num_bytes = bool_values_->bytes_written();</div> |
| <div class="line"><a name="l00358"></a><span class="lineno"> 358</span>  current_page_->header.uncompressed_page_size += num_bytes;</div> |
| <div class="line"><a name="l00359"></a><span class="lineno"> 359</span>  <span class="comment">// Call into superclass to handle the rest.</span></div> |
| <div class="line"><a name="l00360"></a><span class="lineno"> 360</span>  BaseColumnWriter::FinalizeCurrentPage();</div> |
| <div class="line"><a name="l00361"></a><span class="lineno"> 361</span>  bool_values_->Clear();</div> |
| <div class="line"><a name="l00362"></a><span class="lineno"> 362</span>  }</div> |
| <div class="line"><a name="l00363"></a><span class="lineno"> 363</span> </div> |
| <div class="line"><a name="l00364"></a><span class="lineno"> 364</span>  <span class="keyword">private</span>:</div> |
| <div class="line"><a name="l00365"></a><span class="lineno"> 365</span>  <span class="comment">// Used to encode bools as single bit values. This is reused across pages.</span></div> |
| <div class="line"><a name="l00366"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a4c42d54f3f086d230f9311c80046c95f"> 366</a></span>  <a class="code" href="classimpala_1_1BitWriter.html">BitWriter</a>* <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a4c42d54f3f086d230f9311c80046c95f">bool_values_</a>;</div> |
| <div class="line"><a name="l00367"></a><span class="lineno"> 367</span> };</div> |
| <div class="line"><a name="l00368"></a><span class="lineno"> 368</span> </div> |
| <div class="line"><a name="l00369"></a><span class="lineno"> 369</span> }</div> |
| <div class="line"><a name="l00370"></a><span class="lineno"> 370</span> </div> |
| <div class="line"><a name="l00371"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a5ee5e263966386b0c48087ca790a4f29"> 371</a></span> <span class="keyword">inline</span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a5ee5e263966386b0c48087ca790a4f29">HdfsParquetTableWriter::BaseColumnWriter::AppendRow</a>(<a class="code" href="classimpala_1_1TupleRow.html">TupleRow</a>* row) {</div> |
| <div class="line"><a name="l00372"></a><span class="lineno"> 372</span>  ++num_values_;</div> |
| <div class="line"><a name="l00373"></a><span class="lineno"> 373</span>  <span class="keywordtype">void</span>* value = expr_ctx_->GetValue(row);</div> |
| <div class="line"><a name="l00374"></a><span class="lineno"> 374</span>  <span class="keywordflow">if</span> (current_page_ == NULL) NewPage();</div> |
| <div class="line"><a name="l00375"></a><span class="lineno"> 375</span> </div> |
| <div class="line"><a name="l00376"></a><span class="lineno"> 376</span>  <span class="comment">// We might need to try again if this current page is not big enough</span></div> |
| <div class="line"><a name="l00377"></a><span class="lineno"> 377</span>  <span class="keywordflow">while</span> (<span class="keyword">true</span>) {</div> |
| <div class="line"><a name="l00378"></a><span class="lineno"> 378</span>  <span class="keywordflow">if</span> (!def_levels_->Put(value != NULL)) {</div> |
| <div class="line"><a name="l00379"></a><span class="lineno"> 379</span>  FinalizeCurrentPage();</div> |
| <div class="line"><a name="l00380"></a><span class="lineno"> 380</span>  NewPage();</div> |
| <div class="line"><a name="l00381"></a><span class="lineno"> 381</span>  <span class="keywordtype">bool</span> ret = def_levels_->Put(value != NULL);</div> |
| <div class="line"><a name="l00382"></a><span class="lineno"> 382</span>  DCHECK(ret);</div> |
| <div class="line"><a name="l00383"></a><span class="lineno"> 383</span>  }</div> |
| <div class="line"><a name="l00384"></a><span class="lineno"> 384</span> </div> |
| <div class="line"><a name="l00385"></a><span class="lineno"> 385</span>  <span class="comment">// Nulls don't get encoded.</span></div> |
| <div class="line"><a name="l00386"></a><span class="lineno"> 386</span>  <span class="keywordflow">if</span> (value == NULL) <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00387"></a><span class="lineno"> 387</span>  ++current_page_->num_non_null;</div> |
| <div class="line"><a name="l00388"></a><span class="lineno"> 388</span> </div> |
| <div class="line"><a name="l00389"></a><span class="lineno"> 389</span>  int64_t bytes_needed = 0;</div> |
| <div class="line"><a name="l00390"></a><span class="lineno"> 390</span>  <span class="keywordflow">if</span> (EncodeValue(value, &bytes_needed)) <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00391"></a><span class="lineno"> 391</span> </div> |
| <div class="line"><a name="l00392"></a><span class="lineno"> 392</span>  <span class="comment">// Value didn't fit on page, try again on a new page.</span></div> |
| <div class="line"><a name="l00393"></a><span class="lineno"> 393</span>  FinalizeCurrentPage();</div> |
| <div class="line"><a name="l00394"></a><span class="lineno"> 394</span> </div> |
| <div class="line"><a name="l00395"></a><span class="lineno"> 395</span>  <span class="comment">// Check how much space it is needed to write this value. If that is larger than the</span></div> |
| <div class="line"><a name="l00396"></a><span class="lineno"> 396</span>  <span class="comment">// page size then increase page size and try again.</span></div> |
| <div class="line"><a name="l00397"></a><span class="lineno"> 397</span>  <span class="keywordflow">if</span> (<a class="code" href="compiler-util_8h.html#a9acc330d508b9a3b775cfdf7ce405e7d">UNLIKELY</a>(bytes_needed > page_size_)) {</div> |
| <div class="line"><a name="l00398"></a><span class="lineno"> 398</span>  page_size_ = bytes_needed;</div> |
| <div class="line"><a name="l00399"></a><span class="lineno"> 399</span>  <span class="keywordflow">if</span> (page_size_ > MAX_DATA_PAGE_SIZE) {</div> |
| <div class="line"><a name="l00400"></a><span class="lineno"> 400</span>  stringstream ss;</div> |
| <div class="line"><a name="l00401"></a><span class="lineno"> 401</span>  ss << <span class="stringliteral">"Cannot write value of size "</span></div> |
| <div class="line"><a name="l00402"></a><span class="lineno"> 402</span>  << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(bytes_needed, TUnit::BYTES) << <span class="stringliteral">" bytes to a Parquet "</span></div> |
| <div class="line"><a name="l00403"></a><span class="lineno"> 403</span>  << <span class="stringliteral">"data page that exceeds the max page limit "</span></div> |
| <div class="line"><a name="l00404"></a><span class="lineno"> 404</span>  << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(MAX_DATA_PAGE_SIZE , TUnit::BYTES) << <span class="stringliteral">"."</span>;</div> |
| <div class="line"><a name="l00405"></a><span class="lineno"> 405</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00406"></a><span class="lineno"> 406</span>  }</div> |
| <div class="line"><a name="l00407"></a><span class="lineno"> 407</span>  values_buffer_len_ = page_size_;</div> |
| <div class="line"><a name="l00408"></a><span class="lineno"> 408</span>  values_buffer_ = parent_->reusable_col_mem_pool_->Allocate(values_buffer_len_);</div> |
| <div class="line"><a name="l00409"></a><span class="lineno"> 409</span>  }</div> |
| <div class="line"><a name="l00410"></a><span class="lineno"> 410</span>  NewPage();</div> |
| <div class="line"><a name="l00411"></a><span class="lineno"> 411</span>  }</div> |
| <div class="line"><a name="l00412"></a><span class="lineno"> 412</span>  ++current_page_->header.data_page_header.num_values;</div> |
| <div class="line"><a name="l00413"></a><span class="lineno"> 413</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00414"></a><span class="lineno"> 414</span> }</div> |
| <div class="line"><a name="l00415"></a><span class="lineno"> 415</span> </div> |
| <div class="line"><a name="l00416"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8608f5639292d04ccab1c9fedb73ea7f"> 416</a></span> <span class="keyword">inline</span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8608f5639292d04ccab1c9fedb73ea7f">HdfsParquetTableWriter::BaseColumnWriter::WriteDictDataPage</a>() {</div> |
| <div class="line"><a name="l00417"></a><span class="lineno"> 417</span>  DCHECK(dict_encoder_base_ != NULL);</div> |
| <div class="line"><a name="l00418"></a><span class="lineno"> 418</span>  DCHECK_EQ(current_page_->header.uncompressed_page_size, 0);</div> |
| <div class="line"><a name="l00419"></a><span class="lineno"> 419</span>  <span class="keywordflow">if</span> (current_page_->num_non_null == 0) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00420"></a><span class="lineno"> 420</span>  <span class="keywordtype">int</span> len = dict_encoder_base_->WriteData(values_buffer_, values_buffer_len_);</div> |
| <div class="line"><a name="l00421"></a><span class="lineno"> 421</span>  <span class="keywordflow">while</span> (<a class="code" href="compiler-util_8h.html#a9acc330d508b9a3b775cfdf7ce405e7d">UNLIKELY</a>(len < 0)) {</div> |
| <div class="line"><a name="l00422"></a><span class="lineno"> 422</span>  <span class="comment">// len < 0 indicates the data doesn't fit into a data page. Allocate a larger data</span></div> |
| <div class="line"><a name="l00423"></a><span class="lineno"> 423</span>  <span class="comment">// page.</span></div> |
| <div class="line"><a name="l00424"></a><span class="lineno"> 424</span>  values_buffer_len_ *= 2;</div> |
| <div class="line"><a name="l00425"></a><span class="lineno"> 425</span>  values_buffer_ = parent_->reusable_col_mem_pool_->Allocate(values_buffer_len_);</div> |
| <div class="line"><a name="l00426"></a><span class="lineno"> 426</span>  len = dict_encoder_base_->WriteData(values_buffer_, values_buffer_len_);</div> |
| <div class="line"><a name="l00427"></a><span class="lineno"> 427</span>  }</div> |
| <div class="line"><a name="l00428"></a><span class="lineno"> 428</span>  dict_encoder_base_->ClearIndices();</div> |
| <div class="line"><a name="l00429"></a><span class="lineno"> 429</span>  current_page_->header.uncompressed_page_size = len;</div> |
| <div class="line"><a name="l00430"></a><span class="lineno"> 430</span> }</div> |
| <div class="line"><a name="l00431"></a><span class="lineno"> 431</span> </div> |
| <div class="line"><a name="l00432"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8ad355e28e35c5d73086b838b6eb56aa"> 432</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8ad355e28e35c5d73086b838b6eb56aa">HdfsParquetTableWriter::BaseColumnWriter::Flush</a>(int64_t* file_pos,</div> |
| <div class="line"><a name="l00433"></a><span class="lineno"> 433</span>  int64_t* first_data_page, int64_t* first_dictionary_page) {</div> |
| <div class="line"><a name="l00434"></a><span class="lineno"> 434</span>  <span class="keywordflow">if</span> (current_page_ == NULL) {</div> |
| <div class="line"><a name="l00435"></a><span class="lineno"> 435</span>  <span class="comment">// This column/file is empty</span></div> |
| <div class="line"><a name="l00436"></a><span class="lineno"> 436</span>  *first_data_page = *file_pos;</div> |
| <div class="line"><a name="l00437"></a><span class="lineno"> 437</span>  *first_dictionary_page = -1;</div> |
| <div class="line"><a name="l00438"></a><span class="lineno"> 438</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00439"></a><span class="lineno"> 439</span>  }</div> |
| <div class="line"><a name="l00440"></a><span class="lineno"> 440</span> </div> |
| <div class="line"><a name="l00441"></a><span class="lineno"> 441</span>  FinalizeCurrentPage();</div> |
| <div class="line"><a name="l00442"></a><span class="lineno"> 442</span> </div> |
| <div class="line"><a name="l00443"></a><span class="lineno"> 443</span>  *first_dictionary_page = -1;</div> |
| <div class="line"><a name="l00444"></a><span class="lineno"> 444</span>  <span class="comment">// First write the dictionary page before any of the data pages.</span></div> |
| <div class="line"><a name="l00445"></a><span class="lineno"> 445</span>  <span class="keywordflow">if</span> (dict_encoder_base_ != NULL) {</div> |
| <div class="line"><a name="l00446"></a><span class="lineno"> 446</span>  *first_dictionary_page = *file_pos;</div> |
| <div class="line"><a name="l00447"></a><span class="lineno"> 447</span>  <span class="comment">// Write dictionary page header</span></div> |
| <div class="line"><a name="l00448"></a><span class="lineno"> 448</span>  DictionaryPageHeader dict_header;</div> |
| <div class="line"><a name="l00449"></a><span class="lineno"> 449</span>  dict_header.num_values = dict_encoder_base_->num_entries();</div> |
| <div class="line"><a name="l00450"></a><span class="lineno"> 450</span>  dict_header.encoding = Encoding::PLAIN_DICTIONARY;</div> |
| <div class="line"><a name="l00451"></a><span class="lineno"> 451</span> </div> |
| <div class="line"><a name="l00452"></a><span class="lineno"> 452</span>  PageHeader header;</div> |
| <div class="line"><a name="l00453"></a><span class="lineno"> 453</span>  header.type = PageType::DICTIONARY_PAGE;</div> |
| <div class="line"><a name="l00454"></a><span class="lineno"> 454</span>  header.uncompressed_page_size = dict_encoder_base_->dict_encoded_size();</div> |
| <div class="line"><a name="l00455"></a><span class="lineno"> 455</span>  header.__set_dictionary_page_header(dict_header);</div> |
| <div class="line"><a name="l00456"></a><span class="lineno"> 456</span> </div> |
| <div class="line"><a name="l00457"></a><span class="lineno"> 457</span>  <span class="comment">// Write the dictionary page data, compressing it if necessary.</span></div> |
| <div class="line"><a name="l00458"></a><span class="lineno"> 458</span>  uint8_t* dict_buffer = parent_->per_file_mem_pool_->Allocate(</div> |
| <div class="line"><a name="l00459"></a><span class="lineno"> 459</span>  header.uncompressed_page_size);</div> |
| <div class="line"><a name="l00460"></a><span class="lineno"> 460</span>  dict_encoder_base_->WriteDict(dict_buffer);</div> |
| <div class="line"><a name="l00461"></a><span class="lineno"> 461</span>  <span class="keywordflow">if</span> (compressor_.get() != NULL) {</div> |
| <div class="line"><a name="l00462"></a><span class="lineno"> 462</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(parent_->parent_->compress_timer());</div> |
| <div class="line"><a name="l00463"></a><span class="lineno"> 463</span>  int64_t max_compressed_size =</div> |
| <div class="line"><a name="l00464"></a><span class="lineno"> 464</span>  compressor_->MaxOutputLen(header.uncompressed_page_size);</div> |
| <div class="line"><a name="l00465"></a><span class="lineno"> 465</span>  DCHECK_GT(max_compressed_size, 0);</div> |
| <div class="line"><a name="l00466"></a><span class="lineno"> 466</span>  uint8_t* compressed_data =</div> |
| <div class="line"><a name="l00467"></a><span class="lineno"> 467</span>  parent_->per_file_mem_pool_->Allocate(max_compressed_size);</div> |
| <div class="line"><a name="l00468"></a><span class="lineno"> 468</span>  header.compressed_page_size = max_compressed_size;</div> |
| <div class="line"><a name="l00469"></a><span class="lineno"> 469</span>  compressor_->ProcessBlock32(<span class="keyword">true</span>, header.uncompressed_page_size, dict_buffer,</div> |
| <div class="line"><a name="l00470"></a><span class="lineno"> 470</span>  &header.compressed_page_size, &compressed_data);</div> |
| <div class="line"><a name="l00471"></a><span class="lineno"> 471</span>  dict_buffer = compressed_data;</div> |
| <div class="line"><a name="l00472"></a><span class="lineno"> 472</span>  <span class="comment">// We allocated the output based on the guessed size, return the extra allocated</span></div> |
| <div class="line"><a name="l00473"></a><span class="lineno"> 473</span>  <span class="comment">// bytes back to the mem pool.</span></div> |
| <div class="line"><a name="l00474"></a><span class="lineno"> 474</span>  parent_->per_file_mem_pool_->ReturnPartialAllocation(</div> |
| <div class="line"><a name="l00475"></a><span class="lineno"> 475</span>  max_compressed_size - header.compressed_page_size);</div> |
| <div class="line"><a name="l00476"></a><span class="lineno"> 476</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00477"></a><span class="lineno"> 477</span>  header.compressed_page_size = header.uncompressed_page_size;</div> |
| <div class="line"><a name="l00478"></a><span class="lineno"> 478</span>  }</div> |
| <div class="line"><a name="l00479"></a><span class="lineno"> 479</span> </div> |
| <div class="line"><a name="l00480"></a><span class="lineno"> 480</span>  uint8_t* header_buffer;</div> |
| <div class="line"><a name="l00481"></a><span class="lineno"> 481</span>  uint32_t header_len;</div> |
| <div class="line"><a name="l00482"></a><span class="lineno"> 482</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(parent_->thrift_serializer_->Serialize(</div> |
| <div class="line"><a name="l00483"></a><span class="lineno"> 483</span>  &header, &header_len, &header_buffer));</div> |
| <div class="line"><a name="l00484"></a><span class="lineno"> 484</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(parent_->Write(header_buffer, header_len));</div> |
| <div class="line"><a name="l00485"></a><span class="lineno"> 485</span>  *file_pos += header_len;</div> |
| <div class="line"><a name="l00486"></a><span class="lineno"> 486</span>  total_compressed_byte_size_ += header_len;</div> |
| <div class="line"><a name="l00487"></a><span class="lineno"> 487</span>  total_uncompressed_byte_size_ += header_len;</div> |
| <div class="line"><a name="l00488"></a><span class="lineno"> 488</span> </div> |
| <div class="line"><a name="l00489"></a><span class="lineno"> 489</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(parent_->Write(dict_buffer, header.compressed_page_size));</div> |
| <div class="line"><a name="l00490"></a><span class="lineno"> 490</span>  *file_pos += header.compressed_page_size;</div> |
| <div class="line"><a name="l00491"></a><span class="lineno"> 491</span>  total_compressed_byte_size_ += header.compressed_page_size;</div> |
| <div class="line"><a name="l00492"></a><span class="lineno"> 492</span>  total_uncompressed_byte_size_ += header.uncompressed_page_size;</div> |
| <div class="line"><a name="l00493"></a><span class="lineno"> 493</span>  }</div> |
| <div class="line"><a name="l00494"></a><span class="lineno"> 494</span> </div> |
| <div class="line"><a name="l00495"></a><span class="lineno"> 495</span>  *first_data_page = *file_pos;</div> |
| <div class="line"><a name="l00496"></a><span class="lineno"> 496</span>  <span class="comment">// Write data pages</span></div> |
| <div class="line"><a name="l00497"></a><span class="lineno"> 497</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < num_data_pages_; ++i) {</div> |
| <div class="line"><a name="l00498"></a><span class="lineno"> 498</span>  <a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html">DataPage</a>& page = pages_[i];</div> |
| <div class="line"><a name="l00499"></a><span class="lineno"> 499</span> </div> |
| <div class="line"><a name="l00500"></a><span class="lineno"> 500</span>  <span class="comment">// Last page might be empty</span></div> |
| <div class="line"><a name="l00501"></a><span class="lineno"> 501</span>  <span class="keywordflow">if</span> (page.<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">header</a>.data_page_header.num_values == 0) {</div> |
| <div class="line"><a name="l00502"></a><span class="lineno"> 502</span>  DCHECK_EQ(page.<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">header</a>.compressed_page_size, 0);</div> |
| <div class="line"><a name="l00503"></a><span class="lineno"> 503</span>  DCHECK_EQ(i, num_data_pages_ - 1);</div> |
| <div class="line"><a name="l00504"></a><span class="lineno"> 504</span>  <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00505"></a><span class="lineno"> 505</span>  }</div> |
| <div class="line"><a name="l00506"></a><span class="lineno"> 506</span> </div> |
| <div class="line"><a name="l00507"></a><span class="lineno"> 507</span>  <span class="comment">// Write data page header</span></div> |
| <div class="line"><a name="l00508"></a><span class="lineno"> 508</span>  uint8_t* buffer = NULL;</div> |
| <div class="line"><a name="l00509"></a><span class="lineno"> 509</span>  uint32_t len = 0;</div> |
| <div class="line"><a name="l00510"></a><span class="lineno"> 510</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(</div> |
| <div class="line"><a name="l00511"></a><span class="lineno"> 511</span>  parent_->thrift_serializer_->Serialize(&page.<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">header</a>, &len, &buffer));</div> |
| <div class="line"><a name="l00512"></a><span class="lineno"> 512</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(parent_->Write(buffer, len));</div> |
| <div class="line"><a name="l00513"></a><span class="lineno"> 513</span>  *file_pos += len;</div> |
| <div class="line"><a name="l00514"></a><span class="lineno"> 514</span> </div> |
| <div class="line"><a name="l00515"></a><span class="lineno"> 515</span>  <span class="comment">// Write the page data</span></div> |
| <div class="line"><a name="l00516"></a><span class="lineno"> 516</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(parent_->Write(page.<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a4b0955451fd2bbcc8ef33f631033c627">data</a>, page.<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">header</a>.compressed_page_size));</div> |
| <div class="line"><a name="l00517"></a><span class="lineno"> 517</span>  *file_pos += page.<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">header</a>.compressed_page_size;</div> |
| <div class="line"><a name="l00518"></a><span class="lineno"> 518</span>  }</div> |
| <div class="line"><a name="l00519"></a><span class="lineno"> 519</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00520"></a><span class="lineno"> 520</span> }</div> |
| <div class="line"><a name="l00521"></a><span class="lineno"> 521</span> </div> |
| <div class="line"><a name="l00522"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a466fbf71b0a2a2b1701e8d1b3e0fa64d"> 522</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a466fbf71b0a2a2b1701e8d1b3e0fa64d">HdfsParquetTableWriter::BaseColumnWriter::FinalizeCurrentPage</a>() {</div> |
| <div class="line"><a name="l00523"></a><span class="lineno"> 523</span>  DCHECK(current_page_ != NULL);</div> |
| <div class="line"><a name="l00524"></a><span class="lineno"> 524</span>  <span class="keywordflow">if</span> (current_page_->finalized) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00525"></a><span class="lineno"> 525</span> </div> |
| <div class="line"><a name="l00526"></a><span class="lineno"> 526</span>  <span class="comment">// If the entire page was NULL, encode it as PLAIN since there is no</span></div> |
| <div class="line"><a name="l00527"></a><span class="lineno"> 527</span>  <span class="comment">// data anyway. We don't output a useless dictionary page and it works</span></div> |
| <div class="line"><a name="l00528"></a><span class="lineno"> 528</span>  <span class="comment">// around a parquet MR bug (see IMPALA-759 for more details).</span></div> |
| <div class="line"><a name="l00529"></a><span class="lineno"> 529</span>  <span class="keywordflow">if</span> (current_page_->num_non_null == 0) current_encoding_ = <a class="code" href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">Encoding::PLAIN</a>;</div> |
| <div class="line"><a name="l00530"></a><span class="lineno"> 530</span> </div> |
| <div class="line"><a name="l00531"></a><span class="lineno"> 531</span>  <span class="keywordflow">if</span> (current_encoding_ == Encoding::PLAIN_DICTIONARY) WriteDictDataPage();</div> |
| <div class="line"><a name="l00532"></a><span class="lineno"> 532</span> </div> |
| <div class="line"><a name="l00533"></a><span class="lineno"> 533</span>  PageHeader& header = current_page_->header;</div> |
| <div class="line"><a name="l00534"></a><span class="lineno"> 534</span>  header.data_page_header.encoding = current_encoding_;</div> |
| <div class="line"><a name="l00535"></a><span class="lineno"> 535</span> </div> |
| <div class="line"><a name="l00536"></a><span class="lineno"> 536</span>  <span class="comment">// Compute size of definition bits</span></div> |
| <div class="line"><a name="l00537"></a><span class="lineno"> 537</span>  def_levels_->Flush();</div> |
| <div class="line"><a name="l00538"></a><span class="lineno"> 538</span>  current_page_->num_def_bytes = <span class="keyword">sizeof</span>(int32_t) + def_levels_->len();</div> |
| <div class="line"><a name="l00539"></a><span class="lineno"> 539</span>  header.uncompressed_page_size += current_page_->num_def_bytes;</div> |
| <div class="line"><a name="l00540"></a><span class="lineno"> 540</span> </div> |
| <div class="line"><a name="l00541"></a><span class="lineno"> 541</span>  <span class="comment">// At this point we know all the data for the data page. Combine them into one buffer.</span></div> |
| <div class="line"><a name="l00542"></a><span class="lineno"> 542</span>  uint8_t* uncompressed_data = NULL;</div> |
| <div class="line"><a name="l00543"></a><span class="lineno"> 543</span>  <span class="keywordflow">if</span> (compressor_.get() == NULL) {</div> |
| <div class="line"><a name="l00544"></a><span class="lineno"> 544</span>  uncompressed_data =</div> |
| <div class="line"><a name="l00545"></a><span class="lineno"> 545</span>  parent_->per_file_mem_pool_->Allocate(header.uncompressed_page_size);</div> |
| <div class="line"><a name="l00546"></a><span class="lineno"> 546</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00547"></a><span class="lineno"> 547</span>  <span class="comment">// We have compression. Combine into the staging buffer.</span></div> |
| <div class="line"><a name="l00548"></a><span class="lineno"> 548</span>  parent_->compression_staging_buffer_.resize(</div> |
| <div class="line"><a name="l00549"></a><span class="lineno"> 549</span>  header.uncompressed_page_size);</div> |
| <div class="line"><a name="l00550"></a><span class="lineno"> 550</span>  uncompressed_data = &parent_->compression_staging_buffer_[0];</div> |
| <div class="line"><a name="l00551"></a><span class="lineno"> 551</span>  }</div> |
| <div class="line"><a name="l00552"></a><span class="lineno"> 552</span> </div> |
| <div class="line"><a name="l00553"></a><span class="lineno"> 553</span>  <a class="code" href="classimpala_1_1BufferBuilder.html">BufferBuilder</a> buffer(uncompressed_data, header.uncompressed_page_size);</div> |
| <div class="line"><a name="l00554"></a><span class="lineno"> 554</span> </div> |
| <div class="line"><a name="l00555"></a><span class="lineno"> 555</span>  <span class="comment">// Copy the definition (null) data</span></div> |
| <div class="line"><a name="l00556"></a><span class="lineno"> 556</span>  <span class="keywordtype">int</span> num_def_level_bytes = def_levels_->len();</div> |
| <div class="line"><a name="l00557"></a><span class="lineno"> 557</span> </div> |
| <div class="line"><a name="l00558"></a><span class="lineno"> 558</span>  buffer.<a class="code" href="classimpala_1_1BufferBuilder.html#a4cc096476e8a7e7fd794023da8f80460">Append</a>(num_def_level_bytes);</div> |
| <div class="line"><a name="l00559"></a><span class="lineno"> 559</span>  buffer.<a class="code" href="classimpala_1_1BufferBuilder.html#a4cc096476e8a7e7fd794023da8f80460">Append</a>(def_levels_->buffer(), num_def_level_bytes);</div> |
| <div class="line"><a name="l00560"></a><span class="lineno"> 560</span>  <span class="comment">// TODO: copy repetition data when we support nested types.</span></div> |
| <div class="line"><a name="l00561"></a><span class="lineno"> 561</span>  buffer.<a class="code" href="classimpala_1_1BufferBuilder.html#a4cc096476e8a7e7fd794023da8f80460">Append</a>(values_buffer_, buffer.<a class="code" href="classimpala_1_1BufferBuilder.html#a64ca6e2b963d5001dd220f98be4c0379">capacity</a>() - buffer.<a class="code" href="classimpala_1_1BufferBuilder.html#a8c1c2548614c9bb70efe407dc3558290">size</a>());</div> |
| <div class="line"><a name="l00562"></a><span class="lineno"> 562</span> </div> |
| <div class="line"><a name="l00563"></a><span class="lineno"> 563</span>  <span class="comment">// Apply compression if necessary</span></div> |
| <div class="line"><a name="l00564"></a><span class="lineno"> 564</span>  <span class="keywordflow">if</span> (compressor_.get() == NULL) {</div> |
| <div class="line"><a name="l00565"></a><span class="lineno"> 565</span>  current_page_->data = <span class="keyword">reinterpret_cast<</span>uint8_t*<span class="keyword">></span>(uncompressed_data);</div> |
| <div class="line"><a name="l00566"></a><span class="lineno"> 566</span>  header.compressed_page_size = header.uncompressed_page_size;</div> |
| <div class="line"><a name="l00567"></a><span class="lineno"> 567</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00568"></a><span class="lineno"> 568</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(parent_->parent_->compress_timer());</div> |
| <div class="line"><a name="l00569"></a><span class="lineno"> 569</span>  int64_t max_compressed_size =</div> |
| <div class="line"><a name="l00570"></a><span class="lineno"> 570</span>  compressor_->MaxOutputLen(header.uncompressed_page_size);</div> |
| <div class="line"><a name="l00571"></a><span class="lineno"> 571</span>  DCHECK_GT(max_compressed_size, 0);</div> |
| <div class="line"><a name="l00572"></a><span class="lineno"> 572</span>  uint8_t* compressed_data = parent_->per_file_mem_pool_->Allocate(max_compressed_size);</div> |
| <div class="line"><a name="l00573"></a><span class="lineno"> 573</span>  header.compressed_page_size = max_compressed_size;</div> |
| <div class="line"><a name="l00574"></a><span class="lineno"> 574</span>  compressor_->ProcessBlock32(<span class="keyword">true</span>, header.uncompressed_page_size, uncompressed_data,</div> |
| <div class="line"><a name="l00575"></a><span class="lineno"> 575</span>  &header.compressed_page_size, &compressed_data);</div> |
| <div class="line"><a name="l00576"></a><span class="lineno"> 576</span>  current_page_->data = compressed_data;</div> |
| <div class="line"><a name="l00577"></a><span class="lineno"> 577</span> </div> |
| <div class="line"><a name="l00578"></a><span class="lineno"> 578</span>  <span class="comment">// We allocated the output based on the guessed size, return the extra allocated</span></div> |
| <div class="line"><a name="l00579"></a><span class="lineno"> 579</span>  <span class="comment">// bytes back to the mem pool.</span></div> |
| <div class="line"><a name="l00580"></a><span class="lineno"> 580</span>  parent_->per_file_mem_pool_->ReturnPartialAllocation(</div> |
| <div class="line"><a name="l00581"></a><span class="lineno"> 581</span>  max_compressed_size - header.compressed_page_size);</div> |
| <div class="line"><a name="l00582"></a><span class="lineno"> 582</span>  }</div> |
| <div class="line"><a name="l00583"></a><span class="lineno"> 583</span> </div> |
| <div class="line"><a name="l00584"></a><span class="lineno"> 584</span>  <span class="comment">// Add the size of the data page header</span></div> |
| <div class="line"><a name="l00585"></a><span class="lineno"> 585</span>  uint8_t* header_buffer;</div> |
| <div class="line"><a name="l00586"></a><span class="lineno"> 586</span>  uint32_t header_len = 0;</div> |
| <div class="line"><a name="l00587"></a><span class="lineno"> 587</span>  parent_->thrift_serializer_->Serialize(</div> |
| <div class="line"><a name="l00588"></a><span class="lineno"> 588</span>  &current_page_->header, &header_len, &header_buffer);</div> |
| <div class="line"><a name="l00589"></a><span class="lineno"> 589</span> </div> |
| <div class="line"><a name="l00590"></a><span class="lineno"> 590</span>  current_page_->finalized = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00591"></a><span class="lineno"> 591</span>  total_compressed_byte_size_ += header_len + header.compressed_page_size;</div> |
| <div class="line"><a name="l00592"></a><span class="lineno"> 592</span>  total_uncompressed_byte_size_ += header_len + header.uncompressed_page_size;</div> |
| <div class="line"><a name="l00593"></a><span class="lineno"> 593</span>  parent_->file_size_estimate_ += header_len + header.compressed_page_size;</div> |
| <div class="line"><a name="l00594"></a><span class="lineno"> 594</span>  def_levels_->Clear();</div> |
| <div class="line"><a name="l00595"></a><span class="lineno"> 595</span> }</div> |
| <div class="line"><a name="l00596"></a><span class="lineno"> 596</span> </div> |
| <div class="line"><a name="l00597"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#aefde0f92c3063e01e162d87dc207a2ab"> 597</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#aefde0f92c3063e01e162d87dc207a2ab">HdfsParquetTableWriter::BaseColumnWriter::NewPage</a>() {</div> |
| <div class="line"><a name="l00598"></a><span class="lineno"> 598</span>  <span class="keywordflow">if</span> (num_data_pages_ < pages_.size()) {</div> |
| <div class="line"><a name="l00599"></a><span class="lineno"> 599</span>  <span class="comment">// Reuse an existing page</span></div> |
| <div class="line"><a name="l00600"></a><span class="lineno"> 600</span>  current_page_ = &pages_[num_data_pages_++];</div> |
| <div class="line"><a name="l00601"></a><span class="lineno"> 601</span>  current_page_->header.data_page_header.num_values = 0;</div> |
| <div class="line"><a name="l00602"></a><span class="lineno"> 602</span>  current_page_->header.compressed_page_size = 0;</div> |
| <div class="line"><a name="l00603"></a><span class="lineno"> 603</span>  current_page_->header.uncompressed_page_size = 0;</div> |
| <div class="line"><a name="l00604"></a><span class="lineno"> 604</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00605"></a><span class="lineno"> 605</span>  pages_.push_back(<a class="code" href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html">DataPage</a>());</div> |
| <div class="line"><a name="l00606"></a><span class="lineno"> 606</span>  current_page_ = &pages_[num_data_pages_++];</div> |
| <div class="line"><a name="l00607"></a><span class="lineno"> 607</span> </div> |
| <div class="line"><a name="l00608"></a><span class="lineno"> 608</span>  DataPageHeader header;</div> |
| <div class="line"><a name="l00609"></a><span class="lineno"> 609</span>  header.num_values = 0;</div> |
| <div class="line"><a name="l00610"></a><span class="lineno"> 610</span>  header.definition_level_encoding = Encoding::RLE;</div> |
| <div class="line"><a name="l00611"></a><span class="lineno"> 611</span>  header.repetition_level_encoding = Encoding::BIT_PACKED;</div> |
| <div class="line"><a name="l00612"></a><span class="lineno"> 612</span>  current_page_->header.__set_data_page_header(header);</div> |
| <div class="line"><a name="l00613"></a><span class="lineno"> 613</span>  }</div> |
| <div class="line"><a name="l00614"></a><span class="lineno"> 614</span>  current_page_->finalized = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00615"></a><span class="lineno"> 615</span>  current_page_->num_non_null = 0;</div> |
| <div class="line"><a name="l00616"></a><span class="lineno"> 616</span> }</div> |
| <div class="line"><a name="l00617"></a><span class="lineno"> 617</span> </div> |
| <div class="line"><a name="l00618"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a2134f91402d30cc178526c949c00603c"> 618</a></span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a2134f91402d30cc178526c949c00603c">HdfsParquetTableWriter::HdfsParquetTableWriter</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html">HdfsTableSink</a>* parent, <a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state,</div> |
| <div class="line"><a name="l00619"></a><span class="lineno"> 619</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output, <span class="keyword">const</span> <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>* part_desc,</div> |
| <div class="line"><a name="l00620"></a><span class="lineno"> 620</span>  <span class="keyword">const</span> <a class="code" href="classimpala_1_1HdfsTableDescriptor.html">HdfsTableDescriptor</a>* table_desc, <span class="keyword">const</span> vector<ExprContext*>& output_expr_ctxs)</div> |
| <div class="line"><a name="l00621"></a><span class="lineno"> 621</span>  : <a class="code" href="classimpala_1_1HdfsTableWriter.html">HdfsTableWriter</a>(</div> |
| <div class="line"><a name="l00622"></a><span class="lineno"> 622</span>  parent, state, output, part_desc, table_desc, output_expr_ctxs),</div> |
| <div class="line"><a name="l00623"></a><span class="lineno"> 623</span>  thrift_serializer_(new <a class="code" href="classimpala_1_1ThriftSerializer.html">ThriftSerializer</a>(true)),</div> |
| <div class="line"><a name="l00624"></a><span class="lineno"> 624</span>  current_row_group_(NULL),</div> |
| <div class="line"><a name="l00625"></a><span class="lineno"> 625</span>  row_count_(0),</div> |
| <div class="line"><a name="l00626"></a><span class="lineno"> 626</span>  file_size_limit_(0),</div> |
| <div class="line"><a name="l00627"></a><span class="lineno"> 627</span>  reusable_col_mem_pool_(new <a class="code" href="classimpala_1_1MemPool.html">MemPool</a>(parent_->mem_tracker())),</div> |
| <div class="line"><a name="l00628"></a><span class="lineno"> 628</span>  per_file_mem_pool_(new <a class="code" href="classimpala_1_1MemPool.html">MemPool</a>(parent_->mem_tracker())),</div> |
| <div class="line"><a name="l00629"></a><span class="lineno"> 629</span>  row_idx_(0) {</div> |
| <div class="line"><a name="l00630"></a><span class="lineno"> 630</span> }</div> |
| <div class="line"><a name="l00631"></a><span class="lineno"> 631</span> </div> |
| <div class="line"><a name="l00632"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#afa31a28f32a0b57ebaebe319846cdd43"> 632</a></span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#afa31a28f32a0b57ebaebe319846cdd43">HdfsParquetTableWriter::~HdfsParquetTableWriter</a>() {</div> |
| <div class="line"><a name="l00633"></a><span class="lineno"> 633</span> }</div> |
| <div class="line"><a name="l00634"></a><span class="lineno"> 634</span> </div> |
| <div class="line"><a name="l00635"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a9725689b4baf53907247e60778283094"> 635</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a9725689b4baf53907247e60778283094">HdfsParquetTableWriter::Init</a>() {</div> |
| <div class="line"><a name="l00636"></a><span class="lineno"> 636</span>  <span class="comment">// Initialize file metadata</span></div> |
| <div class="line"><a name="l00637"></a><span class="lineno"> 637</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.version = <a class="code" href="namespaceimpala.html#a23c0355951ae6f3602e935a0688da7dc">PARQUET_CURRENT_VERSION</a>;</div> |
| <div class="line"><a name="l00638"></a><span class="lineno"> 638</span> </div> |
| <div class="line"><a name="l00639"></a><span class="lineno"> 639</span>  stringstream created_by;</div> |
| <div class="line"><a name="l00640"></a><span class="lineno"> 640</span>  created_by << <span class="stringliteral">"impala version "</span> << <a class="code" href="version_8h.html#a6cfbcb9525a23b464946d05af6177421">IMPALA_BUILD_VERSION</a></div> |
| <div class="line"><a name="l00641"></a><span class="lineno"> 641</span>  << <span class="stringliteral">" (build "</span> << <a class="code" href="version_8h.html#af38fbd78d9901e8a11ab4b413edd5bca">IMPALA_BUILD_HASH</a> << <span class="stringliteral">")"</span>;</div> |
| <div class="line"><a name="l00642"></a><span class="lineno"> 642</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.__set_created_by(created_by.str());</div> |
| <div class="line"><a name="l00643"></a><span class="lineno"> 643</span> </div> |
| <div class="line"><a name="l00644"></a><span class="lineno"> 644</span>  <span class="comment">// Default to snappy compressed</span></div> |
| <div class="line"><a name="l00645"></a><span class="lineno"> 645</span>  THdfsCompression::type codec = THdfsCompression::SNAPPY;</div> |
| <div class="line"><a name="l00646"></a><span class="lineno"> 646</span> </div> |
| <div class="line"><a name="l00647"></a><span class="lineno"> 647</span>  <span class="keyword">const</span> TQueryOptions& query_options = <a class="code" href="classimpala_1_1HdfsTableWriter.html#af0582e210619f97f5ac5db62669bbc40">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>();</div> |
| <div class="line"><a name="l00648"></a><span class="lineno"> 648</span>  <span class="keywordflow">if</span> (query_options.__isset.compression_codec) {</div> |
| <div class="line"><a name="l00649"></a><span class="lineno"> 649</span>  codec = query_options.compression_codec;</div> |
| <div class="line"><a name="l00650"></a><span class="lineno"> 650</span>  }</div> |
| <div class="line"><a name="l00651"></a><span class="lineno"> 651</span>  <span class="keywordflow">if</span> (!(codec == THdfsCompression::NONE ||</div> |
| <div class="line"><a name="l00652"></a><span class="lineno"> 652</span>  codec == THdfsCompression::GZIP ||</div> |
| <div class="line"><a name="l00653"></a><span class="lineno"> 653</span>  codec == THdfsCompression::SNAPPY)) {</div> |
| <div class="line"><a name="l00654"></a><span class="lineno"> 654</span>  stringstream ss;</div> |
| <div class="line"><a name="l00655"></a><span class="lineno"> 655</span>  ss << <span class="stringliteral">"Invalid parquet compression codec "</span> << <a class="code" href="classimpala_1_1Codec.html#a8e34e69fbf492c3c03cd417f68f75bd4">Codec::GetCodecName</a>(codec);</div> |
| <div class="line"><a name="l00656"></a><span class="lineno"> 656</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00657"></a><span class="lineno"> 657</span>  }</div> |
| <div class="line"><a name="l00658"></a><span class="lineno"> 658</span> </div> |
| <div class="line"><a name="l00659"></a><span class="lineno"> 659</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"Using compression codec: "</span> << codec;</div> |
| <div class="line"><a name="l00660"></a><span class="lineno"> 660</span> </div> |
| <div class="line"><a name="l00661"></a><span class="lineno"> 661</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.resize(<a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#a665ceaf9c184ec39544c3ecc448c0d11">num_cols</a>() - <a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">num_clustering_cols</a>());</div> |
| <div class="line"><a name="l00662"></a><span class="lineno"> 662</span>  <span class="comment">// Initialize each column structure.</span></div> |
| <div class="line"><a name="l00663"></a><span class="lineno"> 663</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00664"></a><span class="lineno"> 664</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">BaseColumnWriter</a>* writer = NULL;</div> |
| <div class="line"><a name="l00665"></a><span class="lineno"> 665</span>  <span class="keyword">const</span> <a class="code" href="structimpala_1_1ColumnType.html">ColumnType</a>& type = <a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i]->root()->type();</div> |
| <div class="line"><a name="l00666"></a><span class="lineno"> 666</span>  <span class="keywordflow">switch</span> (type.<a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a>) {</div> |
| <div class="line"><a name="l00667"></a><span class="lineno"> 667</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a936286363e052834631db5c19be3187d">TYPE_BOOLEAN</a>:</div> |
| <div class="line"><a name="l00668"></a><span class="lineno"> 668</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a749e28247ff20797c6b920c1c892fd14">BoolColumnWriter</a>(</div> |
| <div class="line"><a name="l00669"></a><span class="lineno"> 669</span>  <span class="keyword">this</span>, <a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i], codec);</div> |
| <div class="line"><a name="l00670"></a><span class="lineno"> 670</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00671"></a><span class="lineno"> 671</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a7a19f69a9969431237f11ac0f982add1">TYPE_TINYINT</a>:</div> |
| <div class="line"><a name="l00672"></a><span class="lineno"> 672</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<int8_t></a>(</div> |
| <div class="line"><a name="l00673"></a><span class="lineno"> 673</span>  <span class="keyword">this</span>, <a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i], codec);</div> |
| <div class="line"><a name="l00674"></a><span class="lineno"> 674</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00675"></a><span class="lineno"> 675</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ad19c24e4b9c67ed07a2331e704cf08ba">TYPE_SMALLINT</a>:</div> |
| <div class="line"><a name="l00676"></a><span class="lineno"> 676</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<int16_t></a>(</div> |
| <div class="line"><a name="l00677"></a><span class="lineno"> 677</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00678"></a><span class="lineno"> 678</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00679"></a><span class="lineno"> 679</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a8e892339c263b2aaa3712139b0a0c944">TYPE_INT</a>:</div> |
| <div class="line"><a name="l00680"></a><span class="lineno"> 680</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<int32_t></a>(</div> |
| <div class="line"><a name="l00681"></a><span class="lineno"> 681</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00682"></a><span class="lineno"> 682</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00683"></a><span class="lineno"> 683</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ac5602f92ce48738739f551df8ff3e4a7">TYPE_BIGINT</a>:</div> |
| <div class="line"><a name="l00684"></a><span class="lineno"> 684</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<int64_t></a>(</div> |
| <div class="line"><a name="l00685"></a><span class="lineno"> 685</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00686"></a><span class="lineno"> 686</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00687"></a><span class="lineno"> 687</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ac750a41f3696ba9ddece634df5452994">TYPE_FLOAT</a>:</div> |
| <div class="line"><a name="l00688"></a><span class="lineno"> 688</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<float></a>(</div> |
| <div class="line"><a name="l00689"></a><span class="lineno"> 689</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00690"></a><span class="lineno"> 690</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00691"></a><span class="lineno"> 691</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a40a3023d094eb5a5a8c7335aa27bb810">TYPE_DOUBLE</a>:</div> |
| <div class="line"><a name="l00692"></a><span class="lineno"> 692</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<double></a>(</div> |
| <div class="line"><a name="l00693"></a><span class="lineno"> 693</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00694"></a><span class="lineno"> 694</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00695"></a><span class="lineno"> 695</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871afdce7fe1e22d49d6fd07b168cb5b4f7b">TYPE_TIMESTAMP</a>:</div> |
| <div class="line"><a name="l00696"></a><span class="lineno"> 696</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<TimestampValue></a>(</div> |
| <div class="line"><a name="l00697"></a><span class="lineno"> 697</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00698"></a><span class="lineno"> 698</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00699"></a><span class="lineno"> 699</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a12d80a0bfe690580f4974e8baca466b0">TYPE_VARCHAR</a>:</div> |
| <div class="line"><a name="l00700"></a><span class="lineno"> 700</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a81087da87fdc569ea9ad101e34391359">TYPE_STRING</a>:</div> |
| <div class="line"><a name="l00701"></a><span class="lineno"> 701</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ae5ec6e333587a27f8b83c5bd9c815a2a">TYPE_CHAR</a>:</div> |
| <div class="line"><a name="l00702"></a><span class="lineno"> 702</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<StringValue></a>(</div> |
| <div class="line"><a name="l00703"></a><span class="lineno"> 703</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00704"></a><span class="lineno"> 704</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00705"></a><span class="lineno"> 705</span>  <span class="keywordflow">case</span> <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a62719f90a13430bfdc4a8455724f6fdf">TYPE_DECIMAL</a>:</div> |
| <div class="line"><a name="l00706"></a><span class="lineno"> 706</span>  <span class="keywordflow">switch</span> (output_expr_ctxs_[i]->root()-><a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a7ea48982ea52afc394148b5813afe9b7">type</a>().<a class="code" href="structimpala_1_1ColumnType.html#a6fbb7face292b91017b89af34defcad9">GetByteSize</a>()) {</div> |
| <div class="line"><a name="l00707"></a><span class="lineno"> 707</span>  <span class="keywordflow">case</span> 4:</div> |
| <div class="line"><a name="l00708"></a><span class="lineno"> 708</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<Decimal4Value></a>(</div> |
| <div class="line"><a name="l00709"></a><span class="lineno"> 709</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00710"></a><span class="lineno"> 710</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00711"></a><span class="lineno"> 711</span>  <span class="keywordflow">case</span> 8:</div> |
| <div class="line"><a name="l00712"></a><span class="lineno"> 712</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<Decimal8Value></a>(</div> |
| <div class="line"><a name="l00713"></a><span class="lineno"> 713</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00714"></a><span class="lineno"> 714</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00715"></a><span class="lineno"> 715</span>  <span class="keywordflow">case</span> 16:</div> |
| <div class="line"><a name="l00716"></a><span class="lineno"> 716</span>  writer = <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">ColumnWriter<Decimal16Value></a>(</div> |
| <div class="line"><a name="l00717"></a><span class="lineno"> 717</span>  <span class="keyword">this</span>, output_expr_ctxs_[i], codec);</div> |
| <div class="line"><a name="l00718"></a><span class="lineno"> 718</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00719"></a><span class="lineno"> 719</span>  <span class="keywordflow">default</span>:</div> |
| <div class="line"><a name="l00720"></a><span class="lineno"> 720</span>  DCHECK(<span class="keyword">false</span>);</div> |
| <div class="line"><a name="l00721"></a><span class="lineno"> 721</span>  }</div> |
| <div class="line"><a name="l00722"></a><span class="lineno"> 722</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00723"></a><span class="lineno"> 723</span>  <span class="keywordflow">default</span>:</div> |
| <div class="line"><a name="l00724"></a><span class="lineno"> 724</span>  DCHECK(<span class="keyword">false</span>);</div> |
| <div class="line"><a name="l00725"></a><span class="lineno"> 725</span>  }</div> |
| <div class="line"><a name="l00726"></a><span class="lineno"> 726</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i] = <a class="code" href="classimpala_1_1HdfsTableWriter.html#af0582e210619f97f5ac5db62669bbc40">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(writer);</div> |
| <div class="line"><a name="l00727"></a><span class="lineno"> 727</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->Reset();</div> |
| <div class="line"><a name="l00728"></a><span class="lineno"> 728</span>  }</div> |
| <div class="line"><a name="l00729"></a><span class="lineno"> 729</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a77fca6b59dc2094950262e1eebb6d1c6">CreateSchema</a>());</div> |
| <div class="line"><a name="l00730"></a><span class="lineno"> 730</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00731"></a><span class="lineno"> 731</span> }</div> |
| <div class="line"><a name="l00732"></a><span class="lineno"> 732</span> </div> |
| <div class="line"><a name="l00733"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a77fca6b59dc2094950262e1eebb6d1c6"> 733</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a77fca6b59dc2094950262e1eebb6d1c6">HdfsParquetTableWriter::CreateSchema</a>() {</div> |
| <div class="line"><a name="l00734"></a><span class="lineno"> 734</span>  <span class="keywordtype">int</span> num_clustering_cols = <a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">num_clustering_cols</a>();</div> |
| <div class="line"><a name="l00735"></a><span class="lineno"> 735</span> </div> |
| <div class="line"><a name="l00736"></a><span class="lineno"> 736</span>  <span class="comment">// Create flattened tree with a single root.</span></div> |
| <div class="line"><a name="l00737"></a><span class="lineno"> 737</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.schema.resize(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size() + 1);</div> |
| <div class="line"><a name="l00738"></a><span class="lineno"> 738</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.schema[0].__set_num_children(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size());</div> |
| <div class="line"><a name="l00739"></a><span class="lineno"> 739</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.schema[0].name = <span class="stringliteral">"schema"</span>;</div> |
| <div class="line"><a name="l00740"></a><span class="lineno"> 740</span> </div> |
| <div class="line"><a name="l00741"></a><span class="lineno"> 741</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00742"></a><span class="lineno"> 742</span>  parquet::SchemaElement& node = <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.schema[i + 1];</div> |
| <div class="line"><a name="l00743"></a><span class="lineno"> 743</span>  node.name = <a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">col_names</a>()[i + num_clustering_cols];</div> |
| <div class="line"><a name="l00744"></a><span class="lineno"> 744</span>  node.__set_type(<a class="code" href="namespaceimpala.html#a9e083b0deadc8fd39e06d065b4d66b7b">IMPALA_TO_PARQUET_TYPES</a>[<a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i]->root()->type().type]);</div> |
| <div class="line"><a name="l00745"></a><span class="lineno"> 745</span>  node.__set_repetition_type(FieldRepetitionType::OPTIONAL);</div> |
| <div class="line"><a name="l00746"></a><span class="lineno"> 746</span>  <span class="keyword">const</span> <a class="code" href="structimpala_1_1ColumnType.html">ColumnType</a>& type = <a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i]->root()->type();</div> |
| <div class="line"><a name="l00747"></a><span class="lineno"> 747</span>  <span class="keywordflow">if</span> (type.<a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a> == <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a62719f90a13430bfdc4a8455724f6fdf">TYPE_DECIMAL</a>) {</div> |
| <div class="line"><a name="l00748"></a><span class="lineno"> 748</span>  <span class="comment">// This column is type decimal. Update the file metadata to include the</span></div> |
| <div class="line"><a name="l00749"></a><span class="lineno"> 749</span>  <span class="comment">// additional fields:</span></div> |
| <div class="line"><a name="l00750"></a><span class="lineno"> 750</span>  <span class="comment">// 1) converted_type: indicate this is really a decimal column.</span></div> |
| <div class="line"><a name="l00751"></a><span class="lineno"> 751</span>  <span class="comment">// 2) type_length: the number of bytes used per decimal value in the data</span></div> |
| <div class="line"><a name="l00752"></a><span class="lineno"> 752</span>  <span class="comment">// 3) precision/scale</span></div> |
| <div class="line"><a name="l00753"></a><span class="lineno"> 753</span>  node.__set_converted_type(ConvertedType::DECIMAL);</div> |
| <div class="line"><a name="l00754"></a><span class="lineno"> 754</span>  node.__set_type_length(</div> |
| <div class="line"><a name="l00755"></a><span class="lineno"> 755</span>  <a class="code" href="classimpala_1_1ParquetPlainEncoder.html#a5f888cf60ea0a62b73c9a3cb9e67dac0">ParquetPlainEncoder::DecimalSize</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i]->root()->type()));</div> |
| <div class="line"><a name="l00756"></a><span class="lineno"> 756</span>  node.__set_scale(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i]->root()->type().scale);</div> |
| <div class="line"><a name="l00757"></a><span class="lineno"> 757</span>  node.__set_precision(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">output_expr_ctxs_</a>[i]->root()->type().precision);</div> |
| <div class="line"><a name="l00758"></a><span class="lineno"> 758</span>  } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (type.<a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a> == <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a12d80a0bfe690580f4974e8baca466b0">TYPE_VARCHAR</a> || type.<a class="code" href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">type</a> == <a class="code" href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ae5ec6e333587a27f8b83c5bd9c815a2a">TYPE_CHAR</a>) {</div> |
| <div class="line"><a name="l00759"></a><span class="lineno"> 759</span>  node.__set_converted_type(ConvertedType::UTF8);</div> |
| <div class="line"><a name="l00760"></a><span class="lineno"> 760</span>  }</div> |
| <div class="line"><a name="l00761"></a><span class="lineno"> 761</span>  }</div> |
| <div class="line"><a name="l00762"></a><span class="lineno"> 762</span> </div> |
| <div class="line"><a name="l00763"></a><span class="lineno"> 763</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00764"></a><span class="lineno"> 764</span> }</div> |
| <div class="line"><a name="l00765"></a><span class="lineno"> 765</span> </div> |
| <div class="line"><a name="l00766"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a507adb3975fd7bc994da72f5777a3ef7"> 766</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a507adb3975fd7bc994da72f5777a3ef7">HdfsParquetTableWriter::AddRowGroup</a>() {</div> |
| <div class="line"><a name="l00767"></a><span class="lineno"> 767</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a> != NULL) <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a74b2e94e82156cfdaa1d140cc5d512e7">FlushCurrentRowGroup</a>());</div> |
| <div class="line"><a name="l00768"></a><span class="lineno"> 768</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.row_groups.push_back(RowGroup());</div> |
| <div class="line"><a name="l00769"></a><span class="lineno"> 769</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a> = &<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.row_groups[<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.row_groups.size() - 1];</div> |
| <div class="line"><a name="l00770"></a><span class="lineno"> 770</span> </div> |
| <div class="line"><a name="l00771"></a><span class="lineno"> 771</span>  <span class="comment">// Initialize new row group metadata.</span></div> |
| <div class="line"><a name="l00772"></a><span class="lineno"> 772</span>  <span class="keywordtype">int</span> num_clustering_cols = <a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">num_clustering_cols</a>();</div> |
| <div class="line"><a name="l00773"></a><span class="lineno"> 773</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns.resize(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size());</div> |
| <div class="line"><a name="l00774"></a><span class="lineno"> 774</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00775"></a><span class="lineno"> 775</span>  ColumnMetaData metadata;</div> |
| <div class="line"><a name="l00776"></a><span class="lineno"> 776</span>  metadata.type = <a class="code" href="namespaceimpala.html#a9e083b0deadc8fd39e06d065b4d66b7b">IMPALA_TO_PARQUET_TYPES</a>[<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->expr_ctx_->root()->type().type];</div> |
| <div class="line"><a name="l00777"></a><span class="lineno"> 777</span>  <span class="comment">// Add all encodings that were used in this file. Currently we use PLAIN and</span></div> |
| <div class="line"><a name="l00778"></a><span class="lineno"> 778</span>  <span class="comment">// PLAIN_DICTIONARY for data values and RLE for the definition levels.</span></div> |
| <div class="line"><a name="l00779"></a><span class="lineno"> 779</span>  metadata.encodings.push_back(Encoding::RLE);</div> |
| <div class="line"><a name="l00780"></a><span class="lineno"> 780</span>  <span class="comment">// Columns are initially dictionary encoded</span></div> |
| <div class="line"><a name="l00781"></a><span class="lineno"> 781</span>  <span class="comment">// TODO: we might not have PLAIN encoding in this case</span></div> |
| <div class="line"><a name="l00782"></a><span class="lineno"> 782</span>  metadata.encodings.push_back(Encoding::PLAIN_DICTIONARY);</div> |
| <div class="line"><a name="l00783"></a><span class="lineno"> 783</span>  metadata.encodings.push_back(<a class="code" href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">Encoding::PLAIN</a>);</div> |
| <div class="line"><a name="l00784"></a><span class="lineno"> 784</span>  metadata.path_in_schema.push_back(<a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">col_names</a>()[i + num_clustering_cols]);</div> |
| <div class="line"><a name="l00785"></a><span class="lineno"> 785</span>  metadata.codec = columns_[i]->codec();</div> |
| <div class="line"><a name="l00786"></a><span class="lineno"> 786</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].__set_meta_data(metadata);</div> |
| <div class="line"><a name="l00787"></a><span class="lineno"> 787</span>  }</div> |
| <div class="line"><a name="l00788"></a><span class="lineno"> 788</span> </div> |
| <div class="line"><a name="l00789"></a><span class="lineno"> 789</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00790"></a><span class="lineno"> 790</span> }</div> |
| <div class="line"><a name="l00791"></a><span class="lineno"> 791</span> </div> |
| <div class="line"><a name="l00792"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a899619de5a18c34129aed0b5ad217af5"> 792</a></span> int64_t <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a899619de5a18c34129aed0b5ad217af5">HdfsParquetTableWriter::MinBlockSize</a>()<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l00793"></a><span class="lineno"> 793</span>  <span class="comment">// See file_size_limit_ calculation in InitNewFile().</span></div> |
| <div class="line"><a name="l00794"></a><span class="lineno"> 794</span>  <span class="keywordflow">return</span> 3 * <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a633dd0324c1b260dd46ae81ff5970277">DEFAULT_DATA_PAGE_SIZE</a> * <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size();</div> |
| <div class="line"><a name="l00795"></a><span class="lineno"> 795</span> }</div> |
| <div class="line"><a name="l00796"></a><span class="lineno"> 796</span> </div> |
| <div class="line"><a name="l00797"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a3e94af6168818406871e631b54287ff6"> 797</a></span> <a class="code" href="classuint64__t.html">uint64_t</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3e94af6168818406871e631b54287ff6">HdfsParquetTableWriter::default_block_size</a>()<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l00798"></a><span class="lineno"> 798</span>  int64_t block_size;</div> |
| <div class="line"><a name="l00799"></a><span class="lineno"> 799</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsTableWriter.html#af0582e210619f97f5ac5db62669bbc40">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().__isset.parquet_file_size &&</div> |
| <div class="line"><a name="l00800"></a><span class="lineno"> 800</span>  <a class="code" href="classimpala_1_1HdfsTableWriter.html#af0582e210619f97f5ac5db62669bbc40">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().parquet_file_size > 0) {</div> |
| <div class="line"><a name="l00801"></a><span class="lineno"> 801</span>  <span class="comment">// If the user specified a value explicitly, use it. InitNewFile() will verify that</span></div> |
| <div class="line"><a name="l00802"></a><span class="lineno"> 802</span>  <span class="comment">// the actual file's block size is sufficient.</span></div> |
| <div class="line"><a name="l00803"></a><span class="lineno"> 803</span>  block_size = <a class="code" href="classimpala_1_1HdfsTableWriter.html#af0582e210619f97f5ac5db62669bbc40">state_</a>-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().parquet_file_size;</div> |
| <div class="line"><a name="l00804"></a><span class="lineno"> 804</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00805"></a><span class="lineno"> 805</span>  block_size = <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ad9da7d51aeeb791ff5d3e14039aa58bd">HDFS_BLOCK_SIZE</a>;</div> |
| <div class="line"><a name="l00806"></a><span class="lineno"> 806</span>  <span class="comment">// Blocks are usually HDFS_BLOCK_SIZE bytes, unless there are many columns, in</span></div> |
| <div class="line"><a name="l00807"></a><span class="lineno"> 807</span>  <span class="comment">// which case a per-column minimum kicks in.</span></div> |
| <div class="line"><a name="l00808"></a><span class="lineno"> 808</span>  block_size = max(block_size, <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a899619de5a18c34129aed0b5ad217af5">MinBlockSize</a>());</div> |
| <div class="line"><a name="l00809"></a><span class="lineno"> 809</span>  }</div> |
| <div class="line"><a name="l00810"></a><span class="lineno"> 810</span>  <span class="comment">// HDFS does not like block sizes that are not aligned</span></div> |
| <div class="line"><a name="l00811"></a><span class="lineno"> 811</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1BitUtil.html#aaf681500e2dd65a55beb6e5628c66a77">BitUtil::RoundUp</a>(block_size, <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a069c640daf028ed2b1da6af395d388c0">HDFS_BLOCK_ALIGNMENT</a>);</div> |
| <div class="line"><a name="l00812"></a><span class="lineno"> 812</span> }</div> |
| <div class="line"><a name="l00813"></a><span class="lineno"> 813</span> </div> |
| <div class="line"><a name="l00814"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#aed34565a4d5ac112ec1c24935314931d"> 814</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aed34565a4d5ac112ec1c24935314931d">HdfsParquetTableWriter::InitNewFile</a>() {</div> |
| <div class="line"><a name="l00815"></a><span class="lineno"> 815</span>  DCHECK(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a> == NULL);</div> |
| <div class="line"><a name="l00816"></a><span class="lineno"> 816</span> </div> |
| <div class="line"><a name="l00817"></a><span class="lineno"> 817</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a67a6abe5cd883902e5adddea9c68f54e">per_file_mem_pool_</a>->Clear();</div> |
| <div class="line"><a name="l00818"></a><span class="lineno"> 818</span> </div> |
| <div class="line"><a name="l00819"></a><span class="lineno"> 819</span>  <span class="comment">// Get the file limit</span></div> |
| <div class="line"><a name="l00820"></a><span class="lineno"> 820</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#abf1f8a799d6f3fe3c1d8539537791a50">HdfsTableSink::GetFileBlockSize</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a25eed1b5b3cb444d8bfdc62890238ff7">output_</a>, &<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a>));</div> |
| <div class="line"><a name="l00821"></a><span class="lineno"> 821</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a> < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a7a904ead6161c202460dd0fa4efd9104">HDFS_MIN_FILE_SIZE</a>) {</div> |
| <div class="line"><a name="l00822"></a><span class="lineno"> 822</span>  stringstream ss;</div> |
| <div class="line"><a name="l00823"></a><span class="lineno"> 823</span>  ss << <span class="stringliteral">"Hdfs file size ("</span> << <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a> << <span class="stringliteral">") is too small."</span>;</div> |
| <div class="line"><a name="l00824"></a><span class="lineno"> 824</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00825"></a><span class="lineno"> 825</span>  }</div> |
| <div class="line"><a name="l00826"></a><span class="lineno"> 826</span> </div> |
| <div class="line"><a name="l00827"></a><span class="lineno"> 827</span>  <span class="comment">// We want to output HDFS files that are no more than file_size_limit_. If we</span></div> |
| <div class="line"><a name="l00828"></a><span class="lineno"> 828</span>  <span class="comment">// go over the limit, HDFS will split the file into multiple blocks which</span></div> |
| <div class="line"><a name="l00829"></a><span class="lineno"> 829</span>  <span class="comment">// is undesirable. We are under the limit, we potentially end up with more</span></div> |
| <div class="line"><a name="l00830"></a><span class="lineno"> 830</span>  <span class="comment">// files than necessary. Either way, it is not going to generate a invalid</span></div> |
| <div class="line"><a name="l00831"></a><span class="lineno"> 831</span>  <span class="comment">// file.</span></div> |
| <div class="line"><a name="l00832"></a><span class="lineno"> 832</span>  <span class="comment">// With arbitrary encoding schemes, it is not possible to know if appending</span></div> |
| <div class="line"><a name="l00833"></a><span class="lineno"> 833</span>  <span class="comment">// a new row will push us over the limit until after encoding it. Rolling back</span></div> |
| <div class="line"><a name="l00834"></a><span class="lineno"> 834</span>  <span class="comment">// a row can be tricky as well so instead we will stop the file when it is</span></div> |
| <div class="line"><a name="l00835"></a><span class="lineno"> 835</span>  <span class="comment">// 2 * DEFAULT_DATA_PAGE_SIZE * num_cols short of the limit. e.g. 50 cols with 8K data</span></div> |
| <div class="line"><a name="l00836"></a><span class="lineno"> 836</span>  <span class="comment">// pages, means we stop 800KB shy of the limit.</span></div> |
| <div class="line"><a name="l00837"></a><span class="lineno"> 837</span>  <span class="comment">// Data pages calculate their size precisely when they are complete so having</span></div> |
| <div class="line"><a name="l00838"></a><span class="lineno"> 838</span>  <span class="comment">// a two page buffer guarantees we will never go over (unless there are huge values</span></div> |
| <div class="line"><a name="l00839"></a><span class="lineno"> 839</span>  <span class="comment">// that require increasing the page size).</span></div> |
| <div class="line"><a name="l00840"></a><span class="lineno"> 840</span>  <span class="comment">// TODO: this should be made dynamic based on the size of rows seen so far.</span></div> |
| <div class="line"><a name="l00841"></a><span class="lineno"> 841</span>  <span class="comment">// This would for example, let us account for very long string columns.</span></div> |
| <div class="line"><a name="l00842"></a><span class="lineno"> 842</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a> < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a899619de5a18c34129aed0b5ad217af5">MinBlockSize</a>()) {</div> |
| <div class="line"><a name="l00843"></a><span class="lineno"> 843</span>  stringstream ss;</div> |
| <div class="line"><a name="l00844"></a><span class="lineno"> 844</span>  ss << <span class="stringliteral">"Parquet file size "</span> << <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a> << <span class="stringliteral">" bytes is too small for "</span></div> |
| <div class="line"><a name="l00845"></a><span class="lineno"> 845</span>  << <span class="stringliteral">"a table with "</span> << <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size() << <span class="stringliteral">" columns. Set query option "</span></div> |
| <div class="line"><a name="l00846"></a><span class="lineno"> 846</span>  << <span class="stringliteral">"PARQUET_FILE_SIZE to at least "</span> << <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a899619de5a18c34129aed0b5ad217af5">MinBlockSize</a>() << <span class="stringliteral">"."</span>;</div> |
| <div class="line"><a name="l00847"></a><span class="lineno"> 847</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00848"></a><span class="lineno"> 848</span>  }</div> |
| <div class="line"><a name="l00849"></a><span class="lineno"> 849</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a> -= 2 * <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a633dd0324c1b260dd46ae81ff5970277">DEFAULT_DATA_PAGE_SIZE</a> * <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size();</div> |
| <div class="line"><a name="l00850"></a><span class="lineno"> 850</span>  DCHECK_GE(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a>, <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a633dd0324c1b260dd46ae81ff5970277">DEFAULT_DATA_PAGE_SIZE</a> * <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size());</div> |
| <div class="line"><a name="l00851"></a><span class="lineno"> 851</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">file_pos_</a> = 0;</div> |
| <div class="line"><a name="l00852"></a><span class="lineno"> 852</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a7b45fc959e21a86651488dff45869829">row_count_</a> = 0;</div> |
| <div class="line"><a name="l00853"></a><span class="lineno"> 853</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1357fae4c01f52a481f6996e24951a4d">file_size_estimate_</a> = 0;</div> |
| <div class="line"><a name="l00854"></a><span class="lineno"> 854</span> </div> |
| <div class="line"><a name="l00855"></a><span class="lineno"> 855</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.row_groups.clear();</div> |
| <div class="line"><a name="l00856"></a><span class="lineno"> 856</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a507adb3975fd7bc994da72f5777a3ef7">AddRowGroup</a>());</div> |
| <div class="line"><a name="l00857"></a><span class="lineno"> 857</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a6386ce29bdc47fde38d6a3b11c668af0">WriteFileHeader</a>());</div> |
| <div class="line"><a name="l00858"></a><span class="lineno"> 858</span> </div> |
| <div class="line"><a name="l00859"></a><span class="lineno"> 859</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00860"></a><span class="lineno"> 860</span> }</div> |
| <div class="line"><a name="l00861"></a><span class="lineno"> 861</span> </div> |
| <div class="line"><a name="l00862"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a93e294730c06b45f635bba83d84d96ec"> 862</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a93e294730c06b45f635bba83d84d96ec">HdfsParquetTableWriter::AppendRowBatch</a>(<a class="code" href="classimpala_1_1RowBatch.html">RowBatch</a>* batch,</div> |
| <div class="line"><a name="l00863"></a><span class="lineno"> 863</span>  <span class="keyword">const</span> vector<int32_t>& row_group_indices, <span class="keywordtype">bool</span>* new_file) {</div> |
| <div class="line"><a name="l00864"></a><span class="lineno"> 864</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a7faad5d8d7b9b63df88fc64634e8c68e">parent_</a>-><a class="code" href="classimpala_1_1HdfsTableSink.html#aaaac900efe9fa7b75331cf70d40a92ce">encode_timer</a>());</div> |
| <div class="line"><a name="l00865"></a><span class="lineno"> 865</span>  *new_file = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00866"></a><span class="lineno"> 866</span>  <span class="keywordtype">int</span> limit;</div> |
| <div class="line"><a name="l00867"></a><span class="lineno"> 867</span>  <span class="keywordflow">if</span> (row_group_indices.empty()) {</div> |
| <div class="line"><a name="l00868"></a><span class="lineno"> 868</span>  limit = batch-><a class="code" href="classimpala_1_1RowBatch.html#ac695df3b85ee416b3d99844813ae813d">num_rows</a>();</div> |
| <div class="line"><a name="l00869"></a><span class="lineno"> 869</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00870"></a><span class="lineno"> 870</span>  limit = row_group_indices.size();</div> |
| <div class="line"><a name="l00871"></a><span class="lineno"> 871</span>  }</div> |
| <div class="line"><a name="l00872"></a><span class="lineno"> 872</span> </div> |
| <div class="line"><a name="l00873"></a><span class="lineno"> 873</span>  <span class="keywordtype">bool</span> all_rows = row_group_indices.empty();</div> |
| <div class="line"><a name="l00874"></a><span class="lineno"> 874</span>  <span class="keywordflow">for</span> (; <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ab210a41de213aa2d78e1f8ac4714eec2">row_idx_</a> < limit;) {</div> |
| <div class="line"><a name="l00875"></a><span class="lineno"> 875</span>  <a class="code" href="classimpala_1_1TupleRow.html">TupleRow</a>* current_row = all_rows ?</div> |
| <div class="line"><a name="l00876"></a><span class="lineno"> 876</span>  batch-><a class="code" href="classimpala_1_1RowBatch.html#a9320ed986b0717ef26d73d871f3b3a42">GetRow</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ab210a41de213aa2d78e1f8ac4714eec2">row_idx_</a>) : batch-><a class="code" href="classimpala_1_1RowBatch.html#a9320ed986b0717ef26d73d871f3b3a42">GetRow</a>(row_group_indices[<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ab210a41de213aa2d78e1f8ac4714eec2">row_idx_</a>]);</div> |
| <div class="line"><a name="l00877"></a><span class="lineno"> 877</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> j = 0; j < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size(); ++j) {</div> |
| <div class="line"><a name="l00878"></a><span class="lineno"> 878</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[j]->AppendRow(current_row));</div> |
| <div class="line"><a name="l00879"></a><span class="lineno"> 879</span>  }</div> |
| <div class="line"><a name="l00880"></a><span class="lineno"> 880</span>  ++<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ab210a41de213aa2d78e1f8ac4714eec2">row_idx_</a>;</div> |
| <div class="line"><a name="l00881"></a><span class="lineno"> 881</span>  ++<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a7b45fc959e21a86651488dff45869829">row_count_</a>;</div> |
| <div class="line"><a name="l00882"></a><span class="lineno"> 882</span>  ++<a class="code" href="classimpala_1_1HdfsTableWriter.html#a25eed1b5b3cb444d8bfdc62890238ff7">output_</a>-><a class="code" href="structimpala_1_1OutputPartition.html#a6c5c9cee1f82ec607f54eda9bc339543">num_rows</a>;</div> |
| <div class="line"><a name="l00883"></a><span class="lineno"> 883</span> </div> |
| <div class="line"><a name="l00884"></a><span class="lineno"> 884</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1357fae4c01f52a481f6996e24951a4d">file_size_estimate_</a> > <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">file_size_limit_</a>) {</div> |
| <div class="line"><a name="l00885"></a><span class="lineno"> 885</span>  <span class="comment">// This file is full. We need a new file.</span></div> |
| <div class="line"><a name="l00886"></a><span class="lineno"> 886</span>  *new_file = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00887"></a><span class="lineno"> 887</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00888"></a><span class="lineno"> 888</span>  }</div> |
| <div class="line"><a name="l00889"></a><span class="lineno"> 889</span>  }</div> |
| <div class="line"><a name="l00890"></a><span class="lineno"> 890</span> </div> |
| <div class="line"><a name="l00891"></a><span class="lineno"> 891</span>  <span class="comment">// Reset the row_idx_ when we exhaust the batch. We can exit before exhausting</span></div> |
| <div class="line"><a name="l00892"></a><span class="lineno"> 892</span>  <span class="comment">// the batch if we run out of file space and will continue from the last index.</span></div> |
| <div class="line"><a name="l00893"></a><span class="lineno"> 893</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ab210a41de213aa2d78e1f8ac4714eec2">row_idx_</a> = 0;</div> |
| <div class="line"><a name="l00894"></a><span class="lineno"> 894</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00895"></a><span class="lineno"> 895</span> }</div> |
| <div class="line"><a name="l00896"></a><span class="lineno"> 896</span> </div> |
| <div class="line"><a name="l00897"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a65014902eeea2359dce460e709b04a84"> 897</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a65014902eeea2359dce460e709b04a84">HdfsParquetTableWriter::Finalize</a>() {</div> |
| <div class="line"><a name="l00898"></a><span class="lineno"> 898</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a7faad5d8d7b9b63df88fc64634e8c68e">parent_</a>-><a class="code" href="classimpala_1_1HdfsTableSink.html#a8019ba8c269e32342b407c6e74bc931e">hdfs_write_timer</a>());</div> |
| <div class="line"><a name="l00899"></a><span class="lineno"> 899</span> </div> |
| <div class="line"><a name="l00900"></a><span class="lineno"> 900</span>  <span class="comment">// At this point we write out the rest of the file. We first update the file</span></div> |
| <div class="line"><a name="l00901"></a><span class="lineno"> 901</span>  <span class="comment">// metadata, now that all the values have been seen.</span></div> |
| <div class="line"><a name="l00902"></a><span class="lineno"> 902</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>.num_rows = <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a7b45fc959e21a86651488dff45869829">row_count_</a>;</div> |
| <div class="line"><a name="l00903"></a><span class="lineno"> 903</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a74b2e94e82156cfdaa1d140cc5d512e7">FlushCurrentRowGroup</a>());</div> |
| <div class="line"><a name="l00904"></a><span class="lineno"> 904</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a25e7c1ba35a095ceddc08db22e3f8619">WriteFileFooter</a>());</div> |
| <div class="line"><a name="l00905"></a><span class="lineno"> 905</span>  <a class="code" href="classimpala_1_1HdfsTableWriter.html#a017d589ce2bdcc502582924169f5d2d7">stats_</a>.__set_parquet_stats(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a05f899a25bef44bd805dbe71d0c142e6">parquet_stats_</a>);</div> |
| <div class="line"><a name="l00906"></a><span class="lineno"> 906</span>  <a class="code" href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a7faad5d8d7b9b63df88fc64634e8c68e">parent_</a>-><a class="code" href="classimpala_1_1HdfsTableSink.html#a2025d785fa7bfb70e8dc84bb91114410">rows_inserted_counter</a>(), <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a7b45fc959e21a86651488dff45869829">row_count_</a>);</div> |
| <div class="line"><a name="l00907"></a><span class="lineno"> 907</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00908"></a><span class="lineno"> 908</span> }</div> |
| <div class="line"><a name="l00909"></a><span class="lineno"> 909</span> </div> |
| <div class="line"><a name="l00910"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a73683272d74284275464830d01945223"> 910</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a73683272d74284275464830d01945223">HdfsParquetTableWriter::Close</a>() {</div> |
| <div class="line"><a name="l00911"></a><span class="lineno"> 911</span>  <span class="comment">// Release all accumulated memory</span></div> |
| <div class="line"><a name="l00912"></a><span class="lineno"> 912</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00913"></a><span class="lineno"> 913</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->Close();</div> |
| <div class="line"><a name="l00914"></a><span class="lineno"> 914</span>  }</div> |
| <div class="line"><a name="l00915"></a><span class="lineno"> 915</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3e155c5da813244fd5a3066f7410fa2f">reusable_col_mem_pool_</a>->FreeAll();</div> |
| <div class="line"><a name="l00916"></a><span class="lineno"> 916</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a67a6abe5cd883902e5adddea9c68f54e">per_file_mem_pool_</a>->FreeAll();</div> |
| <div class="line"><a name="l00917"></a><span class="lineno"> 917</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a500303f24046a1b6bf90967ca73af2f0">compression_staging_buffer_</a>.clear();</div> |
| <div class="line"><a name="l00918"></a><span class="lineno"> 918</span> }</div> |
| <div class="line"><a name="l00919"></a><span class="lineno"> 919</span> </div> |
| <div class="line"><a name="l00920"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a6386ce29bdc47fde38d6a3b11c668af0"> 920</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a6386ce29bdc47fde38d6a3b11c668af0">HdfsParquetTableWriter::WriteFileHeader</a>() {</div> |
| <div class="line"><a name="l00921"></a><span class="lineno"> 921</span>  DCHECK_EQ(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">file_pos_</a>, 0);</div> |
| <div class="line"><a name="l00922"></a><span class="lineno"> 922</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a1be7c5b50e90418d3bc60984a2788179">Write</a>(<a class="code" href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">PARQUET_VERSION_NUMBER</a>, <span class="keyword">sizeof</span>(<a class="code" href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">PARQUET_VERSION_NUMBER</a>)));</div> |
| <div class="line"><a name="l00923"></a><span class="lineno"> 923</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">file_pos_</a> += <span class="keyword">sizeof</span>(<a class="code" href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">PARQUET_VERSION_NUMBER</a>);</div> |
| <div class="line"><a name="l00924"></a><span class="lineno"> 924</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1357fae4c01f52a481f6996e24951a4d">file_size_estimate_</a> += <span class="keyword">sizeof</span>(<a class="code" href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">PARQUET_VERSION_NUMBER</a>);</div> |
| <div class="line"><a name="l00925"></a><span class="lineno"> 925</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00926"></a><span class="lineno"> 926</span> }</div> |
| <div class="line"><a name="l00927"></a><span class="lineno"> 927</span> </div> |
| <div class="line"><a name="l00928"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a74b2e94e82156cfdaa1d140cc5d512e7"> 928</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a74b2e94e82156cfdaa1d140cc5d512e7">HdfsParquetTableWriter::FlushCurrentRowGroup</a>() {</div> |
| <div class="line"><a name="l00929"></a><span class="lineno"> 929</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a> == NULL) <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00930"></a><span class="lineno"> 930</span> </div> |
| <div class="line"><a name="l00931"></a><span class="lineno"> 931</span>  <span class="keywordtype">int</span> num_clustering_cols = <a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">num_clustering_cols</a>();</div> |
| <div class="line"><a name="l00932"></a><span class="lineno"> 932</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00933"></a><span class="lineno"> 933</span>  int64_t data_page_offset, dict_page_offset;</div> |
| <div class="line"><a name="l00934"></a><span class="lineno"> 934</span>  <span class="comment">// Flush this column. This updates the final metadata sizes for this column.</span></div> |
| <div class="line"><a name="l00935"></a><span class="lineno"> 935</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->Flush(&<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">file_pos_</a>, &data_page_offset, &dict_page_offset));</div> |
| <div class="line"><a name="l00936"></a><span class="lineno"> 936</span>  DCHECK_GT(data_page_offset, 0);</div> |
| <div class="line"><a name="l00937"></a><span class="lineno"> 937</span> </div> |
| <div class="line"><a name="l00938"></a><span class="lineno"> 938</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].meta_data.data_page_offset = data_page_offset;</div> |
| <div class="line"><a name="l00939"></a><span class="lineno"> 939</span>  <span class="keywordflow">if</span> (dict_page_offset >= 0) {</div> |
| <div class="line"><a name="l00940"></a><span class="lineno"> 940</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].meta_data.__set_dictionary_page_offset(</div> |
| <div class="line"><a name="l00941"></a><span class="lineno"> 941</span>  dict_page_offset);</div> |
| <div class="line"><a name="l00942"></a><span class="lineno"> 942</span>  }</div> |
| <div class="line"><a name="l00943"></a><span class="lineno"> 943</span> </div> |
| <div class="line"><a name="l00944"></a><span class="lineno"> 944</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].meta_data.num_values = <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->num_values();</div> |
| <div class="line"><a name="l00945"></a><span class="lineno"> 945</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].meta_data.total_uncompressed_size =</div> |
| <div class="line"><a name="l00946"></a><span class="lineno"> 946</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->total_uncompressed_size();</div> |
| <div class="line"><a name="l00947"></a><span class="lineno"> 947</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].meta_data.total_compressed_size =</div> |
| <div class="line"><a name="l00948"></a><span class="lineno"> 948</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->total_compressed_size();</div> |
| <div class="line"><a name="l00949"></a><span class="lineno"> 949</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->total_byte_size += <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->total_compressed_size();</div> |
| <div class="line"><a name="l00950"></a><span class="lineno"> 950</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->num_rows = <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->num_values();</div> |
| <div class="line"><a name="l00951"></a><span class="lineno"> 951</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i].file_offset = <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">file_pos_</a>;</div> |
| <div class="line"><a name="l00952"></a><span class="lineno"> 952</span>  <span class="keyword">const</span> <span class="keywordtype">string</span>& col_name = <a class="code" href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">col_names</a>()[i + num_clustering_cols];</div> |
| <div class="line"><a name="l00953"></a><span class="lineno"> 953</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a05f899a25bef44bd805dbe71d0c142e6">parquet_stats_</a>.per_column_size[col_name] += <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->total_compressed_size();</div> |
| <div class="line"><a name="l00954"></a><span class="lineno"> 954</span> </div> |
| <div class="line"><a name="l00955"></a><span class="lineno"> 955</span>  <span class="comment">// Since we don't supported complex schemas, all columns should have the same</span></div> |
| <div class="line"><a name="l00956"></a><span class="lineno"> 956</span>  <span class="comment">// number of values.</span></div> |
| <div class="line"><a name="l00957"></a><span class="lineno"> 957</span>  DCHECK_EQ(<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[0].meta_data.num_values,</div> |
| <div class="line"><a name="l00958"></a><span class="lineno"> 958</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->num_values());</div> |
| <div class="line"><a name="l00959"></a><span class="lineno"> 959</span> </div> |
| <div class="line"><a name="l00960"></a><span class="lineno"> 960</span>  <span class="comment">// Metadata for this column is complete, write it out to file. The column metadata</span></div> |
| <div class="line"><a name="l00961"></a><span class="lineno"> 961</span>  <span class="comment">// goes at the end so that when we have collocated files, the column data can be</span></div> |
| <div class="line"><a name="l00962"></a><span class="lineno"> 962</span>  <span class="comment">// written without buffering.</span></div> |
| <div class="line"><a name="l00963"></a><span class="lineno"> 963</span>  uint8_t* buffer = NULL;</div> |
| <div class="line"><a name="l00964"></a><span class="lineno"> 964</span>  uint32_t len = 0;</div> |
| <div class="line"><a name="l00965"></a><span class="lineno"> 965</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(</div> |
| <div class="line"><a name="l00966"></a><span class="lineno"> 966</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ade9111e210f0789b6c22b310f5a0687d">thrift_serializer_</a>->Serialize(&<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a>->columns[i], &len, &buffer));</div> |
| <div class="line"><a name="l00967"></a><span class="lineno"> 967</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a1be7c5b50e90418d3bc60984a2788179">Write</a>(buffer, len));</div> |
| <div class="line"><a name="l00968"></a><span class="lineno"> 968</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">file_pos_</a> += len;</div> |
| <div class="line"><a name="l00969"></a><span class="lineno"> 969</span> </div> |
| <div class="line"><a name="l00970"></a><span class="lineno"> 970</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">columns_</a>[i]->Reset();</div> |
| <div class="line"><a name="l00971"></a><span class="lineno"> 971</span>  }</div> |
| <div class="line"><a name="l00972"></a><span class="lineno"> 972</span> </div> |
| <div class="line"><a name="l00973"></a><span class="lineno"> 973</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">current_row_group_</a> = NULL;</div> |
| <div class="line"><a name="l00974"></a><span class="lineno"> 974</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00975"></a><span class="lineno"> 975</span> }</div> |
| <div class="line"><a name="l00976"></a><span class="lineno"> 976</span> </div> |
| <div class="line"><a name="l00977"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsParquetTableWriter.html#a25e7c1ba35a095ceddc08db22e3f8619"> 977</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a25e7c1ba35a095ceddc08db22e3f8619">HdfsParquetTableWriter::WriteFileFooter</a>() {</div> |
| <div class="line"><a name="l00978"></a><span class="lineno"> 978</span>  <span class="comment">// Write file_meta_data</span></div> |
| <div class="line"><a name="l00979"></a><span class="lineno"> 979</span>  uint32_t file_metadata_len = 0;</div> |
| <div class="line"><a name="l00980"></a><span class="lineno"> 980</span>  uint8_t* buffer = NULL;</div> |
| <div class="line"><a name="l00981"></a><span class="lineno"> 981</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(</div> |
| <div class="line"><a name="l00982"></a><span class="lineno"> 982</span>  <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#ade9111e210f0789b6c22b310f5a0687d">thrift_serializer_</a>->Serialize(&<a class="code" href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">file_metadata_</a>, &file_metadata_len, &buffer));</div> |
| <div class="line"><a name="l00983"></a><span class="lineno"> 983</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a1be7c5b50e90418d3bc60984a2788179">Write</a>(buffer, file_metadata_len));</div> |
| <div class="line"><a name="l00984"></a><span class="lineno"> 984</span> </div> |
| <div class="line"><a name="l00985"></a><span class="lineno"> 985</span>  <span class="comment">// Write footer</span></div> |
| <div class="line"><a name="l00986"></a><span class="lineno"> 986</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(Write<uint32_t>(file_metadata_len));</div> |
| <div class="line"><a name="l00987"></a><span class="lineno"> 987</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableWriter.html#a1be7c5b50e90418d3bc60984a2788179">Write</a>(<a class="code" href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">PARQUET_VERSION_NUMBER</a>, <span class="keyword">sizeof</span>(<a class="code" href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">PARQUET_VERSION_NUMBER</a>)));</div> |
| <div class="line"><a name="l00988"></a><span class="lineno"> 988</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00989"></a><span class="lineno"> 989</span> }</div> |
| <div class="line"><a name="l00990"></a><span class="lineno"> 990</span> </div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_a2c4a2e702c8776b1e5e3493a157bc4d2"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a2c4a2e702c8776b1e5e3493a157bc4d2">impala::HdfsParquetTableWriter::ColumnWriter::encoded_value_size_</a></div><div class="ttdeci">int64_t encoded_value_size_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00311">hdfs-parquet-table-writer.cc:311</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ad7f28231c208b04ff9bedf46f1673f4b"><div class="ttname"><a href="namespaceimpala.html#ad7f28231c208b04ff9bedf46f1673f4b">impala::IMPALA_TO_PARQUET_CODEC</a></div><div class="ttdeci">const parquet::CompressionCodec::type IMPALA_TO_PARQUET_CODEC[]</div><div class="ttdoc">Mapping of Impala codec enums to Parquet enums. </div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00064">parquet-common.h:64</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_ade9111e210f0789b6c22b310f5a0687d"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#ade9111e210f0789b6c22b310f5a0687d">impala::HdfsParquetTableWriter::thrift_serializer_</a></div><div class="ttdeci">boost::scoped_ptr< ThriftSerializer > thrift_serializer_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00134">hdfs-parquet-table-writer.h:134</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a3e155c5da813244fd5a3066f7410fa2f"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a3e155c5da813244fd5a3066f7410fa2f">impala::HdfsParquetTableWriter::reusable_col_mem_pool_</a></div><div class="ttdeci">boost::scoped_ptr< MemPool > reusable_col_mem_pool_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00165">hdfs-parquet-table-writer.h:165</a></div></div> |
| <div class="ttc" id="row-batch_8h_html"><div class="ttname"><a href="row-batch_8h.html">row-batch.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_aacec45fe3db2013a0124fa966eaa3486"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#aacec45fe3db2013a0124fa966eaa3486">impala::HdfsParquetTableWriter::file_size_limit_</a></div><div class="ttdeci">int64_t file_size_limit_</div><div class="ttdoc">Limit on the total size of the file. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00156">hdfs-parquet-table-writer.h:156</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a500303f24046a1b6bf90967ca73af2f0"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a500303f24046a1b6bf90967ca73af2f0">impala::HdfsParquetTableWriter::compression_staging_buffer_</a></div><div class="ttdeci">std::vector< uint8_t > compression_staging_buffer_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00178">hdfs-parquet-table-writer.h:178</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a5ee5e263966386b0c48087ca790a4f29"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a5ee5e263966386b0c48087ca790a4f29">impala::HdfsParquetTableWriter::BaseColumnWriter::AppendRow</a></div><div class="ttdeci">Status AppendRow(TupleRow *row)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00371">hdfs-parquet-table-writer.cc:371</a></div></div> |
| <div class="ttc" id="classimpala_1_1ParquetPlainEncoder_html_a5f888cf60ea0a62b73c9a3cb9e67dac0"><div class="ttname"><a href="classimpala_1_1ParquetPlainEncoder.html#a5f888cf60ea0a62b73c9a3cb9e67dac0">impala::ParquetPlainEncoder::DecimalSize</a></div><div class="ttdeci">static int DecimalSize(const ColumnType &t)</div><div class="ttdoc">The minimum byte size to store decimals of with precision t.precision. </div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00116">parquet-common.h:116</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage_html_a35b49574b69acd7b34ded9083c9f2ef8"><div class="ttname"><a href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a35b49574b69acd7b34ded9083c9f2ef8">impala::HdfsParquetTableWriter::BaseColumnWriter::DataPage::num_def_bytes</a></div><div class="ttdeci">int num_def_bytes</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00178">hdfs-parquet-table-writer.cc:178</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_abf1f8a799d6f3fe3c1d8539537791a50"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#abf1f8a799d6f3fe3c1d8539537791a50">impala::HdfsTableSink::GetFileBlockSize</a></div><div class="ttdeci">static Status GetFileBlockSize(OutputPartition *output_partition, int64_t *size)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00628">hdfs-table-sink.cc:628</a></div></div> |
| <div class="ttc" id="dict-encoding_8h_html"><div class="ttname"><a href="dict-encoding_8h.html">dict-encoding.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a40a3023d094eb5a5a8c7335aa27bb810"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a40a3023d094eb5a5a8c7335aa27bb810">impala::TYPE_DOUBLE</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00036">types.h:36</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage_html"><div class="ttname"><a href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html">impala::HdfsParquetTableWriter::BaseColumnWriter::DataPage</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00173">hdfs-parquet-table-writer.cc:173</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowBatch_html_ac695df3b85ee416b3d99844813ae813d"><div class="ttname"><a href="classimpala_1_1RowBatch.html#ac695df3b85ee416b3d99844813ae813d">impala::RowBatch::num_rows</a></div><div class="ttdeci">int num_rows() const </div><div class="ttdef"><b>Definition:</b> <a href="row-batch_8h_source.html#l00215">row-batch.h:215</a></div></div> |
| <div class="ttc" id="bit-stream-utils_8h_html"><div class="ttname"><a href="bit-stream-utils_8h.html">bit-stream-utils.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_a7faad5d8d7b9b63df88fc64634e8c68e"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#a7faad5d8d7b9b63df88fc64634e8c68e">impala::HdfsTableWriter::parent_</a></div><div class="ttdeci">HdfsTableSink * parent_</div><div class="ttdoc">Parent table sink object. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00112">hdfs-table-writer.h:112</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_ad3d393e5df5ebc0d0a3def6d1b9ef6ca"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ad3d393e5df5ebc0d0a3def6d1b9ef6ca">impala::HdfsParquetTableWriter::BaseColumnWriter::current_page_</a></div><div class="ttdeci">DataPage * current_page_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00214">hdfs-parquet-table-writer.cc:214</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_ab210a41de213aa2d78e1f8ac4714eec2"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#ab210a41de213aa2d78e1f8ac4714eec2">impala::HdfsParquetTableWriter::row_idx_</a></div><div class="ttdeci">int row_idx_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00174">hdfs-parquet-table-writer.h:174</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a18854cd2e08266b09600a6e559c7c6e1"><div class="ttname"><a href="classimpala_1_1Codec.html#a18854cd2e08266b09600a6e559c7c6e1">impala::Codec::CreateCompressor</a></div><div class="ttdeci">static Status CreateCompressor(MemPool *mem_pool, bool reuse, THdfsCompression::type format, boost::scoped_ptr< Codec > *compressor)</div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a58fbe85de5770a41f8936ed57e47f6a7"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a58fbe85de5770a41f8936ed57e47f6a7">impala::HdfsParquetTableWriter::BaseColumnWriter::num_values_</a></div><div class="ttdeci">int64_t num_values_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00215">hdfs-parquet-table-writer.cc:215</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a091883d9ab1382372609ed1a8fd88cf3"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a091883d9ab1382372609ed1a8fd88cf3">impala::HdfsParquetTableWriter::file_metadata_</a></div><div class="ttdeci">parquet::FileMetaData file_metadata_</div><div class="ttdoc">File metdata thrift description. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00137">hdfs-parquet-table-writer.h:137</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage_html_a4b0955451fd2bbcc8ef33f631033c627"><div class="ttname"><a href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a4b0955451fd2bbcc8ef33f631033c627">impala::HdfsParquetTableWriter::BaseColumnWriter::DataPage::data</a></div><div class="ttdeci">uint8_t * data</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00183">hdfs-parquet-table-writer.cc:183</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871ae5ec6e333587a27f8b83c5bd9c815a2a"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ae5ec6e333587a27f8b83c5bd9c815a2a">impala::TYPE_CHAR</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00047">types.h:47</a></div></div> |
| <div class="ttc" id="structimpala_1_1StringValue_html"><div class="ttname"><a href="structimpala_1_1StringValue.html">impala::StringValue</a></div><div class="ttdef"><b>Definition:</b> <a href="string-value_8h_source.html#l00033">string-value.h:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a633dd0324c1b260dd46ae81ff5970277"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a633dd0324c1b260dd46ae81ff5970277">impala::HdfsParquetTableWriter::DEFAULT_DATA_PAGE_SIZE</a></div><div class="ttdeci">static const int DEFAULT_DATA_PAGE_SIZE</div><div class="ttdoc">Default data page size. In bytes. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00083">hdfs-parquet-table-writer.h:83</a></div></div> |
| <div class="ttc" id="version_8h_html_af38fbd78d9901e8a11ab4b413edd5bca"><div class="ttname"><a href="version_8h.html#af38fbd78d9901e8a11ab4b413edd5bca">IMPALA_BUILD_HASH</a></div><div class="ttdeci">#define IMPALA_BUILD_HASH</div><div class="ttdef"><b>Definition:</b> <a href="version_8h_source.html#l00022">version.h:22</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a77fca6b59dc2094950262e1eebb6d1c6"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a77fca6b59dc2094950262e1eebb6d1c6">impala::HdfsParquetTableWriter::CreateSchema</a></div><div class="ttdeci">Status CreateSchema()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00733">hdfs-parquet-table-writer.cc:733</a></div></div> |
| <div class="ttc" id="classimpala_1_1RleEncoder_html"><div class="ttname"><a href="classimpala_1_1RleEncoder.html">impala::RleEncoder</a></div><div class="ttdef"><b>Definition:</b> <a href="rle-encoding_8h_source.html#l00111">rle-encoding.h:111</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a54959e4d1e9a68988a6050493d9efc46"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a54959e4d1e9a68988a6050493d9efc46">impala::HdfsParquetTableWriter::BaseColumnWriter::BaseColumnWriter</a></div><div class="ttdeci">BaseColumnWriter(HdfsParquetTableWriter *parent, ExprContext *expr_ctx, const THdfsCompression::type &codec)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00090">hdfs-parquet-table-writer.cc:90</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a1cd7152204acac51f963dcccb7a2c87c"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a1cd7152204acac51f963dcccb7a2c87c">impala::HdfsParquetTableWriter::current_row_group_</a></div><div class="ttdeci">parquet::RowGroup * current_row_group_</div><div class="ttdoc">The current row group being written to. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00140">hdfs-parquet-table-writer.h:140</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a12d80a0bfe690580f4974e8baca466b0"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a12d80a0bfe690580f4974e8baca466b0">impala::TYPE_VARCHAR</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00048">types.h:48</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html">impala::HdfsParquetTableWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00049">hdfs-parquet-table-writer.h:49</a></div></div> |
| <div class="ttc" id="classimpala_1_1DictEncoderBase_html"><div class="ttname"><a href="classimpala_1_1DictEncoderBase.html">impala::DictEncoderBase</a></div><div class="ttdef"><b>Definition:</b> <a href="dict-encoding_8h_source.html#l00048">dict-encoding.h:48</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a9725689b4baf53907247e60778283094"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a9725689b4baf53907247e60778283094">impala::HdfsParquetTableWriter::Init</a></div><div class="ttdeci">virtual Status Init()</div><div class="ttdoc">Initialize column information. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00635">hdfs-parquet-table-writer.cc:635</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_ae9a4f4f1916abea5c92001bf43d5d436"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#ae9a4f4f1916abea5c92001bf43d5d436">impala::HdfsParquetTableWriter::ColumnWriter::num_values_since_dict_size_check_</a></div><div class="ttdeci">int num_values_since_dict_size_check_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00308">hdfs-parquet-table-writer.cc:308</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a73683272d74284275464830d01945223"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a73683272d74284275464830d01945223">impala::HdfsParquetTableWriter::Close</a></div><div class="ttdeci">virtual void Close()</div><div class="ttdoc">Called once when this writer should cleanup any resources. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00910">hdfs-parquet-table-writer.cc:910</a></div></div> |
| <div class="ttc" id="classimpala_1_1BufferBuilder_html_a8c1c2548614c9bb70efe407dc3558290"><div class="ttname"><a href="classimpala_1_1BufferBuilder.html#a8c1c2548614c9bb70efe407dc3558290">impala::BufferBuilder::size</a></div><div class="ttdeci">int size() const </div><div class="ttdef"><b>Definition:</b> <a href="buffer-builder_8h_source.html#l00048">buffer-builder.h:48</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_af9f34726a2605d7d0a13881089343746"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#af9f34726a2605d7d0a13881089343746">impala::HdfsParquetTableWriter::BaseColumnWriter::Close</a></div><div class="ttdeci">void Close()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00140">hdfs-parquet-table-writer.cc:140</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_af0582e210619f97f5ac5db62669bbc40"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#af0582e210619f97f5ac5db62669bbc40">impala::HdfsTableWriter::state_</a></div><div class="ttdeci">RuntimeState * state_</div><div class="ttdoc">Runtime state. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00115">hdfs-table-writer.h:115</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a27b116b49ec5a6965bf03d7df33019bf"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a27b116b49ec5a6965bf03d7df33019bf">impala::HdfsParquetTableWriter::BaseColumnWriter::compressor_</a></div><div class="ttdeci">scoped_ptr< Codec > compressor_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00199">hdfs-parquet-table-writer.cc:199</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a7b45fc959e21a86651488dff45869829"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a7b45fc959e21a86651488dff45869829">impala::HdfsParquetTableWriter::row_count_</a></div><div class="ttdeci">int64_t row_count_</div><div class="ttdoc">Number of rows in current file. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00146">hdfs-parquet-table-writer.h:146</a></div></div> |
| <div class="ttc" id="compress_8h_html"><div class="ttname"><a href="compress_8h.html">compress.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a53fc672ffcaf912893ba703449e58d77"><div class="ttname"><a href="namespaceimpala.html#a53fc672ffcaf912893ba703449e58d77">impala::PARQUET_VERSION_NUMBER</a></div><div class="ttdeci">const uint8_t PARQUET_VERSION_NUMBER[4]</div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00030">parquet-common.h:30</a></div></div> |
| <div class="ttc" id="classimpala_1_1BufferBuilder_html_a4cc096476e8a7e7fd794023da8f80460"><div class="ttname"><a href="classimpala_1_1BufferBuilder.html#a4cc096476e8a7e7fd794023da8f80460">impala::BufferBuilder::Append</a></div><div class="ttdeci">void Append(const void *buffer, int len)</div><div class="ttdef"><b>Definition:</b> <a href="buffer-builder_8h_source.html#l00036">buffer-builder.h:36</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a16639b10ee6d377ac527c80a882f07ed"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a16639b10ee6d377ac527c80a882f07ed">impala::HdfsParquetTableWriter::BaseColumnWriter::def_levels_</a></div><div class="ttdeci">RleEncoder * def_levels_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00227">hdfs-parquet-table-writer.cc:227</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableDescriptor_html"><div class="ttname"><a href="classimpala_1_1HdfsTableDescriptor.html">impala::HdfsTableDescriptor</a></div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00226">descriptors.h:226</a></div></div> |
| <div class="ttc" id="status_8h_html_a85f7d0e774e15eb35b74f53264305e16"><div class="ttname"><a href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a></div><div class="ttdeci">#define RETURN_IF_ERROR(stmt)</div><div class="ttdoc">some generally useful macros </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00242">status.h:242</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a1357fae4c01f52a481f6996e24951a4d"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a1357fae4c01f52a481f6996e24951a4d">impala::HdfsParquetTableWriter::file_size_estimate_</a></div><div class="ttdeci">int64_t file_size_estimate_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00153">hdfs-parquet-table-writer.h:153</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_af3c813863be4bc935ebb0d5dfff00ca5"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#af3c813863be4bc935ebb0d5dfff00ca5">impala::HdfsParquetTableWriter::file_pos_</a></div><div class="ttdeci">int64_t file_pos_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00161">hdfs-parquet-table-writer.h:161</a></div></div> |
| <div class="ttc" id="hdfs-util_8h_html"><div class="ttname"><a href="hdfs-util_8h.html">hdfs-util.h</a></div></div> |
| <div class="ttc" id="raw-value_8h_html"><div class="ttname"><a href="raw-value_8h.html">raw-value.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_ae401238cb881d9277416cc0db68bbdfc"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae401238cb881d9277416cc0db68bbdfc">impala::HdfsParquetTableWriter::BaseColumnWriter::num_values</a></div><div class="ttdeci">uint64_t num_values() const </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00146">hdfs-parquet-table-writer.cc:146</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_ad9da7d51aeeb791ff5d3e14039aa58bd"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#ad9da7d51aeeb791ff5d3e14039aa58bd">impala::HdfsParquetTableWriter::HDFS_BLOCK_SIZE</a></div><div class="ttdeci">static const int HDFS_BLOCK_SIZE</div><div class="ttdoc">Default hdfs block size. In bytes. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00090">hdfs-parquet-table-writer.h:90</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowBatch_html_a9320ed986b0717ef26d73d871f3b3a42"><div class="ttname"><a href="classimpala_1_1RowBatch.html#a9320ed986b0717ef26d73d871f3b3a42">impala::RowBatch::GetRow</a></div><div class="ttdeci">TupleRow * GetRow(int row_idx)</div><div class="ttdef"><b>Definition:</b> <a href="row-batch_8h_source.html#l00140">row-batch.h:140</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a74b2e94e82156cfdaa1d140cc5d512e7"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a74b2e94e82156cfdaa1d140cc5d512e7">impala::HdfsParquetTableWriter::FlushCurrentRowGroup</a></div><div class="ttdeci">Status FlushCurrentRowGroup()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00928">hdfs-parquet-table-writer.cc:928</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_aed34565a4d5ac112ec1c24935314931d"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#aed34565a4d5ac112ec1c24935314931d">impala::HdfsParquetTableWriter::InitNewFile</a></div><div class="ttdeci">virtual Status InitNewFile()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00814">hdfs-parquet-table-writer.cc:814</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_a5b4fb69116d191d1f70569ae77231460"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a5b4fb69116d191d1f70569ae77231460">impala::HdfsParquetTableWriter::ColumnWriter::dict_encoder_</a></div><div class="ttdeci">scoped_ptr< DictEncoder< T > > dict_encoder_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00305">hdfs-parquet-table-writer.cc:305</a></div></div> |
| <div class="ttc" id="buffer-builder_8h_html"><div class="ttname"><a href="buffer-builder_8h.html">buffer-builder.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExprContext_html"><div class="ttname"><a href="classimpala_1_1ExprContext.html">impala::ExprContext</a></div><div class="ttdef"><b>Definition:</b> <a href="expr-context_8h_source.html#l00040">expr-context.h:40</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_a1be7c5b50e90418d3bc60984a2788179"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#a1be7c5b50e90418d3bc60984a2788179">impala::HdfsTableWriter::Write</a></div><div class="ttdeci">Status Write(const char *data, int32_t len)</div><div class="ttdoc">Write to the current hdfs file. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00101">hdfs-table-writer.h:101</a></div></div> |
| <div class="ttc" id="expr-context_8h_html"><div class="ttname"><a href="expr-context_8h.html">expr-context.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter_html_a4c42d54f3f086d230f9311c80046c95f"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a4c42d54f3f086d230f9311c80046c95f">impala::HdfsParquetTableWriter::BoolColumnWriter::bool_values_</a></div><div class="ttdeci">BitWriter * bool_values_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00366">hdfs-parquet-table-writer.cc:366</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_af147ac530796b3aa05000035596861e0"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#af147ac530796b3aa05000035596861e0">impala::HdfsParquetTableWriter::BaseColumnWriter::expr_ctx_</a></div><div class="ttdeci">ExprContext * expr_ctx_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00194">hdfs-parquet-table-writer.cc:194</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a7a904ead6161c202460dd0fa4efd9104"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a7a904ead6161c202460dd0fa4efd9104">impala::HdfsParquetTableWriter::HDFS_MIN_FILE_SIZE</a></div><div class="ttdeci">static const int HDFS_MIN_FILE_SIZE</div><div class="ttdoc">Minimum file size. If the configured size is less, fail. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00099">hdfs-parquet-table-writer.h:99</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_a753b558d1afe680335aad9b5772772ab"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a753b558d1afe680335aad9b5772772ab">impala::HdfsParquetTableWriter::ColumnWriter::Reset</a></div><div class="ttdeci">virtual void Reset()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00247">hdfs-parquet-table-writer.cc:247</a></div></div> |
| <div class="ttc" id="classimpala_1_1TableDescriptor_html_a665ceaf9c184ec39544c3ecc448c0d11"><div class="ttname"><a href="classimpala_1_1TableDescriptor.html#a665ceaf9c184ec39544c3ecc448c0d11">impala::TableDescriptor::num_cols</a></div><div class="ttdeci">int num_cols() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00152">descriptors.h:152</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter_html_a7df1d2c3919af5601f56c2aed1c21d74"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a7df1d2c3919af5601f56c2aed1c21d74">impala::HdfsParquetTableWriter::BoolColumnWriter::FinalizeCurrentPage</a></div><div class="ttdeci">virtual void FinalizeCurrentPage()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00353">hdfs-parquet-table-writer.cc:353</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html"><div class="ttname"><a href="structimpala_1_1OutputPartition.html">impala::OutputPartition</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00040">hdfs-table-sink.h:40</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a069c640daf028ed2b1da6af395d388c0"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a069c640daf028ed2b1da6af395d388c0">impala::HdfsParquetTableWriter::HDFS_BLOCK_ALIGNMENT</a></div><div class="ttdeci">static const int HDFS_BLOCK_ALIGNMENT</div><div class="ttdoc">Align block sizes to this constant. In bytes. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00093">hdfs-parquet-table-writer.h:93</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html">impala::HdfsParquetTableWriter::BaseColumnWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00087">hdfs-parquet-table-writer.cc:87</a></div></div> |
| <div class="ttc" id="classimpala_1_1ParquetPlainEncoder_html_a42f57c98268a410f74f3cbefde238c28"><div class="ttname"><a href="classimpala_1_1ParquetPlainEncoder.html#a42f57c98268a410f74f3cbefde238c28">impala::ParquetPlainEncoder::ByteSize</a></div><div class="ttdeci">static int ByteSize(const T &v)</div><div class="ttdoc">Returns the byte size of 'v'. </div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00084">parquet-common.h:84</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_ae334368ceff2a98e639a2f249bd97fb4"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#ae334368ceff2a98e639a2f249bd97fb4">impala::HdfsTableWriter::table_desc_</a></div><div class="ttdeci">const HdfsTableDescriptor * table_desc_</div><div class="ttdoc">Table descriptor of table to be written. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00121">hdfs-table-writer.h:121</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a65014902eeea2359dce460e709b04a84"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a65014902eeea2359dce460e709b04a84">impala::HdfsParquetTableWriter::Finalize</a></div><div class="ttdeci">virtual Status Finalize()</div><div class="ttdoc">Write out all the data. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00897">hdfs-parquet-table-writer.cc:897</a></div></div> |
| <div class="ttc" id="classimpala_1_1BufferBuilder_html"><div class="ttname"><a href="classimpala_1_1BufferBuilder.html">impala::BufferBuilder</a></div><div class="ttdoc">Utility class to build an in-memory buffer. </div><div class="ttdef"><b>Definition:</b> <a href="buffer-builder_8h_source.html#l00025">buffer-builder.h:25</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_a0842f330c18b3e4e53e13655add0310a"><div class="ttname"><a href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a></div><div class="ttdeci">#define COUNTER_ADD(c, v)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00055">runtime-profile.h:55</a></div></div> |
| <div class="ttc" id="classimpala_1_1TupleRow_html"><div class="ttname"><a href="classimpala_1_1TupleRow.html">impala::TupleRow</a></div><div class="ttdef"><b>Definition:</b> <a href="tuple-row_8h_source.html#l00028">tuple-row.h:28</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871afdce7fe1e22d49d6fd07b168cb5b4f7b"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871afdce7fe1e22d49d6fd07b168cb5b4f7b">impala::TYPE_TIMESTAMP</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00037">types.h:37</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a17dd47f44b1374d094451e3cb1f5801e"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a17dd47f44b1374d094451e3cb1f5801e">impala::HdfsParquetTableWriter::BaseColumnWriter::total_uncompressed_byte_size_</a></div><div class="ttdeci">int64_t total_uncompressed_byte_size_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00217">hdfs-parquet-table-writer.cc:217</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_a84b7339c7e5ebb25ea9e6e3907b41b3b"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a84b7339c7e5ebb25ea9e6e3907b41b3b">impala::HdfsParquetTableWriter::ColumnWriter::CastValue</a></div><div class="ttdeci">T * CastValue(void *value)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00317">hdfs-parquet-table-writer.cc:317</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_aaa9a2971c6368e3ddd3f5140a0295eb7"><div class="ttname"><a href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a></div><div class="ttdeci">#define SCOPED_TIMER(c)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00053">runtime-profile.h:53</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter_html_a59be46b62d8150d518cfe5c8bd36ea8d"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#a59be46b62d8150d518cfe5c8bd36ea8d">impala::HdfsParquetTableWriter::BoolColumnWriter::EncodeValue</a></div><div class="ttdeci">virtual bool EncodeValue(void *value, int64_t *bytes_needed)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00349">hdfs-parquet-table-writer.cc:349</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_ae4efff2f9c9b600b8bb51e24d51fa74a"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae4efff2f9c9b600b8bb51e24d51fa74a">impala::HdfsParquetTableWriter::BaseColumnWriter::total_compressed_byte_size_</a></div><div class="ttdeci">int64_t total_compressed_byte_size_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00216">hdfs-parquet-table-writer.cc:216</a></div></div> |
| <div class="ttc" id="classimpala_1_1PrettyPrinter_html_a6e5f06a1ce274f747cc4f9928b797c5a"><div class="ttname"><a href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">impala::PrettyPrinter::Print</a></div><div class="ttdeci">static std::string Print(bool value, TUnit::type ignored, bool verbose=false)</div><div class="ttdef"><b>Definition:</b> <a href="pretty-printer_8h_source.html#l00033">pretty-printer.h:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html">impala::HdfsTableSink</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00122">hdfs-table-sink.h:122</a></div></div> |
| <div class="ttc" id="structimpala_1_1StringValue_html_abed17df66bd5a82e95f5c1fcf9c0da1c"><div class="ttname"><a href="structimpala_1_1StringValue.html#abed17df66bd5a82e95f5c1fcf9c0da1c">impala::StringValue::UnpaddedCharLength</a></div><div class="ttdeci">static int64_t UnpaddedCharLength(const char *cptr, int64_t len)</div><div class="ttdoc">Returns number of characters in a char array (ignores trailing spaces) </div><div class="ttdef"><b>Definition:</b> <a href="string-value_8inline_8h_source.html#l00107">string-value.inline.h:107</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage_html_ab25f53e961fe5b3871feded8ea9b97b3"><div class="ttname"><a href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab25f53e961fe5b3871feded8ea9b97b3">impala::HdfsParquetTableWriter::BaseColumnWriter::DataPage::finalized</a></div><div class="ttdeci">bool finalized</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00187">hdfs-parquet-table-writer.cc:187</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a7ea48982ea52afc394148b5813afe9b7"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a7ea48982ea52afc394148b5813afe9b7">impala::HdfsParquetTableWriter::BaseColumnWriter::type</a></div><div class="ttdeci">const ColumnType & type() const </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00145">hdfs-parquet-table-writer.cc:145</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html"><div class="ttname"><a href="classimpala_1_1Status.html">impala::Status</a></div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00081">status.h:81</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a8e34e69fbf492c3c03cd417f68f75bd4"><div class="ttname"><a href="classimpala_1_1Codec.html#a8e34e69fbf492c3c03cd417f68f75bd4">impala::Codec::GetCodecName</a></div><div class="ttdeci">static std::string GetCodecName(THdfsCompression::type)</div><div class="ttdoc">Return the name of a compression algorithm. </div><div class="ttdef"><b>Definition:</b> <a href="codec_8cc_source.html#l00050">codec.cc:50</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_aa0ef7d2984567dd8995251198975113b"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#aa0ef7d2984567dd8995251198975113b">impala::HdfsParquetTableWriter::ColumnWriter::ColumnWriter</a></div><div class="ttdeci">ColumnWriter(HdfsParquetTableWriter *parent, ExprContext *ctx, const THdfsCompression::type &codec)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00240">hdfs-parquet-table-writer.cc:240</a></div></div> |
| <div class="ttc" id="classimpala_1_1ParquetPlainEncoder_html_af05564bd80bdc32ae9fcf9a39240add7"><div class="ttname"><a href="classimpala_1_1ParquetPlainEncoder.html#af05564bd80bdc32ae9fcf9a39240add7">impala::ParquetPlainEncoder::Encode</a></div><div class="ttdeci">static int Encode(uint8_t *buffer, int fixed_len_size, const T &t)</div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00166">parquet-common.h:166</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html">impala::HdfsParquetTableWriter::BoolColumnWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00334">hdfs-parquet-table-writer.cc:334</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a8e892339c263b2aaa3712139b0a0c944"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a8e892339c263b2aaa3712139b0a0c944">impala::TYPE_INT</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00033">types.h:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_aefde0f92c3063e01e162d87dc207a2ab"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#aefde0f92c3063e01e162d87dc207a2ab">impala::HdfsParquetTableWriter::BaseColumnWriter::NewPage</a></div><div class="ttdeci">void NewPage()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00597">hdfs-parquet-table-writer.cc:597</a></div></div> |
| <div class="ttc" id="structimpala_1_1ColumnType_html_a519bee3e0fdcf790708be4b5caebbcf4"><div class="ttname"><a href="structimpala_1_1ColumnType.html#a519bee3e0fdcf790708be4b5caebbcf4">impala::ColumnType::type</a></div><div class="ttdeci">PrimitiveType type</div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00060">types.h:60</a></div></div> |
| <div class="ttc" id="classimpala_1_1ThriftSerializer_html"><div class="ttname"><a href="classimpala_1_1ThriftSerializer.html">impala::ThriftSerializer</a></div><div class="ttdef"><b>Definition:</b> <a href="thrift-util_8h_source.html#l00039">thrift-util.h:39</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a9e083b0deadc8fd39e06d065b4d66b7b"><div class="ttname"><a href="namespaceimpala.html#a9e083b0deadc8fd39e06d065b4d66b7b">impala::IMPALA_TO_PARQUET_TYPES</a></div><div class="ttdeci">const parquet::Type::type IMPALA_TO_PARQUET_TYPES[]</div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00035">parquet-common.h:35</a></div></div> |
| <div class="ttc" id="classimpala_1_1MemPool_html"><div class="ttname"><a href="classimpala_1_1MemPool.html">impala::MemPool</a></div><div class="ttdef"><b>Definition:</b> <a href="mem-pool_8h_source.html#l00077">mem-pool.h:77</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_adc231e17933fbc4385f968d7eecc5ee2"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">impala::RuntimeState::query_options</a></div><div class="ttdeci">const TQueryOptions & query_options() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00095">runtime-state.h:95</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a69016580aa437551ed8126fa652a75dc"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a69016580aa437551ed8126fa652a75dc">impala::HdfsParquetTableWriter::BaseColumnWriter::codec</a></div><div class="ttdeci">parquet::CompressionCodec::type codec() const </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00149">hdfs-parquet-table-writer.cc:149</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html"><div class="ttname"><a href="classimpala_1_1RuntimeState.html">impala::RuntimeState</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00069">runtime-state.h:69</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a226ca85a62a609e56a3bca74f7dc90d6"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a226ca85a62a609e56a3bca74f7dc90d6">impala::HdfsParquetTableWriter::BaseColumnWriter::values_buffer_</a></div><div class="ttdeci">uint8_t * values_buffer_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00230">hdfs-parquet-table-writer.cc:230</a></div></div> |
| <div class="ttc" id="bit-util_8h_html"><div class="ttname"><a href="bit-util_8h.html">bit-util.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871ad19c24e4b9c67ed07a2331e704cf08ba"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ad19c24e4b9c67ed07a2331e704cf08ba">impala::TYPE_SMALLINT</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00032">types.h:32</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a4c01f227dc12d859d5824e7c91d6d1aa"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">impala::RuntimeState::obj_pool</a></div><div class="ttdeci">ObjectPool * obj_pool() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00092">runtime-state.h:92</a></div></div> |
| <div class="ttc" id="debug-util_8h_html"><div class="ttname"><a href="debug-util_8h.html">debug-util.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a303f0997b8e8b3eed05c5bdd21325f82"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a303f0997b8e8b3eed05c5bdd21325f82">impala::HdfsParquetTableWriter::BaseColumnWriter::~BaseColumnWriter</a></div><div class="ttdeci">virtual ~BaseColumnWriter()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00107">hdfs-parquet-table-writer.cc:107</a></div></div> |
| <div class="ttc" id="classimpala_1_1TableDescriptor_html_aa5b2e225d0d68fa5ef0313db79d0ffb9"><div class="ttname"><a href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">impala::TableDescriptor::num_clustering_cols</a></div><div class="ttdeci">int num_clustering_cols() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00153">descriptors.h:153</a></div></div> |
| <div class="ttc" id="thrift-util_8h_html"><div class="ttname"><a href="thrift-util_8h.html">thrift-util.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_afa31a28f32a0b57ebaebe319846cdd43"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#afa31a28f32a0b57ebaebe319846cdd43">impala::HdfsParquetTableWriter::~HdfsParquetTableWriter</a></div><div class="ttdeci">~HdfsParquetTableWriter()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00632">hdfs-parquet-table-writer.cc:632</a></div></div> |
| <div class="ttc" id="structimpala_1_1ColumnType_html_a6fbb7face292b91017b89af34defcad9"><div class="ttname"><a href="structimpala_1_1ColumnType.html#a6fbb7face292b91017b89af34defcad9">impala::ColumnType::GetByteSize</a></div><div class="ttdeci">int GetByteSize() const </div><div class="ttdoc">Returns the byte size of this type. Returns 0 for variable length types. </div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00178">types.h:178</a></div></div> |
| <div class="ttc" id="string-value_8inline_8h_html"><div class="ttname"><a href="string-value_8inline_8h.html">string-value.inline.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a749e28247ff20797c6b920c1c892fd14"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a749e28247ff20797c6b920c1c892fd14">impala::HdfsParquetTableWriter::BoolColumnWriter</a></div><div class="ttdeci">friend class BoolColumnWriter</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00108">hdfs-parquet-table-writer.h:108</a></div></div> |
| <div class="ttc" id="classuint64__t_html"><div class="ttname"><a href="classuint64__t.html">uint64_t</a></div></div> |
| <div class="ttc" id="hdfs-parquet-table-writer_8cc_html_ac7c634c7e5c4f05df3731ec71156d191"><div class="ttname"><a href="hdfs-parquet-table-writer_8cc.html#ac7c634c7e5c4f05df3731ec71156d191">MAX_DICTIONARY_ENTRIES</a></div><div class="ttdeci">static const int MAX_DICTIONARY_ENTRIES</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00057">hdfs-parquet-table-writer.cc:57</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a2134f91402d30cc178526c949c00603c"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a2134f91402d30cc178526c949c00603c">impala::HdfsParquetTableWriter::HdfsParquetTableWriter</a></div><div class="ttdeci">HdfsParquetTableWriter(HdfsTableSink *parent, RuntimeState *state, OutputPartition *output_partition, const HdfsPartitionDescriptor *part_desc, const HdfsTableDescriptor *table_desc, const std::vector< ExprContext * > &output_expr_ctxs)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00618">hdfs-parquet-table-writer.cc:618</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a32b5ddbc74fd317fb16ebdb859c7e167"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a32b5ddbc74fd317fb16ebdb859c7e167">impala::HdfsParquetTableWriter::BaseColumnWriter::num_data_pages_</a></div><div class="ttdeci">int num_data_pages_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00205">hdfs-parquet-table-writer.cc:205</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a936286363e052834631db5c19be3187d"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a936286363e052834631db5c19be3187d">impala::TYPE_BOOLEAN</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00030">types.h:30</a></div></div> |
| <div class="ttc" id="rle-encoding_8h_html"><div class="ttname"><a href="rle-encoding_8h.html">rle-encoding.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_a3601c468e238682d353bd48a51ba7432"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#a3601c468e238682d353bd48a51ba7432">impala::HdfsTableWriter::output_expr_ctxs_</a></div><div class="ttdeci">std::vector< ExprContext * > output_expr_ctxs_</div><div class="ttdoc">Expressions that materialize output values. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00124">hdfs-table-writer.h:124</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871ac5602f92ce48738739f551df8ff3e4a7"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ac5602f92ce48738739f551df8ff3e4a7">impala::TYPE_BIGINT</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00034">types.h:34</a></div></div> |
| <div class="ttc" id="classimpala_1_1DictEncoder_html"><div class="ttname"><a href="classimpala_1_1DictEncoder.html">impala::DictEncoder</a></div><div class="ttdef"><b>Definition:</b> <a href="dict-encoding_8h_source.html#l00102">dict-encoding.h:102</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_adc18fee9bd07043ebc621d91bb8f57f2"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#adc18fee9bd07043ebc621d91bb8f57f2">impala::HdfsParquetTableWriter::BaseColumnWriter::parent_</a></div><div class="ttdeci">HdfsParquetTableWriter * parent_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00193">hdfs-parquet-table-writer.cc:193</a></div></div> |
| <div class="ttc" id="runtime-state_8h_html"><div class="ttname"><a href="runtime-state_8h.html">runtime-state.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1TableDescriptor_html_adbb3a74120d7bddf6d56eee5364aea2d"><div class="ttname"><a href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">impala::TableDescriptor::col_names</a></div><div class="ttdeci">const std::vector< std::string > & col_names() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00165">descriptors.h:165</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a6c5c9cee1f82ec607f54eda9bc339543"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a6c5c9cee1f82ec607f54eda9bc339543">impala::OutputPartition::num_rows</a></div><div class="ttdeci">int64_t num_rows</div><div class="ttdoc">Records number of rows appended to the current file in this partition. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00077">hdfs-table-sink.h:77</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a466fbf71b0a2a2b1701e8d1b3e0fa64d"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a466fbf71b0a2a2b1701e8d1b3e0fa64d">impala::HdfsParquetTableWriter::BaseColumnWriter::FinalizeCurrentPage</a></div><div class="ttdeci">virtual void FinalizeCurrentPage()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00522">hdfs-parquet-table-writer.cc:522</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowBatch_html"><div class="ttname"><a href="classimpala_1_1RowBatch.html">impala::RowBatch</a></div><div class="ttdef"><b>Definition:</b> <a href="row-batch_8h_source.html#l00066">row-batch.h:66</a></div></div> |
| <div class="ttc" id="version_8h_html"><div class="ttname"><a href="version_8h.html">version.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a6386ce29bdc47fde38d6a3b11c668af0"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a6386ce29bdc47fde38d6a3b11c668af0">impala::HdfsParquetTableWriter::WriteFileHeader</a></div><div class="ttdeci">Status WriteFileHeader()</div><div class="ttdoc">Write the file header information to the output file. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00920">hdfs-parquet-table-writer.cc:920</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a6c65b99c47cbd2ad8e475fc5bac00e3a"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a6c65b99c47cbd2ad8e475fc5bac00e3a">impala::HdfsParquetTableWriter::BaseColumnWriter::pages_</a></div><div class="ttdeci">vector< DataPage > pages_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00201">hdfs-parquet-table-writer.cc:201</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a3fcd37f3239058858623c0a52c420f21"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a3fcd37f3239058858623c0a52c420f21">impala::HdfsParquetTableWriter::columns_</a></div><div class="ttdeci">std::vector< BaseColumnWriter * > columns_</div><div class="ttdoc">array of pointers to column information. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00143">hdfs-parquet-table-writer.h:143</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html">impala::HdfsTableWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00033">hdfs-table-writer.h:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_aaa09b09c831fa3d0f1f29e1b67169eae"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#aaa09b09c831fa3d0f1f29e1b67169eae">impala::HdfsParquetTableWriter::BaseColumnWriter::Reset</a></div><div class="ttdeci">virtual void Reset()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00130">hdfs-parquet-table-writer.cc:130</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a67a6abe5cd883902e5adddea9c68f54e"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a67a6abe5cd883902e5adddea9c68f54e">impala::HdfsParquetTableWriter::per_file_mem_pool_</a></div><div class="ttdeci">boost::scoped_ptr< MemPool > per_file_mem_pool_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00169">hdfs-parquet-table-writer.h:169</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_a742827844080d45d514719742cb3e7f5"><div class="ttname"><a href="classimpala_1_1Expr.html#a742827844080d45d514719742cb3e7f5">impala::Expr::type</a></div><div class="ttdeci">const ColumnType & type() const </div><div class="ttdef"><b>Definition:</b> <a href="expr_8h_source.html#l00145">expr.h:145</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a8608f5639292d04ccab1c9fedb73ea7f"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8608f5639292d04ccab1c9fedb73ea7f">impala::HdfsParquetTableWriter::BaseColumnWriter::WriteDictDataPage</a></div><div class="ttdeci">void WriteDictDataPage()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00416">hdfs-parquet-table-writer.cc:416</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a76ba4a28fb2cad5241eb48f32fdc2941"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a76ba4a28fb2cad5241eb48f32fdc2941">impala::HdfsParquetTableWriter::BaseColumnWriter::total_compressed_size</a></div><div class="ttdeci">uint64_t total_compressed_size() const </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00147">hdfs-parquet-table-writer.cc:147</a></div></div> |
| <div class="ttc" id="compiler-util_8h_html_a9acc330d508b9a3b775cfdf7ce405e7d"><div class="ttname"><a href="compiler-util_8h.html#a9acc330d508b9a3b775cfdf7ce405e7d">UNLIKELY</a></div><div class="ttdeci">#define UNLIKELY(expr)</div><div class="ttdef"><b>Definition:</b> <a href="compiler-util_8h_source.html#l00033">compiler-util.h:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a8019ba8c269e32342b407c6e74bc931e"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a8019ba8c269e32342b407c6e74bc931e">impala::HdfsTableSink::hdfs_write_timer</a></div><div class="ttdeci">RuntimeProfile::Counter * hdfs_write_timer()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00154">hdfs-table-sink.h:154</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_ab74915a4a656b8cb4cce2cf71f31036a"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ab74915a4a656b8cb4cce2cf71f31036a">impala::HdfsParquetTableWriter::BaseColumnWriter::dict_encoder_base_</a></div><div class="ttdeci">DictEncoderBase * dict_encoder_base_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00221">hdfs-parquet-table-writer.cc:221</a></div></div> |
| <div class="ttc" id="classimpala_1_1BitWriter_html"><div class="ttname"><a href="classimpala_1_1BitWriter.html">impala::BitWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="bit-stream-utils_8h_source.html#l00030">bit-stream-utils.h:30</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a2025d785fa7bfb70e8dc84bb91114410"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a2025d785fa7bfb70e8dc84bb91114410">impala::HdfsTableSink::rows_inserted_counter</a></div><div class="ttdeci">RuntimeProfile::Counter * rows_inserted_counter()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00151">hdfs-table-sink.h:151</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_a017d589ce2bdcc502582924169f5d2d7"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#a017d589ce2bdcc502582924169f5d2d7">impala::HdfsTableWriter::stats_</a></div><div class="ttdeci">TInsertStats stats_</div><div class="ttdoc">Subclass should populate any file format specific stats. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00127">hdfs-table-writer.h:127</a></div></div> |
| <div class="ttc" id="structimpala_1_1ColumnType_html"><div class="ttname"><a href="structimpala_1_1ColumnType.html">impala::ColumnType</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00059">types.h:59</a></div></div> |
| <div class="ttc" id="decimal-value_8h_html"><div class="ttname"><a href="decimal-value_8h.html">decimal-value.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a580565665ea944eb64f3f495b1bee1e0"><div class="ttname"><a href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">impala::Status::OK</a></div><div class="ttdeci">static const Status OK</div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00087">status.h:87</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_aaaac900efe9fa7b75331cf70d40a92ce"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#aaaac900efe9fa7b75331cf70d40a92ce">impala::HdfsTableSink::encode_timer</a></div><div class="ttdeci">RuntimeProfile::Counter * encode_timer()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00153">hdfs-table-sink.h:153</a></div></div> |
| <div class="ttc" id="structimpala_1_1StringValue_html_a60791f03f5efcddd11a3afb9ddc833e4"><div class="ttname"><a href="structimpala_1_1StringValue.html#a60791f03f5efcddd11a3afb9ddc833e4">impala::StringValue::CharSlotToPtr</a></div><div class="ttdeci">static char * CharSlotToPtr(void *slot, const ColumnType &type)</div><div class="ttdef"><b>Definition:</b> <a href="string-value_8inline_8h_source.html#l00115">string-value.inline.h:115</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a3e94af6168818406871e631b54287ff6"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a3e94af6168818406871e631b54287ff6">impala::HdfsParquetTableWriter::default_block_size</a></div><div class="ttdeci">virtual uint64_t default_block_size() const </div><div class="ttdoc">Returns the target HDFS block size to use. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00797">hdfs-parquet-table-writer.cc:797</a></div></div> |
| <div class="ttc" id="expr_8h_html"><div class="ttname"><a href="expr_8h.html">expr.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a507adb3975fd7bc994da72f5777a3ef7"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a507adb3975fd7bc994da72f5777a3ef7">impala::HdfsParquetTableWriter::AddRowGroup</a></div><div class="ttdeci">Status AddRowGroup()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00766">hdfs-parquet-table-writer.cc:766</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_ae8afd359ab02b0c78aeb28807da48f32"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ae8afd359ab02b0c78aeb28807da48f32">impala::HdfsParquetTableWriter::BaseColumnWriter::values_buffer_len_</a></div><div class="ttdeci">int values_buffer_len_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00232">hdfs-parquet-table-writer.cc:232</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html">impala::HdfsPartitionDescriptor</a></div><div class="ttdoc">Metadata for a single partition inside an Hdfs table. </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00177">descriptors.h:177</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a93e294730c06b45f635bba83d84d96ec"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a93e294730c06b45f635bba83d84d96ec">impala::HdfsParquetTableWriter::AppendRowBatch</a></div><div class="ttdeci">virtual Status AppendRowBatch(RowBatch *batch, const std::vector< int32_t > &row_group_indices, bool *new_file)</div><div class="ttdoc">Appends parquet representation of rows in the batch to the current file. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00862">hdfs-parquet-table-writer.cc:862</a></div></div> |
| <div class="ttc" id="names_8h_html"><div class="ttname"><a href="names_8h.html">names.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html">impala::HdfsParquetTableWriter::ColumnWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00237">hdfs-parquet-table-writer.cc:237</a></div></div> |
| <div class="ttc" id="classimpala_1_1ObjectPool_html_ab191078c99825682a0a9915ca1e0c05c"><div class="ttname"><a href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">impala::ObjectPool::Add</a></div><div class="ttdeci">T * Add(T *t)</div><div class="ttdef"><b>Definition:</b> <a href="object-pool_8h_source.html#l00042">object-pool.h:42</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a8621a48438ae5353ff6b864294a8cf4e"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8621a48438ae5353ff6b864294a8cf4e">impala::HdfsParquetTableWriter::BaseColumnWriter::total_uncompressed_size</a></div><div class="ttdeci">uint64_t total_uncompressed_size() const </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00148">hdfs-parquet-table-writer.cc:148</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a8ad355e28e35c5d73086b838b6eb56aa"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a8ad355e28e35c5d73086b838b6eb56aa">impala::HdfsParquetTableWriter::BaseColumnWriter::Flush</a></div><div class="ttdeci">Status Flush(int64_t *file_pos, int64_t *first_data_page, int64_t *first_dictionary_page)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00432">hdfs-parquet-table-writer.cc:432</a></div></div> |
| <div class="ttc" id="version_8h_html_a6cfbcb9525a23b464946d05af6177421"><div class="ttname"><a href="version_8h.html#a6cfbcb9525a23b464946d05af6177421">IMPALA_BUILD_VERSION</a></div><div class="ttdeci">#define IMPALA_BUILD_VERSION</div><div class="ttdef"><b>Definition:</b> <a href="version_8h_source.html#l00021">version.h:21</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a23c0355951ae6f3602e935a0688da7dc"><div class="ttname"><a href="namespaceimpala.html#a23c0355951ae6f3602e935a0688da7dc">impala::PARQUET_CURRENT_VERSION</a></div><div class="ttdeci">const uint32_t PARQUET_CURRENT_VERSION</div><div class="ttdef"><b>Definition:</b> <a href="parquet-common_8h_source.html#l00031">parquet-common.h:31</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter_html_acf6e2e3b8c295c2a485c619fbf5596d9"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BoolColumnWriter.html#acf6e2e3b8c295c2a485c619fbf5596d9">impala::HdfsParquetTableWriter::BoolColumnWriter::BoolColumnWriter</a></div><div class="ttdeci">BoolColumnWriter(HdfsParquetTableWriter *parent, ExprContext *ctx, const THdfsCompression::type &codec)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00337">hdfs-parquet-table-writer.cc:337</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a81087da87fdc569ea9ad101e34391359"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a81087da87fdc569ea9ad101e34391359">impala::TYPE_STRING</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00038">types.h:38</a></div></div> |
| <div class="ttc" id="logging_8h_html_a0bdc3628c740982e847cd24114c07994"><div class="ttname"><a href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a></div><div class="ttdeci">#define VLOG_FILE</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00058">logging.h:58</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a3726f7987a677970a9a193a585cd14dc"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a3726f7987a677970a9a193a585cd14dc">impala::HdfsParquetTableWriter::BaseColumnWriter::codec_</a></div><div class="ttdeci">THdfsCompression::type codec_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00196">hdfs-parquet-table-writer.cc:196</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a25e7c1ba35a095ceddc08db22e3f8619"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a25e7c1ba35a095ceddc08db22e3f8619">impala::HdfsParquetTableWriter::WriteFileFooter</a></div><div class="ttdeci">Status WriteFileFooter()</div><div class="ttdoc">Write the file metadata and footer. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00977">hdfs-parquet-table-writer.cc:977</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a7a19f69a9969431237f11ac0f982add1"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a7a19f69a9969431237f11ac0f982add1">impala::TYPE_TINYINT</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00031">types.h:31</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage_html_ab21b705742505262e2c7c96910400c55"><div class="ttname"><a href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#ab21b705742505262e2c7c96910400c55">impala::HdfsParquetTableWriter::BaseColumnWriter::DataPage::header</a></div><div class="ttdeci">PageHeader header</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00175">hdfs-parquet-table-writer.cc:175</a></div></div> |
| <div class="ttc" id="classimpala_1_1BufferBuilder_html_a64ca6e2b963d5001dd220f98be4c0379"><div class="ttname"><a href="classimpala_1_1BufferBuilder.html#a64ca6e2b963d5001dd220f98be4c0379">impala::BufferBuilder::capacity</a></div><div class="ttdeci">int capacity() const </div><div class="ttdef"><b>Definition:</b> <a href="buffer-builder_8h_source.html#l00047">buffer-builder.h:47</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_ab3c85c3219e6cc628543c2ab08b1b204"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#ab3c85c3219e6cc628543c2ab08b1b204">impala::HdfsParquetTableWriter::BaseColumnWriter::current_encoding_</a></div><div class="ttdeci">Encoding::type current_encoding_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00218">hdfs-parquet-table-writer.cc:218</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e"><div class="ttname"><a href="namespaceimpala.html#a99e965f9979b51a61b3cdb86c0d5b315ad7e3bfc9d42f7e586700e37b00e1af0e">impala::PLAIN</a></div><div class="ttdef"><b>Definition:</b> <a href="webserver_8cc_source.html#l00122">webserver.cc:122</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExprContext_html_a2e652a852e99d5cd375dce3f91520aee"><div class="ttname"><a href="classimpala_1_1ExprContext.html#a2e652a852e99d5cd375dce3f91520aee">impala::ExprContext::root</a></div><div class="ttdeci">Expr * root()</div><div class="ttdef"><b>Definition:</b> <a href="expr-context_8h_source.html#l00105">expr-context.h:105</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a05f899a25bef44bd805dbe71d0c142e6"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a05f899a25bef44bd805dbe71d0c142e6">impala::HdfsParquetTableWriter::parquet_stats_</a></div><div class="ttdeci">TParquetInsertStats parquet_stats_</div><div class="ttdoc">For each column, the on disk size written. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00181">hdfs-parquet-table-writer.h:181</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableWriter_html_a25eed1b5b3cb444d8bfdc62890238ff7"><div class="ttname"><a href="classimpala_1_1HdfsTableWriter.html#a25eed1b5b3cb444d8bfdc62890238ff7">impala::HdfsTableWriter::output_</a></div><div class="ttdeci">OutputPartition * output_</div><div class="ttdoc">Structure describing partition written to by this writer. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-writer_8h_source.html#l00118">hdfs-table-writer.h:118</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_a934d462020552f342270418e27135dbd"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a934d462020552f342270418e27135dbd">impala::HdfsParquetTableWriter::ColumnWriter::EncodeValue</a></div><div class="ttdeci">virtual bool EncodeValue(void *value, int64_t *bytes_needed)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00258">hdfs-parquet-table-writer.cc:258</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_html_a076783ce2c4d6603675879867334fe7d"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter.html#a076783ce2c4d6603675879867334fe7d">impala::HdfsParquetTableWriter::BaseColumnWriter::page_size_</a></div><div class="ttdeci">int64_t page_size_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00212">hdfs-parquet-table-writer.cc:212</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871ac750a41f3696ba9ddece634df5452994"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871ac750a41f3696ba9ddece634df5452994">impala::TYPE_FLOAT</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00035">types.h:35</a></div></div> |
| <div class="ttc" id="hdfs-parquet-table-writer_8h_html"><div class="ttname"><a href="hdfs-parquet-table-writer_8h.html">hdfs-parquet-table-writer.h</a></div></div> |
| <div class="ttc" id="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage_html_a570d69e9568d8b34483d2746cd90daac"><div class="ttname"><a href="structimpala_1_1HdfsParquetTableWriter_1_1BaseColumnWriter_1_1DataPage.html#a570d69e9568d8b34483d2746cd90daac">impala::HdfsParquetTableWriter::BaseColumnWriter::DataPage::num_non_null</a></div><div class="ttdeci">int num_non_null</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00190">hdfs-parquet-table-writer.cc:190</a></div></div> |
| <div class="ttc" id="classimpala_1_1BitUtil_html_aaf681500e2dd65a55beb6e5628c66a77"><div class="ttname"><a href="classimpala_1_1BitUtil.html#aaf681500e2dd65a55beb6e5628c66a77">impala::BitUtil::RoundUp</a></div><div class="ttdeci">static int RoundUp(int value, int factor)</div><div class="ttdoc">Returns 'value' rounded up to the nearest multiple of 'factor'. </div><div class="ttdef"><b>Definition:</b> <a href="bit-util_8h_source.html#l00037">bit-util.h:37</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa5bce0d3309643287a67504146c2f871a62719f90a13430bfdc4a8455724f6fdf"><div class="ttname"><a href="namespaceimpala.html#aa5bce0d3309643287a67504146c2f871a62719f90a13430bfdc4a8455724f6fdf">impala::TYPE_DECIMAL</a></div><div class="ttdef"><b>Definition:</b> <a href="types_8h_source.html#l00042">types.h:42</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html_a899619de5a18c34129aed0b5ad217af5"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html#a899619de5a18c34129aed0b5ad217af5">impala::HdfsParquetTableWriter::MinBlockSize</a></div><div class="ttdeci">int64_t MinBlockSize() const </div><div class="ttdoc">Minimum allowable block size in bytes. This is a function of the number of columns. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00792">hdfs-parquet-table-writer.cc:792</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter_html_a7530ca781087774aa969e2f03600e234"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter_1_1ColumnWriter.html#a7530ca781087774aa969e2f03600e234">impala::HdfsParquetTableWriter::ColumnWriter::temp_</a></div><div class="ttdeci">StringValue temp_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8cc_source.html#l00314">hdfs-parquet-table-writer.cc:314</a></div></div> |
| </div><!-- fragment --></div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_e5d120be6b5e8a44336cbfd013b25604.html">be</a></li><li class="navelem"><a class="el" href="dir_68cf7cafb51a962d5bc4848b83cab0de.html">src</a></li><li class="navelem"><a class="el" href="dir_479336c0a15f2c0737bcafcf969f884c.html">exec</a></li><li class="navelem"><a class="el" href="hdfs-parquet-table-writer_8cc.html">hdfs-parquet-table-writer.cc</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:35 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
