| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: be/src/exec/hdfs-table-sink.cc Source File</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('hdfs-table-sink_8cc_source.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">hdfs-table-sink.cc</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| <a href="hdfs-table-sink_8cc.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">// Copyright 2012 Cloudera Inc.</span></div> |
| <div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment">// Licensed under the Apache License, Version 2.0 (the "License");</span></div> |
| <div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment">// you may not use this file except in compliance with the License.</span></div> |
| <div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment">// You may obtain a copy of the License at</span></div> |
| <div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment">// http://www.apache.org/licenses/LICENSE-2.0</span></div> |
| <div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment">// Unless required by applicable law or agreed to in writing, software</span></div> |
| <div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment">// distributed under the License is distributed on an "AS IS" BASIS,</span></div> |
| <div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment">// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div> |
| <div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment">// See the License for the specific language governing permissions and</span></div> |
| <div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment">// limitations under the License.</span></div> |
| <div class="line"><a name="l00014"></a><span class="lineno"> 14</span> </div> |
| <div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="preprocessor">#include "<a class="code" href="hdfs-table-sink_8h.html">exec/hdfs-table-sink.h</a>"</span></div> |
| <div class="line"><a name="l00016"></a><span class="lineno"> 16</span> <span class="preprocessor">#include "<a class="code" href="hdfs-text-table-writer_8h.html">exec/hdfs-text-table-writer.h</a>"</span></div> |
| <div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="preprocessor">#include "<a class="code" href="hdfs-sequence-table-writer_8h.html">exec/hdfs-sequence-table-writer.h</a>"</span></div> |
| <div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="preprocessor">#include "<a class="code" href="hdfs-avro-table-writer_8h.html">exec/hdfs-avro-table-writer.h</a>"</span></div> |
| <div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="preprocessor">#include "<a class="code" href="hdfs-parquet-table-writer_8h.html">exec/hdfs-parquet-table-writer.h</a>"</span></div> |
| <div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="preprocessor">#include "<a class="code" href="exec-node_8h.html">exec/exec-node.h</a>"</span></div> |
| <div class="line"><a name="l00021"></a><span class="lineno"> 21</span> <span class="preprocessor">#include "gen-cpp/ImpalaInternalService_constants.h"</span></div> |
| <div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="preprocessor">#include "<a class="code" href="hdfs-util_8h.html">util/hdfs-util.h</a>"</span></div> |
| <div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="preprocessor">#include "<a class="code" href="expr_8h.html">exprs/expr.h</a>"</span></div> |
| <div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="preprocessor">#include "<a class="code" href="expr-context_8h.html">exprs/expr-context.h</a>"</span></div> |
| <div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="preprocessor">#include "<a class="code" href="hdfs-fs-cache_8h.html">runtime/hdfs-fs-cache.h</a>"</span></div> |
| <div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="preprocessor">#include "<a class="code" href="raw-value_8h.html">runtime/raw-value.h</a>"</span></div> |
| <div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="preprocessor">#include "<a class="code" href="row-batch_8h.html">runtime/row-batch.h</a>"</span></div> |
| <div class="line"><a name="l00028"></a><span class="lineno"> 28</span> <span class="preprocessor">#include "<a class="code" href="runtime-state_8h.html">runtime/runtime-state.h</a>"</span></div> |
| <div class="line"><a name="l00029"></a><span class="lineno"> 29</span> <span class="preprocessor">#include "<a class="code" href="string-value_8inline_8h.html">runtime/string-value.inline.h</a>"</span></div> |
| <div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="preprocessor">#include "<a class="code" href="impalad-metrics_8h.html">util/impalad-metrics.h</a>"</span></div> |
| <div class="line"><a name="l00031"></a><span class="lineno"> 31</span> <span class="preprocessor">#include "<a class="code" href="mem-tracker_8h.html">runtime/mem-tracker.h</a>"</span></div> |
| <div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="preprocessor">#include "<a class="code" href="url-coding_8h.html">util/url-coding.h</a>"</span></div> |
| <div class="line"><a name="l00033"></a><span class="lineno"> 33</span> </div> |
| <div class="line"><a name="l00034"></a><span class="lineno"> 34</span> <span class="preprocessor">#include <vector></span></div> |
| <div class="line"><a name="l00035"></a><span class="lineno"> 35</span> <span class="preprocessor">#include <sstream></span></div> |
| <div class="line"><a name="l00036"></a><span class="lineno"> 36</span> <span class="preprocessor">#include <gutil/strings/substitute.h></span></div> |
| <div class="line"><a name="l00037"></a><span class="lineno"> 37</span> <span class="preprocessor">#include <<a class="code" href="hdfs_8h.html">hdfs.h</a>></span></div> |
| <div class="line"><a name="l00038"></a><span class="lineno"> 38</span> <span class="preprocessor">#include <boost/scoped_ptr.hpp></span></div> |
| <div class="line"><a name="l00039"></a><span class="lineno"> 39</span> <span class="preprocessor">#include <boost/date_time/posix_time/posix_time.hpp></span></div> |
| <div class="line"><a name="l00040"></a><span class="lineno"> 40</span> <span class="preprocessor">#include <stdlib.h></span></div> |
| <div class="line"><a name="l00041"></a><span class="lineno"> 41</span> </div> |
| <div class="line"><a name="l00042"></a><span class="lineno"> 42</span> <span class="preprocessor">#include "gen-cpp/ImpalaInternalService_constants.h"</span></div> |
| <div class="line"><a name="l00043"></a><span class="lineno"> 43</span> </div> |
| <div class="line"><a name="l00044"></a><span class="lineno"> 44</span> <span class="preprocessor">#include "<a class="code" href="names_8h.html">common/names.h</a>"</span></div> |
| <div class="line"><a name="l00045"></a><span class="lineno"> 45</span> </div> |
| <div class="line"><a name="l00046"></a><span class="lineno"> 46</span> <span class="keyword">using</span> boost::posix_time::microsec_clock;</div> |
| <div class="line"><a name="l00047"></a><span class="lineno"> 47</span> <span class="keyword">using</span> boost::posix_time::ptime;</div> |
| <div class="line"><a name="l00048"></a><span class="lineno"> 48</span> <span class="keyword">using namespace </span>strings;</div> |
| <div class="line"><a name="l00049"></a><span class="lineno"> 49</span> </div> |
| <div class="line"><a name="l00050"></a><span class="lineno"> 50</span> <span class="keyword">namespace </span>impala {</div> |
| <div class="line"><a name="l00051"></a><span class="lineno"> 51</span> </div> |
| <div class="line"><a name="l00052"></a><span class="lineno"><a class="line" href="namespaceimpala.html#af0847904c2bf82facb74614fc46a1edc"> 52</a></span> <span class="keyword">const</span> <span class="keyword">static</span> <span class="keywordtype">string</span>& <a class="code" href="namespaceimpala.html#af0847904c2bf82facb74614fc46a1edc">ROOT_PARTITION_KEY</a> =</div> |
| <div class="line"><a name="l00053"></a><span class="lineno"> 53</span>  g_ImpalaInternalService_constants.ROOT_PARTITION_KEY;</div> |
| <div class="line"><a name="l00054"></a><span class="lineno"> 54</span> </div> |
| <div class="line"><a name="l00055"></a><span class="lineno"> 55</span> HdfsTableSink::HdfsTableSink(<span class="keyword">const</span> <a class="code" href="classimpala_1_1RowDescriptor.html">RowDescriptor</a>& <a class="code" href="namespaceimpala.html#aa98447566dd6700a2faaaaf3059f4d95">row_desc</a>,</div> |
| <div class="line"><a name="l00056"></a><span class="lineno"> 56</span>  <span class="keyword">const</span> vector<TExpr>& select_list_texprs,</div> |
| <div class="line"><a name="l00057"></a><span class="lineno"> 57</span>  <span class="keyword">const</span> TDataSink& tsink)</div> |
| <div class="line"><a name="l00058"></a><span class="lineno"> 58</span>  : <a class="code" href="namespaceimpala.html#ad393a2093952c5b6f3a61bd3e1302e61">row_desc_</a>(row_desc),</div> |
| <div class="line"><a name="l00059"></a><span class="lineno"> 59</span>  table_id_(tsink.table_sink.target_table_id),</div> |
| <div class="line"><a name="l00060"></a><span class="lineno"> 60</span>  select_list_texprs_(select_list_texprs),</div> |
| <div class="line"><a name="l00061"></a><span class="lineno"> 61</span>  partition_key_texprs_(tsink.table_sink.hdfs_table_sink.partition_key_exprs),</div> |
| <div class="line"><a name="l00062"></a><span class="lineno"> 62</span>  overwrite_(tsink.table_sink.hdfs_table_sink.overwrite),</div> |
| <div class="line"><a name="l00063"></a><span class="lineno"> 63</span>  has_empty_input_batch_(false) {</div> |
| <div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  DCHECK(tsink.__isset.table_sink);</div> |
| <div class="line"><a name="l00065"></a><span class="lineno"> 65</span> }</div> |
| <div class="line"><a name="l00066"></a><span class="lineno"> 66</span> </div> |
| <div class="line"><a name="l00067"></a><span class="lineno"><a class="line" href="structimpala_1_1OutputPartition.html#ae205c7eebdd3cf806fff0e49582e2265"> 67</a></span> OutputPartition::OutputPartition()</div> |
| <div class="line"><a name="l00068"></a><span class="lineno"> 68</span>  : hdfs_connection(NULL), tmp_hdfs_file(NULL), num_rows(0), num_files(0),</div> |
| <div class="line"><a name="l00069"></a><span class="lineno"> 69</span>  partition_descriptor(NULL) {</div> |
| <div class="line"><a name="l00070"></a><span class="lineno"> 70</span> }</div> |
| <div class="line"><a name="l00071"></a><span class="lineno"> 71</span> </div> |
| <div class="line"><a name="l00072"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#ad99889b28415d19eae2164112e6f507c"> 72</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#ad99889b28415d19eae2164112e6f507c">HdfsTableSink::PrepareExprs</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state) {</div> |
| <div class="line"><a name="l00073"></a><span class="lineno"> 73</span>  <span class="comment">// Prepare select list expressions.</span></div> |
| <div class="line"><a name="l00074"></a><span class="lineno"> 74</span>  <span class="comment">// Disable codegen for these - they would be unused anyway.</span></div> |
| <div class="line"><a name="l00075"></a><span class="lineno"> 75</span>  <span class="comment">// TODO: codegen table sink</span></div> |
| <div class="line"><a name="l00076"></a><span class="lineno"> 76</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(</div> |
| <div class="line"><a name="l00077"></a><span class="lineno"> 77</span>  <a class="code" href="classimpala_1_1Expr.html#a3ae02e50debba50ac5c7b6dd9f8016e7">Expr::Prepare</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>, state, <a class="code" href="classimpala_1_1HdfsTableSink.html#a09ff5ccfa2aed127b0e8a267fa085c94">row_desc_</a>, <a class="code" href="classimpala_1_1DataSink.html#a0a447d23b1d3b6df7b2720439f3e486d">expr_mem_tracker_</a>.get()));</div> |
| <div class="line"><a name="l00078"></a><span class="lineno"> 78</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(</div> |
| <div class="line"><a name="l00079"></a><span class="lineno"> 79</span>  <a class="code" href="classimpala_1_1Expr.html#a3ae02e50debba50ac5c7b6dd9f8016e7">Expr::Prepare</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>, state, <a class="code" href="classimpala_1_1HdfsTableSink.html#a09ff5ccfa2aed127b0e8a267fa085c94">row_desc_</a>, <a class="code" href="classimpala_1_1DataSink.html#a0a447d23b1d3b6df7b2720439f3e486d">expr_mem_tracker_</a>.get()));</div> |
| <div class="line"><a name="l00080"></a><span class="lineno"> 80</span> </div> |
| <div class="line"><a name="l00081"></a><span class="lineno"> 81</span>  <span class="comment">// Prepare partition key exprs and gather dynamic partition key exprs.</span></div> |
| <div class="line"><a name="l00082"></a><span class="lineno"> 82</span>  <span class="keywordflow">for</span> (<span class="keywordtype">size_t</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00083"></a><span class="lineno"> 83</span>  <span class="comment">// Remember non-constant partition key exprs for building hash table of Hdfs files.</span></div> |
| <div class="line"><a name="l00084"></a><span class="lineno"> 84</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[i]->root()->IsConstant()) {</div> |
| <div class="line"><a name="l00085"></a><span class="lineno"> 85</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a0e7f566d373b853ce6b7d7b382f05df8">dynamic_partition_key_expr_ctxs_</a>.push_back(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[i]);</div> |
| <div class="line"><a name="l00086"></a><span class="lineno"> 86</span>  }</div> |
| <div class="line"><a name="l00087"></a><span class="lineno"> 87</span>  }</div> |
| <div class="line"><a name="l00088"></a><span class="lineno"> 88</span>  <span class="comment">// Sanity check.</span></div> |
| <div class="line"><a name="l00089"></a><span class="lineno"> 89</span>  DCHECK_LE(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>.size(), <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">col_names</a>().size())</div> |
| <div class="line"><a name="l00090"></a><span class="lineno"> 90</span>  << <a class="code" href="classimpala_1_1HdfsTableSink.html#a938c70d0fcf84b3c606761be535918ac">DebugString</a>();</div> |
| <div class="line"><a name="l00091"></a><span class="lineno"> 91</span>  DCHECK_EQ(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>.size(), <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">num_clustering_cols</a>())</div> |
| <div class="line"><a name="l00092"></a><span class="lineno"> 92</span>  << <a class="code" href="classimpala_1_1HdfsTableSink.html#a938c70d0fcf84b3c606761be535918ac">DebugString</a>();</div> |
| <div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  DCHECK_GE(<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>.size(),</div> |
| <div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#a665ceaf9c184ec39544c3ecc448c0d11">num_cols</a>() - <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">num_clustering_cols</a>()) << <a class="code" href="classimpala_1_1HdfsTableSink.html#a938c70d0fcf84b3c606761be535918ac">DebugString</a>();</div> |
| <div class="line"><a name="l00095"></a><span class="lineno"> 95</span> </div> |
| <div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  <span class="comment">// Prepare literal partition key exprs</span></div> |
| <div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  BOOST_FOREACH(</div> |
| <div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  <span class="keyword">const</span> HdfsTableDescriptor::PartitionIdToDescriptorMap::value_type& id_to_desc,</div> |
| <div class="line"><a name="l00099"></a><span class="lineno"> 99</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#a9c3f56805e3c2577ff77baf3abf49670">partition_descriptors</a>()) {</div> |
| <div class="line"><a name="l00100"></a><span class="lineno"> 100</span>  <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>* partition = id_to_desc.second;</div> |
| <div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(partition-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#abe51d04af9ad683ef8bc891a89da0300">PrepareExprs</a>(state));</div> |
| <div class="line"><a name="l00102"></a><span class="lineno"> 102</span>  }</div> |
| <div class="line"><a name="l00103"></a><span class="lineno"> 103</span> </div> |
| <div class="line"><a name="l00104"></a><span class="lineno"> 104</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00105"></a><span class="lineno"> 105</span> }</div> |
| <div class="line"><a name="l00106"></a><span class="lineno"> 106</span> </div> |
| <div class="line"><a name="l00107"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a3f2e0429bebe4586bc9140eafd8a21f1"> 107</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#a3f2e0429bebe4586bc9140eafd8a21f1">HdfsTableSink::Prepare</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state) {</div> |
| <div class="line"><a name="l00108"></a><span class="lineno"> 108</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1DataSink.html#ad1d8007e6e079be3695d5db545ecc446">DataSink::Prepare</a>(state));</div> |
| <div class="line"><a name="l00109"></a><span class="lineno"> 109</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a117bc8c377e5da4b5424a8a1615d7b82">unique_id_str_</a> = <a class="code" href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">PrintId</a>(state-><a class="code" href="classimpala_1_1RuntimeState.html#ac985744be040bdbe5b1d4d4967c9c0e0">fragment_instance_id</a>(), <span class="stringliteral">"-"</span>);</div> |
| <div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a32d00c3c1b4eb90fcae15c9ffbb5a3bb">runtime_profile_</a> = state-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(</div> |
| <div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1RuntimeProfile.html">RuntimeProfile</a>(state-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>(), <span class="stringliteral">"HdfsTableSink"</span>));</div> |
| <div class="line"><a name="l00112"></a><span class="lineno"> 112</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(runtime_profile_->total_time_counter());</div> |
| <div class="line"><a name="l00113"></a><span class="lineno"> 113</span> </div> |
| <div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  <span class="comment">// TODO: Consider a system-wide random number generator, initialised in a single place.</span></div> |
| <div class="line"><a name="l00115"></a><span class="lineno"> 115</span>  ptime now = microsec_clock::local_time();</div> |
| <div class="line"><a name="l00116"></a><span class="lineno"> 116</span>  <span class="keywordtype">long</span> seed = (now.time_of_day().seconds() * 1000)</div> |
| <div class="line"><a name="l00117"></a><span class="lineno"> 117</span>  + (now.time_of_day().total_microseconds() / 1000);</div> |
| <div class="line"><a name="l00118"></a><span class="lineno"> 118</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Random seed: "</span> << seed;</div> |
| <div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  srand(seed);</div> |
| <div class="line"><a name="l00120"></a><span class="lineno"> 120</span> </div> |
| <div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1Expr.html#a6b9c01a432ae49b57482c9fc42a73681">Expr::CreateExprTrees</a>(</div> |
| <div class="line"><a name="l00122"></a><span class="lineno"> 122</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>(), <a class="code" href="classimpala_1_1HdfsTableSink.html#a00633ae3b4ac61a6419d543852e67dff">partition_key_texprs_</a>, &<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>));</div> |
| <div class="line"><a name="l00123"></a><span class="lineno"> 123</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1Expr.html#a6b9c01a432ae49b57482c9fc42a73681">Expr::CreateExprTrees</a>(</div> |
| <div class="line"><a name="l00124"></a><span class="lineno"> 124</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>(), <a class="code" href="classimpala_1_1HdfsTableSink.html#a171a32851f95cfd9f4f4283b7859fec9">select_list_texprs_</a>, &<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>));</div> |
| <div class="line"><a name="l00125"></a><span class="lineno"> 125</span> </div> |
| <div class="line"><a name="l00126"></a><span class="lineno"> 126</span>  <span class="comment">// Resolve table id and set input tuple descriptor.</span></div> |
| <div class="line"><a name="l00127"></a><span class="lineno"> 127</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a> = <span class="keyword">static_cast<</span><span class="keyword">const </span><a class="code" href="classimpala_1_1HdfsTableDescriptor.html">HdfsTableDescriptor</a>*<span class="keyword">></span>(</div> |
| <div class="line"><a name="l00128"></a><span class="lineno"> 128</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#abab3a5213f629bebdc266a858dc1d45d">desc_tbl</a>().<a class="code" href="classimpala_1_1DescriptorTbl.html#a2e67f43279256b454184fea3daa9c8da">GetTableDescriptor</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a4ca2934de11e4bde6e068d576e295f43">table_id_</a>));</div> |
| <div class="line"><a name="l00129"></a><span class="lineno"> 129</span> </div> |
| <div class="line"><a name="l00130"></a><span class="lineno"> 130</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a> == NULL) {</div> |
| <div class="line"><a name="l00131"></a><span class="lineno"> 131</span>  stringstream error_msg(<span class="stringliteral">"Failed to get table descriptor for table id: "</span>);</div> |
| <div class="line"><a name="l00132"></a><span class="lineno"> 132</span>  error_msg << <a class="code" href="classimpala_1_1HdfsTableSink.html#a4ca2934de11e4bde6e068d576e295f43">table_id_</a>;</div> |
| <div class="line"><a name="l00133"></a><span class="lineno"> 133</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(error_msg.str());</div> |
| <div class="line"><a name="l00134"></a><span class="lineno"> 134</span>  }</div> |
| <div class="line"><a name="l00135"></a><span class="lineno"> 135</span> </div> |
| <div class="line"><a name="l00136"></a><span class="lineno"> 136</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#add5f44b15e0231c50eba4437a85bc627">staging_dir_</a> = Substitute(<span class="stringliteral">"$0/_impala_insert_staging/$1/"</span>, <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#abaa1ebf5914c72c1454f43b1836ff6dd">hdfs_base_dir</a>(),</div> |
| <div class="line"><a name="l00137"></a><span class="lineno"> 137</span>  <a class="code" href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">PrintId</a>(state-><a class="code" href="classimpala_1_1RuntimeState.html#a6648f284f4c7a5c44f694f363b2a0d88">query_id</a>(), <span class="stringliteral">"_"</span>));</div> |
| <div class="line"><a name="l00138"></a><span class="lineno"> 138</span> </div> |
| <div class="line"><a name="l00139"></a><span class="lineno"> 139</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#ad99889b28415d19eae2164112e6f507c">PrepareExprs</a>(state));</div> |
| <div class="line"><a name="l00140"></a><span class="lineno"> 140</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsFsCache.html#af7b18daf8c4da6bfce2c7612a515bf64">HdfsFsCache::instance</a>()->GetConnection(</div> |
| <div class="line"><a name="l00141"></a><span class="lineno"> 141</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#add5f44b15e0231c50eba4437a85bc627">staging_dir_</a>, &<a class="code" href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">hdfs_connection_</a>));</div> |
| <div class="line"><a name="l00142"></a><span class="lineno"> 142</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ab4065ec85d296b6aa9a4f1106ce4ef7a">mem_tracker_</a>.reset(<span class="keyword">new</span> <a class="code" href="classimpala_1_1MemTracker.html">MemTracker</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), -1, -1, <a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>()-><a class="code" href="namespaceimpala.html#aaebb9aadfa9e5aec064bc26565df06d3">name</a>(),</div> |
| <div class="line"><a name="l00143"></a><span class="lineno"> 143</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#ad6c837d1a8e2d4a995aa00a4df7c5af5">instance_mem_tracker</a>()));</div> |
| <div class="line"><a name="l00144"></a><span class="lineno"> 144</span> </div> |
| <div class="line"><a name="l00145"></a><span class="lineno"> 145</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a06230db843c80fc6f4ca6fe2fd756a75">partitions_created_counter_</a> =</div> |
| <div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  <a class="code" href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"PartitionsCreated"</span>, TUnit::UNIT);</div> |
| <div class="line"><a name="l00147"></a><span class="lineno"> 147</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ab39cf9643eeb89dc752711b5e9437035">files_created_counter_</a> =</div> |
| <div class="line"><a name="l00148"></a><span class="lineno"> 148</span>  <a class="code" href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"FilesCreated"</span>, TUnit::UNIT);</div> |
| <div class="line"><a name="l00149"></a><span class="lineno"> 149</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a54bdc0c70b33283ca488c5d5c8619361">rows_inserted_counter_</a> =</div> |
| <div class="line"><a name="l00150"></a><span class="lineno"> 150</span>  <a class="code" href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"RowsInserted"</span>, TUnit::UNIT);</div> |
| <div class="line"><a name="l00151"></a><span class="lineno"> 151</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a4e5f342bd92c77e85c90ede98dda8923">bytes_written_counter_</a> =</div> |
| <div class="line"><a name="l00152"></a><span class="lineno"> 152</span>  <a class="code" href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"BytesWritten"</span>, TUnit::BYTES);</div> |
| <div class="line"><a name="l00153"></a><span class="lineno"> 153</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a2170cd3291d4d58c5cc93431d5d3e1d4">encode_timer_</a> = <a class="code" href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"EncodeTimer"</span>);</div> |
| <div class="line"><a name="l00154"></a><span class="lineno"> 154</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a99f2add9dbe23c8fceaabcccfb2ee0fc">hdfs_write_timer_</a> = <a class="code" href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"HdfsWriteTimer"</span>);</div> |
| <div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ae21c4cf3b20e10ae6b0a6c8cdac92298">compress_timer_</a> = <a class="code" href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"CompressTimer"</span>);</div> |
| <div class="line"><a name="l00156"></a><span class="lineno"> 156</span> </div> |
| <div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00158"></a><span class="lineno"> 158</span> }</div> |
| <div class="line"><a name="l00159"></a><span class="lineno"> 159</span> </div> |
| <div class="line"><a name="l00160"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a97ae2a720cc57b784f2bd23ec4ee9a15"> 160</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#a97ae2a720cc57b784f2bd23ec4ee9a15">HdfsTableSink::Open</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state) {</div> |
| <div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1Expr.html#aad1111f87951868de4d8f8b2dfe87c5f">Expr::Open</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>, state));</div> |
| <div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1Expr.html#aad1111f87951868de4d8f8b2dfe87c5f">Expr::Open</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>, state));</div> |
| <div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  <span class="comment">// Open literal partition key exprs</span></div> |
| <div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  BOOST_FOREACH(</div> |
| <div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  <span class="keyword">const</span> HdfsTableDescriptor::PartitionIdToDescriptorMap::value_type& id_to_desc,</div> |
| <div class="line"><a name="l00166"></a><span class="lineno"> 166</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#a9c3f56805e3c2577ff77baf3abf49670">partition_descriptors</a>()) {</div> |
| <div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>* partition = id_to_desc.second;</div> |
| <div class="line"><a name="l00168"></a><span class="lineno"> 168</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(partition-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#a3f6d43477d551e065f039c7b074bf4b1">OpenExprs</a>(state));</div> |
| <div class="line"><a name="l00169"></a><span class="lineno"> 169</span>  }</div> |
| <div class="line"><a name="l00170"></a><span class="lineno"> 170</span> </div> |
| <div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  <span class="comment">// Get file format for default partition in table descriptor, and build a map from</span></div> |
| <div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  <span class="comment">// partition key values to partition descriptor for multiple output format support. The</span></div> |
| <div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  <span class="comment">// map is keyed on the concatenation of the non-constant keys of the PARTITION clause of</span></div> |
| <div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  <span class="comment">// the INSERT statement.</span></div> |
| <div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  BOOST_FOREACH(</div> |
| <div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  <span class="keyword">const</span> HdfsTableDescriptor::PartitionIdToDescriptorMap::value_type& id_to_desc,</div> |
| <div class="line"><a name="l00177"></a><span class="lineno"> 177</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#a9c3f56805e3c2577ff77baf3abf49670">partition_descriptors</a>()) {</div> |
| <div class="line"><a name="l00178"></a><span class="lineno"> 178</span>  <span class="keywordflow">if</span> (id_to_desc.first == g_ImpalaInternalService_constants.DEFAULT_PARTITION_ID) {</div> |
| <div class="line"><a name="l00179"></a><span class="lineno"> 179</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#aaf0abb7b8d271e272e23399bb1e110ad">default_partition_</a> = id_to_desc.second;</div> |
| <div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00181"></a><span class="lineno"> 181</span>  <span class="comment">// Build a map whose key is computed from the value of dynamic partition keys for a</span></div> |
| <div class="line"><a name="l00182"></a><span class="lineno"> 182</span>  <span class="comment">// particular partition, and whose value is the descriptor for that partition.</span></div> |
| <div class="line"><a name="l00183"></a><span class="lineno"> 183</span> </div> |
| <div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  <span class="comment">// True if this partition might be written to, false otherwise.</span></div> |
| <div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  <span class="comment">// A partition may be written to iff:</span></div> |
| <div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  <span class="comment">// For all partition key exprs e, either:</span></div> |
| <div class="line"><a name="l00187"></a><span class="lineno"> 187</span>  <span class="comment">// 1. e is not constant</span></div> |
| <div class="line"><a name="l00188"></a><span class="lineno"> 188</span>  <span class="comment">// 2. The value supplied by the query for this partition key is equal to e's</span></div> |
| <div class="line"><a name="l00189"></a><span class="lineno"> 189</span>  <span class="comment">// constant value.</span></div> |
| <div class="line"><a name="l00190"></a><span class="lineno"> 190</span>  <span class="comment">// Only relevant partitions are remembered in partition_descriptor_map_.</span></div> |
| <div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  <span class="keywordtype">bool</span> relevant_partition = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00192"></a><span class="lineno"> 192</span>  <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>* partition = id_to_desc.second;</div> |
| <div class="line"><a name="l00193"></a><span class="lineno"> 193</span>  DCHECK_EQ(partition-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#a87507069eae1cbb5686df05d723f98b9">partition_key_value_ctxs</a>().size(),</div> |
| <div class="line"><a name="l00194"></a><span class="lineno"> 194</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>.size());</div> |
| <div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  vector<ExprContext*> dynamic_partition_key_value_ctxs;</div> |
| <div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  <span class="keywordflow">for</span> (<span class="keywordtype">size_t</span> i = 0; i < <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  <span class="comment">// Remember non-constant partition key exprs for building hash table of Hdfs files</span></div> |
| <div class="line"><a name="l00198"></a><span class="lineno"> 198</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[i]->root()->IsConstant()) {</div> |
| <div class="line"><a name="l00199"></a><span class="lineno"> 199</span>  dynamic_partition_key_value_ctxs.push_back(</div> |
| <div class="line"><a name="l00200"></a><span class="lineno"> 200</span>  partition-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#a87507069eae1cbb5686df05d723f98b9">partition_key_value_ctxs</a>()[i]);</div> |
| <div class="line"><a name="l00201"></a><span class="lineno"> 201</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00202"></a><span class="lineno"> 202</span>  <span class="comment">// Deal with the following: one partition has (year=2009, month=3); another has</span></div> |
| <div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  <span class="comment">// (year=2010, month=3).</span></div> |
| <div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  <span class="comment">// A query like: INSERT INTO TABLE... PARTITION(year=2009) SELECT month FROM...</span></div> |
| <div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  <span class="comment">// would lead to both partitions having the same key modulo ignored constant</span></div> |
| <div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  <span class="comment">// partition keys. So only keep a reference to the partition which matches</span></div> |
| <div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  <span class="comment">// partition_key_values for constant values, since only that is written to.</span></div> |
| <div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  <span class="keywordtype">void</span>* table_partition_key_value =</div> |
| <div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  partition-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#a87507069eae1cbb5686df05d723f98b9">partition_key_value_ctxs</a>()[i]->GetValue(NULL);</div> |
| <div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  <span class="keywordtype">void</span>* target_partition_key_value = <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[i]->GetValue(NULL);</div> |
| <div class="line"><a name="l00211"></a><span class="lineno"> 211</span>  <span class="keywordflow">if</span> (table_partition_key_value == NULL && target_partition_key_value == NULL) {</div> |
| <div class="line"><a name="l00212"></a><span class="lineno"> 212</span>  <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00213"></a><span class="lineno"> 213</span>  }</div> |
| <div class="line"><a name="l00214"></a><span class="lineno"> 214</span>  <span class="keywordflow">if</span> (table_partition_key_value == NULL || target_partition_key_value == NULL</div> |
| <div class="line"><a name="l00215"></a><span class="lineno"> 215</span>  || !<a class="code" href="classimpala_1_1RawValue.html#af526a6e3729755632d191f2bf08628e7">RawValue::Eq</a>(table_partition_key_value, target_partition_key_value,</div> |
| <div class="line"><a name="l00216"></a><span class="lineno"> 216</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[i]->root()->type())) {</div> |
| <div class="line"><a name="l00217"></a><span class="lineno"> 217</span>  relevant_partition = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00218"></a><span class="lineno"> 218</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  }</div> |
| <div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  }</div> |
| <div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  }</div> |
| <div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  <span class="keywordflow">if</span> (relevant_partition) {</div> |
| <div class="line"><a name="l00223"></a><span class="lineno"> 223</span>  <span class="keywordtype">string</span> key;</div> |
| <div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="comment">// It's ok if current_row_ is NULL (which it should be here), since all of these</span></div> |
| <div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  <span class="comment">// expressions are constant, and can therefore be evaluated without a valid row</span></div> |
| <div class="line"><a name="l00226"></a><span class="lineno"> 226</span>  <span class="comment">// context.</span></div> |
| <div class="line"><a name="l00227"></a><span class="lineno"> 227</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a5e6d1761306e721e0e8284fe08ec1229">GetHashTblKey</a>(dynamic_partition_key_value_ctxs, &key);</div> |
| <div class="line"><a name="l00228"></a><span class="lineno"> 228</span>  DCHECK(<a class="code" href="classimpala_1_1HdfsTableSink.html#acc5427a2a031ccd7e40250a6e3aa59c7">partition_descriptor_map_</a>.find(key) == <a class="code" href="classimpala_1_1HdfsTableSink.html#acc5427a2a031ccd7e40250a6e3aa59c7">partition_descriptor_map_</a>.end())</div> |
| <div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  << <span class="stringliteral">"Partitions with duplicate 'static' keys found during INSERT"</span>;</div> |
| <div class="line"><a name="l00230"></a><span class="lineno"> 230</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#acc5427a2a031ccd7e40250a6e3aa59c7">partition_descriptor_map_</a>[key] = partition;</div> |
| <div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  }</div> |
| <div class="line"><a name="l00232"></a><span class="lineno"> 232</span>  }</div> |
| <div class="line"><a name="l00233"></a><span class="lineno"> 233</span>  }</div> |
| <div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsTableSink.html#aaf0abb7b8d271e272e23399bb1e110ad">default_partition_</a> == NULL) {</div> |
| <div class="line"><a name="l00235"></a><span class="lineno"> 235</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"No default partition found for HdfsTextTableSink"</span>);</div> |
| <div class="line"><a name="l00236"></a><span class="lineno"> 236</span>  }</div> |
| <div class="line"><a name="l00237"></a><span class="lineno"> 237</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00238"></a><span class="lineno"> 238</span> }</div> |
| <div class="line"><a name="l00239"></a><span class="lineno"> 239</span> </div> |
| <div class="line"><a name="l00240"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#ae6da527d105f3b2380948e1b4fbc37ce"> 240</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsTableSink.html#ae6da527d105f3b2380948e1b4fbc37ce">HdfsTableSink::BuildHdfsFileNames</a>(</div> |
| <div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  <span class="keyword">const</span> <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>& partition_descriptor,</div> |
| <div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output_partition) {</div> |
| <div class="line"><a name="l00243"></a><span class="lineno"> 243</span> </div> |
| <div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  <span class="comment">// Create final_hdfs_file_name_prefix and tmp_hdfs_file_name_prefix.</span></div> |
| <div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  <span class="comment">// Path: <hdfs_base_dir>/<partition_values>/<unique_id_str></span></div> |
| <div class="line"><a name="l00246"></a><span class="lineno"> 246</span> </div> |
| <div class="line"><a name="l00247"></a><span class="lineno"> 247</span>  <span class="comment">// Temporary files are written under the following path which is unique to this sink:</span></div> |
| <div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  <span class="comment">// <table_dir>/_impala_insert_staging/<query_id>/<per_fragment_unique_id>_dir/</span></div> |
| <div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  <span class="comment">// Both the temporary directory and the file name, when moved to the real partition</span></div> |
| <div class="line"><a name="l00250"></a><span class="lineno"> 250</span>  <span class="comment">// directory must be unique.</span></div> |
| <div class="line"><a name="l00251"></a><span class="lineno"> 251</span>  <span class="comment">// Prefix the directory name with "." to make it hidden and append "_dir" at the end</span></div> |
| <div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  <span class="comment">// of the directory to avoid name clashes for unpartitioned tables.</span></div> |
| <div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  <span class="comment">// The files are located in <partition_values>/<random_value>_data under</span></div> |
| <div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  <span class="comment">// tmp_hdfs_file_name_prefix.</span></div> |
| <div class="line"><a name="l00255"></a><span class="lineno"> 255</span> </div> |
| <div class="line"><a name="l00256"></a><span class="lineno"> 256</span>  <span class="comment">// Use the query id as filename.</span></div> |
| <div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  <span class="keyword">const</span> <span class="keywordtype">string</span>& query_suffix = Substitute(<span class="stringliteral">"$0_$1_data"</span>, <a class="code" href="classimpala_1_1HdfsTableSink.html#a117bc8c377e5da4b5424a8a1615d7b82">unique_id_str_</a>, rand());</div> |
| <div class="line"><a name="l00258"></a><span class="lineno"> 258</span> </div> |
| <div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a443f6bb9e09cdfcc5268e66dc87673c6">tmp_hdfs_dir_name</a> =</div> |
| <div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  Substitute(<span class="stringliteral">"$0/.$1_$2_dir/"</span>, <a class="code" href="classimpala_1_1HdfsTableSink.html#add5f44b15e0231c50eba4437a85bc627">staging_dir_</a>, <a class="code" href="classimpala_1_1HdfsTableSink.html#a117bc8c377e5da4b5424a8a1615d7b82">unique_id_str_</a>, rand());</div> |
| <div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#aae4c6c45ad22865df963002c3f74d25e">tmp_hdfs_file_name_prefix</a> = Substitute(<span class="stringliteral">"$0$1$2"</span>,</div> |
| <div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a443f6bb9e09cdfcc5268e66dc87673c6">tmp_hdfs_dir_name</a>, output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">partition_name</a>,</div> |
| <div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  query_suffix);</div> |
| <div class="line"><a name="l00264"></a><span class="lineno"> 264</span> </div> |
| <div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  <span class="keywordflow">if</span> (partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ae1e66f0a122f72d42cda498511e63c0c">location</a>().empty()) {</div> |
| <div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#aad8b5d001c2558ffc6b6e0b6f09723d8">final_hdfs_file_name_prefix</a> = Substitute(<span class="stringliteral">"$0/$1$2"</span>,</div> |
| <div class="line"><a name="l00267"></a><span class="lineno"> 267</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#abaa1ebf5914c72c1454f43b1836ff6dd">hdfs_base_dir</a>(), output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">partition_name</a>, query_suffix);</div> |
| <div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  <span class="comment">// If the partition descriptor has a location (as set by alter table add partition</span></div> |
| <div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  <span class="comment">// with a location clause), that provides the complete directory path for this</span></div> |
| <div class="line"><a name="l00271"></a><span class="lineno"> 271</span>  <span class="comment">// partition. No partition key suffix ("p=1/j=foo/") should be added.</span></div> |
| <div class="line"><a name="l00272"></a><span class="lineno"> 272</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#aad8b5d001c2558ffc6b6e0b6f09723d8">final_hdfs_file_name_prefix</a> =</div> |
| <div class="line"><a name="l00273"></a><span class="lineno"> 273</span>  Substitute(<span class="stringliteral">"$0/$1"</span>, partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ae1e66f0a122f72d42cda498511e63c0c">location</a>(), query_suffix);</div> |
| <div class="line"><a name="l00274"></a><span class="lineno"> 274</span>  }</div> |
| <div class="line"><a name="l00275"></a><span class="lineno"> 275</span> </div> |
| <div class="line"><a name="l00276"></a><span class="lineno"> 276</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#abfb3777eb9bb647e071f72b3755dfa37">num_files</a> = 0;</div> |
| <div class="line"><a name="l00277"></a><span class="lineno"> 277</span> }</div> |
| <div class="line"><a name="l00278"></a><span class="lineno"> 278</span> </div> |
| <div class="line"><a name="l00279"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#ad05565061b0102660efcd17cfaeb56e1"> 279</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#ad05565061b0102660efcd17cfaeb56e1">HdfsTableSink::CreateNewTmpFile</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state,</div> |
| <div class="line"><a name="l00280"></a><span class="lineno"> 280</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output_partition) {</div> |
| <div class="line"><a name="l00281"></a><span class="lineno"> 281</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"TmpFileCreateTimer"</span>));</div> |
| <div class="line"><a name="l00282"></a><span class="lineno"> 282</span>  stringstream filename;</div> |
| <div class="line"><a name="l00283"></a><span class="lineno"> 283</span>  filename << output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#aae4c6c45ad22865df963002c3f74d25e">tmp_hdfs_file_name_prefix</a></div> |
| <div class="line"><a name="l00284"></a><span class="lineno"> 284</span>  << <span class="stringliteral">"."</span> << output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#abfb3777eb9bb647e071f72b3755dfa37">num_files</a></div> |
| <div class="line"><a name="l00285"></a><span class="lineno"> 285</span>  << <span class="stringliteral">"."</span> << output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->file_extension();</div> |
| <div class="line"><a name="l00286"></a><span class="lineno"> 286</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a> = filename.str();</div> |
| <div class="line"><a name="l00287"></a><span class="lineno"> 287</span>  <span class="comment">// Check if tmp_hdfs_file_name exists.</span></div> |
| <div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  <span class="keyword">const</span> <span class="keywordtype">char</span>* tmp_hdfs_file_name_cstr =</div> |
| <div class="line"><a name="l00289"></a><span class="lineno"> 289</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>.c_str();</div> |
| <div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  <span class="keywordflow">if</span> (hdfsExists(<a class="code" href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">hdfs_connection_</a>, tmp_hdfs_file_name_cstr) == 0) {</div> |
| <div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<a class="code" href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">GetHdfsErrorMsg</a>(<span class="stringliteral">"Temporary HDFS file already exists: "</span>,</div> |
| <div class="line"><a name="l00292"></a><span class="lineno"> 292</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>));</div> |
| <div class="line"><a name="l00293"></a><span class="lineno"> 293</span>  }</div> |
| <div class="line"><a name="l00294"></a><span class="lineno"> 294</span>  <a class="code" href="classuint64__t.html">uint64_t</a> block_size = output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a3346404566a7395694377bab8cd22678">partition_descriptor</a>-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#a084e0f1f5e87833cc943567f31d9813f">block_size</a>();</div> |
| <div class="line"><a name="l00295"></a><span class="lineno"> 295</span>  <span class="keywordflow">if</span> (block_size == 0) block_size = output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->default_block_size();</div> |
| <div class="line"><a name="l00296"></a><span class="lineno"> 296</span> </div> |
| <div class="line"><a name="l00297"></a><span class="lineno"> 297</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">tmp_hdfs_file</a> = hdfsOpenFile(<a class="code" href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">hdfs_connection_</a>,</div> |
| <div class="line"><a name="l00298"></a><span class="lineno"> 298</span>  tmp_hdfs_file_name_cstr, O_WRONLY, 0, 0, block_size);</div> |
| <div class="line"><a name="l00299"></a><span class="lineno"> 299</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"hdfsOpenFile() file="</span> << tmp_hdfs_file_name_cstr;</div> |
| <div class="line"><a name="l00300"></a><span class="lineno"> 300</span>  <span class="keywordflow">if</span> (output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">tmp_hdfs_file</a> == NULL) {</div> |
| <div class="line"><a name="l00301"></a><span class="lineno"> 301</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<a class="code" href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">GetHdfsErrorMsg</a>(<span class="stringliteral">"Failed to open HDFS file for writing: "</span>,</div> |
| <div class="line"><a name="l00302"></a><span class="lineno"> 302</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>));</div> |
| <div class="line"><a name="l00303"></a><span class="lineno"> 303</span>  }</div> |
| <div class="line"><a name="l00304"></a><span class="lineno"> 304</span> </div> |
| <div class="line"><a name="l00305"></a><span class="lineno"> 305</span>  <a class="code" href="classimpala_1_1ImpaladMetrics.html#ac7ac80ffa02fb127c775f88400bb6f8e">ImpaladMetrics::NUM_FILES_OPEN_FOR_INSERT</a>->Increment(1);</div> |
| <div class="line"><a name="l00306"></a><span class="lineno"> 306</span>  <a class="code" href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#ab39cf9643eeb89dc752711b5e9437035">files_created_counter_</a>, 1);</div> |
| <div class="line"><a name="l00307"></a><span class="lineno"> 307</span> </div> |
| <div class="line"><a name="l00308"></a><span class="lineno"> 308</span>  <span class="comment">// Save the ultimate destination for this file (it will be moved by the coordinator)</span></div> |
| <div class="line"><a name="l00309"></a><span class="lineno"> 309</span>  stringstream dest;</div> |
| <div class="line"><a name="l00310"></a><span class="lineno"> 310</span>  dest << output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#aad8b5d001c2558ffc6b6e0b6f09723d8">final_hdfs_file_name_prefix</a></div> |
| <div class="line"><a name="l00311"></a><span class="lineno"> 311</span>  << <span class="stringliteral">"."</span> << output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#abfb3777eb9bb647e071f72b3755dfa37">num_files</a></div> |
| <div class="line"><a name="l00312"></a><span class="lineno"> 312</span>  << <span class="stringliteral">"."</span> << output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->file_extension();</div> |
| <div class="line"><a name="l00313"></a><span class="lineno"> 313</span>  (*state-><a class="code" href="classimpala_1_1RuntimeState.html#a7e3e024159837a80f864c62713d91005">hdfs_files_to_move</a>())[output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>] = dest.str();</div> |
| <div class="line"><a name="l00314"></a><span class="lineno"> 314</span> </div> |
| <div class="line"><a name="l00315"></a><span class="lineno"> 315</span>  ++output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#abfb3777eb9bb647e071f72b3755dfa37">num_files</a>;</div> |
| <div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6c5c9cee1f82ec607f54eda9bc339543">num_rows</a> = 0;</div> |
| <div class="line"><a name="l00317"></a><span class="lineno"> 317</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> status = output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->InitNewFile();</div> |
| <div class="line"><a name="l00318"></a><span class="lineno"> 318</span>  <span class="keywordflow">if</span> (!status.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) {</div> |
| <div class="line"><a name="l00319"></a><span class="lineno"> 319</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ad50de8f7c4cfc89e7dc141b8dabbcc44">ClosePartitionFile</a>(state, output_partition);</div> |
| <div class="line"><a name="l00320"></a><span class="lineno"> 320</span>  hdfsDelete(<a class="code" href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">hdfs_connection_</a>, output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>.c_str(), 0);</div> |
| <div class="line"><a name="l00321"></a><span class="lineno"> 321</span>  }</div> |
| <div class="line"><a name="l00322"></a><span class="lineno"> 322</span>  <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l00323"></a><span class="lineno"> 323</span> }</div> |
| <div class="line"><a name="l00324"></a><span class="lineno"> 324</span> </div> |
| <div class="line"><a name="l00325"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a97f9241ddab33d28853fe12df52c7e3b"> 325</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#a97f9241ddab33d28853fe12df52c7e3b">HdfsTableSink::InitOutputPartition</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state,</div> |
| <div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  <span class="keyword">const</span> <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>& partition_descriptor,</div> |
| <div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output_partition) {</div> |
| <div class="line"><a name="l00328"></a><span class="lineno"> 328</span>  <span class="comment">// Build the unique name for this partition from the partition keys, e.g. "j=1/f=foo/"</span></div> |
| <div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  <span class="comment">// etc.</span></div> |
| <div class="line"><a name="l00330"></a><span class="lineno"> 330</span>  stringstream partition_name_ss;</div> |
| <div class="line"><a name="l00331"></a><span class="lineno"> 331</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> j = 0; j < <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>.size(); ++j) {</div> |
| <div class="line"><a name="l00332"></a><span class="lineno"> 332</span>  partition_name_ss << <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">col_names</a>()[j] << <span class="stringliteral">"="</span>;</div> |
| <div class="line"><a name="l00333"></a><span class="lineno"> 333</span>  <span class="keywordtype">void</span>* value = <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[j]->GetValue(<a class="code" href="classimpala_1_1HdfsTableSink.html#aa301171de526d983f26ad15b6254c4ad">current_row_</a>);</div> |
| <div class="line"><a name="l00334"></a><span class="lineno"> 334</span>  <span class="comment">// NULL partition keys get a special value to be compatible with Hive.</span></div> |
| <div class="line"><a name="l00335"></a><span class="lineno"> 335</span>  <span class="keywordflow">if</span> (value == NULL) {</div> |
| <div class="line"><a name="l00336"></a><span class="lineno"> 336</span>  partition_name_ss << <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#a353f9c28c66a50e66b31e48a77161be3">null_partition_key_value</a>();</div> |
| <div class="line"><a name="l00337"></a><span class="lineno"> 337</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00338"></a><span class="lineno"> 338</span>  <span class="keywordtype">string</span> value_str;</div> |
| <div class="line"><a name="l00339"></a><span class="lineno"> 339</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>[j]->PrintValue(value, &value_str);</div> |
| <div class="line"><a name="l00340"></a><span class="lineno"> 340</span>  <span class="comment">// Directory names containing partition-key values need to be UrlEncoded, in</span></div> |
| <div class="line"><a name="l00341"></a><span class="lineno"> 341</span>  <span class="comment">// particular to avoid problems when '/' is part of the key value (which might</span></div> |
| <div class="line"><a name="l00342"></a><span class="lineno"> 342</span>  <span class="comment">// occur, for example, with date strings). Hive will URL decode the value</span></div> |
| <div class="line"><a name="l00343"></a><span class="lineno"> 343</span>  <span class="comment">// transparently when Impala's frontend asks the metastore for partition key values,</span></div> |
| <div class="line"><a name="l00344"></a><span class="lineno"> 344</span>  <span class="comment">// which makes it particularly important that we use the same encoding as Hive. It's</span></div> |
| <div class="line"><a name="l00345"></a><span class="lineno"> 345</span>  <span class="comment">// also not necessary to encode the values when writing partition metadata. You can</span></div> |
| <div class="line"><a name="l00346"></a><span class="lineno"> 346</span>  <span class="comment">// check this with 'show partitions <tbl>' in Hive, followed by a select from a</span></div> |
| <div class="line"><a name="l00347"></a><span class="lineno"> 347</span>  <span class="comment">// decoded partition key value.</span></div> |
| <div class="line"><a name="l00348"></a><span class="lineno"> 348</span>  <span class="keywordtype">string</span> encoded_str;</div> |
| <div class="line"><a name="l00349"></a><span class="lineno"> 349</span>  <a class="code" href="namespaceimpala.html#a21643b3d264e5669ec62886358cb5341">UrlEncode</a>(value_str, &encoded_str, <span class="keyword">true</span>);</div> |
| <div class="line"><a name="l00350"></a><span class="lineno"> 350</span>  <span class="comment">// If the string is empty, map it to NULL (mimicking Hive's behaviour)</span></div> |
| <div class="line"><a name="l00351"></a><span class="lineno"> 351</span>  partition_name_ss << (encoded_str.empty() ?</div> |
| <div class="line"><a name="l00352"></a><span class="lineno"> 352</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#a353f9c28c66a50e66b31e48a77161be3">null_partition_key_value</a>() : encoded_str);</div> |
| <div class="line"><a name="l00353"></a><span class="lineno"> 353</span>  }</div> |
| <div class="line"><a name="l00354"></a><span class="lineno"> 354</span>  partition_name_ss << <span class="stringliteral">"/"</span>;</div> |
| <div class="line"><a name="l00355"></a><span class="lineno"> 355</span>  }</div> |
| <div class="line"><a name="l00356"></a><span class="lineno"> 356</span> </div> |
| <div class="line"><a name="l00357"></a><span class="lineno"> 357</span>  <span class="comment">// partition_name_ss now holds the unique descriptor for this partition,</span></div> |
| <div class="line"><a name="l00358"></a><span class="lineno"> 358</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">partition_name</a> = partition_name_ss.str();</div> |
| <div class="line"><a name="l00359"></a><span class="lineno"> 359</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ae6da527d105f3b2380948e1b4fbc37ce">BuildHdfsFileNames</a>(partition_descriptor, output_partition);</div> |
| <div class="line"><a name="l00360"></a><span class="lineno"> 360</span> </div> |
| <div class="line"><a name="l00361"></a><span class="lineno"> 361</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a86e51e4750dce9b115b88a61eccef63b">hdfs_connection</a> = <a class="code" href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">hdfs_connection_</a>;</div> |
| <div class="line"><a name="l00362"></a><span class="lineno"> 362</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a3346404566a7395694377bab8cd22678">partition_descriptor</a> = &partition_descriptor;</div> |
| <div class="line"><a name="l00363"></a><span class="lineno"> 363</span> </div> |
| <div class="line"><a name="l00364"></a><span class="lineno"> 364</span>  <span class="keywordtype">bool</span> allow_unsupported_formats =</div> |
| <div class="line"><a name="l00365"></a><span class="lineno"> 365</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().__isset.allow_unsupported_formats &&</div> |
| <div class="line"><a name="l00366"></a><span class="lineno"> 366</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().allow_unsupported_formats;</div> |
| <div class="line"><a name="l00367"></a><span class="lineno"> 367</span>  <span class="keywordflow">if</span> (!allow_unsupported_formats) {</div> |
| <div class="line"><a name="l00368"></a><span class="lineno"> 368</span>  <span class="keywordflow">if</span> (partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>() == THdfsFileFormat::SEQUENCE_FILE ||</div> |
| <div class="line"><a name="l00369"></a><span class="lineno"> 369</span>  partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>() == THdfsFileFormat::AVRO) {</div> |
| <div class="line"><a name="l00370"></a><span class="lineno"> 370</span>  stringstream error_msg;</div> |
| <div class="line"><a name="l00371"></a><span class="lineno"> 371</span>  map<int, const char*>::const_iterator i =</div> |
| <div class="line"><a name="l00372"></a><span class="lineno"> 372</span>  _THdfsFileFormat_VALUES_TO_NAMES.find(partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>());</div> |
| <div class="line"><a name="l00373"></a><span class="lineno"> 373</span>  error_msg << <span class="stringliteral">"Writing to table format "</span> << i->second</div> |
| <div class="line"><a name="l00374"></a><span class="lineno"> 374</span>  << <span class="stringliteral">" is not supported. Use query option ALLOW_UNSUPPORTED_FORMATS"</span></div> |
| <div class="line"><a name="l00375"></a><span class="lineno"> 375</span>  <span class="stringliteral">" to override."</span>;</div> |
| <div class="line"><a name="l00376"></a><span class="lineno"> 376</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(error_msg.str());</div> |
| <div class="line"><a name="l00377"></a><span class="lineno"> 377</span>  }</div> |
| <div class="line"><a name="l00378"></a><span class="lineno"> 378</span>  <span class="keywordflow">if</span> (partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>() == THdfsFileFormat::TEXT &&</div> |
| <div class="line"><a name="l00379"></a><span class="lineno"> 379</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().__isset.compression_codec &&</div> |
| <div class="line"><a name="l00380"></a><span class="lineno"> 380</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">query_options</a>().compression_codec != THdfsCompression::NONE) {</div> |
| <div class="line"><a name="l00381"></a><span class="lineno"> 381</span>  stringstream error_msg;</div> |
| <div class="line"><a name="l00382"></a><span class="lineno"> 382</span>  error_msg << <span class="stringliteral">"Writing to compressed text table is not supported. "</span></div> |
| <div class="line"><a name="l00383"></a><span class="lineno"> 383</span>  <span class="stringliteral">"Use query option ALLOW_UNSUPPORTED_FORMATS to override."</span>;</div> |
| <div class="line"><a name="l00384"></a><span class="lineno"> 384</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(error_msg.str());</div> |
| <div class="line"><a name="l00385"></a><span class="lineno"> 385</span>  }</div> |
| <div class="line"><a name="l00386"></a><span class="lineno"> 386</span>  }</div> |
| <div class="line"><a name="l00387"></a><span class="lineno"> 387</span> </div> |
| <div class="line"><a name="l00388"></a><span class="lineno"> 388</span>  <span class="comment">// It is incorrect to initialize a writer if there are no rows to feed it. The writer</span></div> |
| <div class="line"><a name="l00389"></a><span class="lineno"> 389</span>  <span class="comment">// could incorrectly create an empty file or empty partition.</span></div> |
| <div class="line"><a name="l00390"></a><span class="lineno"> 390</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsTableSink.html#a7cf72bcb73a038dbe3365c229a4fffac">has_empty_input_batch_</a>) <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00391"></a><span class="lineno"> 391</span> </div> |
| <div class="line"><a name="l00392"></a><span class="lineno"> 392</span>  <span class="keywordflow">switch</span> (partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>()) {</div> |
| <div class="line"><a name="l00393"></a><span class="lineno"> 393</span>  <span class="keywordflow">case</span> THdfsFileFormat::TEXT:</div> |
| <div class="line"><a name="l00394"></a><span class="lineno"> 394</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>.reset(</div> |
| <div class="line"><a name="l00395"></a><span class="lineno"> 395</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsTextTableWriter.html">HdfsTextTableWriter</a>(</div> |
| <div class="line"><a name="l00396"></a><span class="lineno"> 396</span>  <span class="keyword">this</span>, state, output_partition, &partition_descriptor, <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>,</div> |
| <div class="line"><a name="l00397"></a><span class="lineno"> 397</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>));</div> |
| <div class="line"><a name="l00398"></a><span class="lineno"> 398</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00399"></a><span class="lineno"> 399</span>  <span class="keywordflow">case</span> THdfsFileFormat::PARQUET:</div> |
| <div class="line"><a name="l00400"></a><span class="lineno"> 400</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>.reset(</div> |
| <div class="line"><a name="l00401"></a><span class="lineno"> 401</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsParquetTableWriter.html">HdfsParquetTableWriter</a>(</div> |
| <div class="line"><a name="l00402"></a><span class="lineno"> 402</span>  <span class="keyword">this</span>, state, output_partition, &partition_descriptor, <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>,</div> |
| <div class="line"><a name="l00403"></a><span class="lineno"> 403</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>));</div> |
| <div class="line"><a name="l00404"></a><span class="lineno"> 404</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00405"></a><span class="lineno"> 405</span>  <span class="keywordflow">case</span> THdfsFileFormat::SEQUENCE_FILE:</div> |
| <div class="line"><a name="l00406"></a><span class="lineno"> 406</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>.reset(</div> |
| <div class="line"><a name="l00407"></a><span class="lineno"> 407</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsSequenceTableWriter.html">HdfsSequenceTableWriter</a>(</div> |
| <div class="line"><a name="l00408"></a><span class="lineno"> 408</span>  <span class="keyword">this</span>, state, output_partition, &partition_descriptor, <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>,</div> |
| <div class="line"><a name="l00409"></a><span class="lineno"> 409</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>));</div> |
| <div class="line"><a name="l00410"></a><span class="lineno"> 410</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00411"></a><span class="lineno"> 411</span>  <span class="keywordflow">case</span> THdfsFileFormat::AVRO:</div> |
| <div class="line"><a name="l00412"></a><span class="lineno"> 412</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>.reset(</div> |
| <div class="line"><a name="l00413"></a><span class="lineno"> 413</span>  <span class="keyword">new</span> <a class="code" href="classimpala_1_1HdfsAvroTableWriter.html">HdfsAvroTableWriter</a>(</div> |
| <div class="line"><a name="l00414"></a><span class="lineno"> 414</span>  <span class="keyword">this</span>, state, output_partition, &partition_descriptor, <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>,</div> |
| <div class="line"><a name="l00415"></a><span class="lineno"> 415</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>));</div> |
| <div class="line"><a name="l00416"></a><span class="lineno"> 416</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00417"></a><span class="lineno"> 417</span>  <span class="keywordflow">default</span>:</div> |
| <div class="line"><a name="l00418"></a><span class="lineno"> 418</span>  stringstream error_msg;</div> |
| <div class="line"><a name="l00419"></a><span class="lineno"> 419</span>  map<int, const char*>::const_iterator i =</div> |
| <div class="line"><a name="l00420"></a><span class="lineno"> 420</span>  _THdfsFileFormat_VALUES_TO_NAMES.find(partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>());</div> |
| <div class="line"><a name="l00421"></a><span class="lineno"> 421</span>  <span class="keywordflow">if</span> (i != _THdfsFileFormat_VALUES_TO_NAMES.end()) {</div> |
| <div class="line"><a name="l00422"></a><span class="lineno"> 422</span>  error_msg << <span class="stringliteral">"Cannot write to table with format "</span> << i->second << <span class="stringliteral">". "</span></div> |
| <div class="line"><a name="l00423"></a><span class="lineno"> 423</span>  << <span class="stringliteral">"Impala only supports writing to TEXT and PARQUET."</span>;</div> |
| <div class="line"><a name="l00424"></a><span class="lineno"> 424</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00425"></a><span class="lineno"> 425</span>  error_msg << <span class="stringliteral">"Cannot write to table. Impala only supports writing to TEXT"</span></div> |
| <div class="line"><a name="l00426"></a><span class="lineno"> 426</span>  << <span class="stringliteral">" and PARQUET tables. (Unknown file format: "</span></div> |
| <div class="line"><a name="l00427"></a><span class="lineno"> 427</span>  << partition_descriptor.<a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">file_format</a>() << <span class="stringliteral">")"</span>;</div> |
| <div class="line"><a name="l00428"></a><span class="lineno"> 428</span>  }</div> |
| <div class="line"><a name="l00429"></a><span class="lineno"> 429</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(error_msg.str());</div> |
| <div class="line"><a name="l00430"></a><span class="lineno"> 430</span>  }</div> |
| <div class="line"><a name="l00431"></a><span class="lineno"> 431</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->Init());</div> |
| <div class="line"><a name="l00432"></a><span class="lineno"> 432</span>  <a class="code" href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a06230db843c80fc6f4ca6fe2fd756a75">partitions_created_counter_</a>, 1);</div> |
| <div class="line"><a name="l00433"></a><span class="lineno"> 433</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1HdfsTableSink.html#ad05565061b0102660efcd17cfaeb56e1">CreateNewTmpFile</a>(state, output_partition);</div> |
| <div class="line"><a name="l00434"></a><span class="lineno"> 434</span> }</div> |
| <div class="line"><a name="l00435"></a><span class="lineno"> 435</span> </div> |
| <div class="line"><a name="l00436"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a5e6d1761306e721e0e8284fe08ec1229"> 436</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsTableSink.html#a5e6d1761306e721e0e8284fe08ec1229">HdfsTableSink::GetHashTblKey</a>(<span class="keyword">const</span> vector<ExprContext*>& ctxs, <span class="keywordtype">string</span>* key) {</div> |
| <div class="line"><a name="l00437"></a><span class="lineno"> 437</span>  stringstream hash_table_key;</div> |
| <div class="line"><a name="l00438"></a><span class="lineno"> 438</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < ctxs.size(); ++i) {</div> |
| <div class="line"><a name="l00439"></a><span class="lineno"> 439</span>  <a class="code" href="classimpala_1_1RawValue.html#a15b702837a5957a127b04676261a7ba9">RawValue::PrintValueAsBytes</a>(</div> |
| <div class="line"><a name="l00440"></a><span class="lineno"> 440</span>  ctxs[i]->GetValue(<a class="code" href="classimpala_1_1HdfsTableSink.html#aa301171de526d983f26ad15b6254c4ad">current_row_</a>), ctxs[i]->root()->type(), &hash_table_key);</div> |
| <div class="line"><a name="l00441"></a><span class="lineno"> 441</span>  <span class="comment">// Additionally append "/" to avoid accidental key collisions.</span></div> |
| <div class="line"><a name="l00442"></a><span class="lineno"> 442</span>  hash_table_key << <span class="stringliteral">"/"</span>;</div> |
| <div class="line"><a name="l00443"></a><span class="lineno"> 443</span>  }</div> |
| <div class="line"><a name="l00444"></a><span class="lineno"> 444</span>  *key = hash_table_key.str();</div> |
| <div class="line"><a name="l00445"></a><span class="lineno"> 445</span> }</div> |
| <div class="line"><a name="l00446"></a><span class="lineno"> 446</span> </div> |
| <div class="line"><a name="l00447"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a33eacd5217233d7dc3e54293c989d52e"> 447</a></span> <span class="keyword">inline</span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#a33eacd5217233d7dc3e54293c989d52e">HdfsTableSink::GetOutputPartition</a>(</div> |
| <div class="line"><a name="l00448"></a><span class="lineno"> 448</span>  <a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state, <span class="keyword">const</span> <span class="keywordtype">string</span>& key, <a class="code" href="classimpala_1_1HdfsTableSink.html#a4fab224a06b3d7b7e38fefe6a0089615">PartitionPair</a>** partition_pair) {</div> |
| <div class="line"><a name="l00449"></a><span class="lineno"> 449</span>  PartitionMap::iterator existing_partition;</div> |
| <div class="line"><a name="l00450"></a><span class="lineno"> 450</span>  existing_partition = <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.find(key);</div> |
| <div class="line"><a name="l00451"></a><span class="lineno"> 451</span>  <span class="keywordflow">if</span> (existing_partition == <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.end()) {</div> |
| <div class="line"><a name="l00452"></a><span class="lineno"> 452</span>  <span class="comment">// Create a new OutputPartition, and add it to</span></div> |
| <div class="line"><a name="l00453"></a><span class="lineno"> 453</span>  <span class="comment">// partition_keys_to_output_partitions.</span></div> |
| <div class="line"><a name="l00454"></a><span class="lineno"> 454</span>  <span class="keyword">const</span> <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>* partition_descriptor = <a class="code" href="classimpala_1_1HdfsTableSink.html#aaf0abb7b8d271e272e23399bb1e110ad">default_partition_</a>;</div> |
| <div class="line"><a name="l00455"></a><span class="lineno"> 455</span>  PartitionDescriptorMap::const_iterator it = <a class="code" href="classimpala_1_1HdfsTableSink.html#acc5427a2a031ccd7e40250a6e3aa59c7">partition_descriptor_map_</a>.find(key);</div> |
| <div class="line"><a name="l00456"></a><span class="lineno"> 456</span>  <span class="keywordflow">if</span> (it != <a class="code" href="classimpala_1_1HdfsTableSink.html#acc5427a2a031ccd7e40250a6e3aa59c7">partition_descriptor_map_</a>.end()) {</div> |
| <div class="line"><a name="l00457"></a><span class="lineno"> 457</span>  partition_descriptor = it->second;</div> |
| <div class="line"><a name="l00458"></a><span class="lineno"> 458</span>  }</div> |
| <div class="line"><a name="l00459"></a><span class="lineno"> 459</span> </div> |
| <div class="line"><a name="l00460"></a><span class="lineno"> 460</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* partition = state-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<span class="keyword">new</span> <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>());</div> |
| <div class="line"><a name="l00461"></a><span class="lineno"> 461</span>  <a class="code" href="classimpala_1_1Status.html">Status</a> status = <a class="code" href="classimpala_1_1HdfsTableSink.html#a97f9241ddab33d28853fe12df52c7e3b">InitOutputPartition</a>(state, *partition_descriptor, partition);</div> |
| <div class="line"><a name="l00462"></a><span class="lineno"> 462</span>  <span class="keywordflow">if</span> (!status.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) {</div> |
| <div class="line"><a name="l00463"></a><span class="lineno"> 463</span>  <span class="comment">// We failed to create the output partition successfully. Clean it up now</span></div> |
| <div class="line"><a name="l00464"></a><span class="lineno"> 464</span>  <span class="comment">// as it is not added to partition_keys_to_output_partitions_ so won't be</span></div> |
| <div class="line"><a name="l00465"></a><span class="lineno"> 465</span>  <span class="comment">// cleaned up in Close().</span></div> |
| <div class="line"><a name="l00466"></a><span class="lineno"> 466</span>  <span class="keywordflow">if</span> (partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>.get() != NULL) partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->Close();</div> |
| <div class="line"><a name="l00467"></a><span class="lineno"> 467</span>  <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l00468"></a><span class="lineno"> 468</span>  }</div> |
| <div class="line"><a name="l00469"></a><span class="lineno"> 469</span> </div> |
| <div class="line"><a name="l00470"></a><span class="lineno"> 470</span>  <span class="comment">// Save the partition name so that the coordinator can create the partition directory</span></div> |
| <div class="line"><a name="l00471"></a><span class="lineno"> 471</span>  <span class="comment">// structure if needed</span></div> |
| <div class="line"><a name="l00472"></a><span class="lineno"> 472</span>  DCHECK(state-><a class="code" href="classimpala_1_1RuntimeState.html#ad319395860c8405b365598838c89bd1b">per_partition_status</a>()->find(partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">partition_name</a>) ==</div> |
| <div class="line"><a name="l00473"></a><span class="lineno"> 473</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#ad319395860c8405b365598838c89bd1b">per_partition_status</a>()->end());</div> |
| <div class="line"><a name="l00474"></a><span class="lineno"> 474</span>  TInsertPartitionStatus partition_status;</div> |
| <div class="line"><a name="l00475"></a><span class="lineno"> 475</span>  partition_status.__set_num_appended_rows(0L);</div> |
| <div class="line"><a name="l00476"></a><span class="lineno"> 476</span>  partition_status.__set_id(partition_descriptor-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#aef01083969b4a1506a4c7d3332f42fea">id</a>());</div> |
| <div class="line"><a name="l00477"></a><span class="lineno"> 477</span>  partition_status.__set_stats(TInsertStats());</div> |
| <div class="line"><a name="l00478"></a><span class="lineno"> 478</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#ad319395860c8405b365598838c89bd1b">per_partition_status</a>()->insert(</div> |
| <div class="line"><a name="l00479"></a><span class="lineno"> 479</span>  make_pair(partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">partition_name</a>, partition_status));</div> |
| <div class="line"><a name="l00480"></a><span class="lineno"> 480</span> </div> |
| <div class="line"><a name="l00481"></a><span class="lineno"> 481</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1HdfsTableSink.html#a7cf72bcb73a038dbe3365c229a4fffac">has_empty_input_batch_</a>) {</div> |
| <div class="line"><a name="l00482"></a><span class="lineno"> 482</span>  <span class="comment">// Indicate that temporary directory is to be deleted after execution</span></div> |
| <div class="line"><a name="l00483"></a><span class="lineno"> 483</span>  (*state-><a class="code" href="classimpala_1_1RuntimeState.html#a7e3e024159837a80f864c62713d91005">hdfs_files_to_move</a>())[partition-><a class="code" href="structimpala_1_1OutputPartition.html#a443f6bb9e09cdfcc5268e66dc87673c6">tmp_hdfs_dir_name</a>] = <span class="stringliteral">""</span>;</div> |
| <div class="line"><a name="l00484"></a><span class="lineno"> 484</span>  }</div> |
| <div class="line"><a name="l00485"></a><span class="lineno"> 485</span> </div> |
| <div class="line"><a name="l00486"></a><span class="lineno"> 486</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>[key].first = partition;</div> |
| <div class="line"><a name="l00487"></a><span class="lineno"> 487</span>  *partition_pair = &<a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>[key];</div> |
| <div class="line"><a name="l00488"></a><span class="lineno"> 488</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00489"></a><span class="lineno"> 489</span>  <span class="comment">// Use existing output_partition partition.</span></div> |
| <div class="line"><a name="l00490"></a><span class="lineno"> 490</span>  *partition_pair = &existing_partition->second;</div> |
| <div class="line"><a name="l00491"></a><span class="lineno"> 491</span>  }</div> |
| <div class="line"><a name="l00492"></a><span class="lineno"> 492</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00493"></a><span class="lineno"> 493</span> }</div> |
| <div class="line"><a name="l00494"></a><span class="lineno"> 494</span> </div> |
| <div class="line"><a name="l00495"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a280d7bde9f6b5180794281d066c24ba6"> 495</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#a280d7bde9f6b5180794281d066c24ba6">HdfsTableSink::Send</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state, <a class="code" href="classimpala_1_1RowBatch.html">RowBatch</a>* batch, <span class="keywordtype">bool</span> eos) {</div> |
| <div class="line"><a name="l00496"></a><span class="lineno"> 496</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a32d00c3c1b4eb90fcae15c9ffbb5a3bb">runtime_profile_</a>-><a class="code" href="classimpala_1_1RuntimeProfile.html#abd28edd77ba3928dbac3af735102fb3b">total_time_counter</a>());</div> |
| <div class="line"><a name="l00497"></a><span class="lineno"> 497</span>  <a class="code" href="classimpala_1_1ExprContext.html#a92f9a380825bf39ff33b5b3168bc377a">ExprContext::FreeLocalAllocations</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>);</div> |
| <div class="line"><a name="l00498"></a><span class="lineno"> 498</span>  <a class="code" href="classimpala_1_1ExprContext.html#a92f9a380825bf39ff33b5b3168bc377a">ExprContext::FreeLocalAllocations</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>);</div> |
| <div class="line"><a name="l00499"></a><span class="lineno"> 499</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(state-><a class="code" href="classimpala_1_1RuntimeState.html#a27bed76f727f93f1d75fbb2762cb0fa6">CheckQueryState</a>());</div> |
| <div class="line"><a name="l00500"></a><span class="lineno"> 500</span>  DCHECK(eos || batch-><a class="code" href="classimpala_1_1RowBatch.html#ac695df3b85ee416b3d99844813ae813d">num_rows</a>() > 0);</div> |
| <div class="line"><a name="l00501"></a><span class="lineno"> 501</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a7cf72bcb73a038dbe3365c229a4fffac">has_empty_input_batch_</a> = batch-><a class="code" href="classimpala_1_1RowBatch.html#ac695df3b85ee416b3d99844813ae813d">num_rows</a>() == 0 && eos;</div> |
| <div class="line"><a name="l00502"></a><span class="lineno"> 502</span> </div> |
| <div class="line"><a name="l00503"></a><span class="lineno"> 503</span>  <span class="comment">// If there are no partition keys then just pass the whole batch to one partition.</span></div> |
| <div class="line"><a name="l00504"></a><span class="lineno"> 504</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1HdfsTableSink.html#a0e7f566d373b853ce6b7d7b382f05df8">dynamic_partition_key_expr_ctxs_</a>.empty()) {</div> |
| <div class="line"><a name="l00505"></a><span class="lineno"> 505</span>  <span class="comment">// If there are no dynamic keys just use an empty key.</span></div> |
| <div class="line"><a name="l00506"></a><span class="lineno"> 506</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a4fab224a06b3d7b7e38fefe6a0089615">PartitionPair</a>* partition_pair;</div> |
| <div class="line"><a name="l00507"></a><span class="lineno"> 507</span>  <span class="comment">// Populate the partition_pair even if the input is empty because we need it to</span></div> |
| <div class="line"><a name="l00508"></a><span class="lineno"> 508</span>  <span class="comment">// delete the existing data for 'insert overwrite'.</span></div> |
| <div class="line"><a name="l00509"></a><span class="lineno"> 509</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a33eacd5217233d7dc3e54293c989d52e">GetOutputPartition</a>(state, <a class="code" href="namespaceimpala.html#af0847904c2bf82facb74614fc46a1edc">ROOT_PARTITION_KEY</a>, &partition_pair));</div> |
| <div class="line"><a name="l00510"></a><span class="lineno"> 510</span>  <span class="keywordflow">if</span> (!has_empty_input_batch_) {</div> |
| <div class="line"><a name="l00511"></a><span class="lineno"> 511</span>  <span class="comment">// Pass the row batch to the writer. If new_file is returned true then the current</span></div> |
| <div class="line"><a name="l00512"></a><span class="lineno"> 512</span>  <span class="comment">// file is finalized and a new file is opened.</span></div> |
| <div class="line"><a name="l00513"></a><span class="lineno"> 513</span>  <span class="comment">// The writer tracks where it is in the batch when it returns with new_file set.</span></div> |
| <div class="line"><a name="l00514"></a><span class="lineno"> 514</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output_partition = partition_pair->first;</div> |
| <div class="line"><a name="l00515"></a><span class="lineno"> 515</span>  <span class="keywordtype">bool</span> new_file;</div> |
| <div class="line"><a name="l00516"></a><span class="lineno"> 516</span>  <span class="keywordflow">do</span> {</div> |
| <div class="line"><a name="l00517"></a><span class="lineno"> 517</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->AppendRowBatch(</div> |
| <div class="line"><a name="l00518"></a><span class="lineno"> 518</span>  batch, partition_pair->second, &new_file));</div> |
| <div class="line"><a name="l00519"></a><span class="lineno"> 519</span>  <span class="keywordflow">if</span> (new_file) {</div> |
| <div class="line"><a name="l00520"></a><span class="lineno"> 520</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a97034a6afc292d4a26a238d402c32012">FinalizePartitionFile</a>(state, output_partition));</div> |
| <div class="line"><a name="l00521"></a><span class="lineno"> 521</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#ad05565061b0102660efcd17cfaeb56e1">CreateNewTmpFile</a>(state, output_partition));</div> |
| <div class="line"><a name="l00522"></a><span class="lineno"> 522</span>  }</div> |
| <div class="line"><a name="l00523"></a><span class="lineno"> 523</span>  } <span class="keywordflow">while</span> (new_file);</div> |
| <div class="line"><a name="l00524"></a><span class="lineno"> 524</span>  }</div> |
| <div class="line"><a name="l00525"></a><span class="lineno"> 525</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00526"></a><span class="lineno"> 526</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < batch-><a class="code" href="classimpala_1_1RowBatch.html#ac695df3b85ee416b3d99844813ae813d">num_rows</a>(); ++i) {</div> |
| <div class="line"><a name="l00527"></a><span class="lineno"> 527</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#aa301171de526d983f26ad15b6254c4ad">current_row_</a> = batch-><a class="code" href="classimpala_1_1RowBatch.html#a9320ed986b0717ef26d73d871f3b3a42">GetRow</a>(i);</div> |
| <div class="line"><a name="l00528"></a><span class="lineno"> 528</span> </div> |
| <div class="line"><a name="l00529"></a><span class="lineno"> 529</span>  <span class="keywordtype">string</span> key;</div> |
| <div class="line"><a name="l00530"></a><span class="lineno"> 530</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a5e6d1761306e721e0e8284fe08ec1229">GetHashTblKey</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a0e7f566d373b853ce6b7d7b382f05df8">dynamic_partition_key_expr_ctxs_</a>, &key);</div> |
| <div class="line"><a name="l00531"></a><span class="lineno"> 531</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a4fab224a06b3d7b7e38fefe6a0089615">PartitionPair</a>* partition_pair = NULL;</div> |
| <div class="line"><a name="l00532"></a><span class="lineno"> 532</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a33eacd5217233d7dc3e54293c989d52e">GetOutputPartition</a>(state, key, &partition_pair));</div> |
| <div class="line"><a name="l00533"></a><span class="lineno"> 533</span>  partition_pair->second.push_back(i);</div> |
| <div class="line"><a name="l00534"></a><span class="lineno"> 534</span>  }</div> |
| <div class="line"><a name="l00535"></a><span class="lineno"> 535</span>  <span class="keywordflow">for</span> (PartitionMap::iterator partition = <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.begin();</div> |
| <div class="line"><a name="l00536"></a><span class="lineno"> 536</span>  partition != <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.end(); ++partition) {</div> |
| <div class="line"><a name="l00537"></a><span class="lineno"> 537</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output_partition = partition->second.first;</div> |
| <div class="line"><a name="l00538"></a><span class="lineno"> 538</span>  <span class="keywordflow">if</span> (partition->second.second.empty()) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00539"></a><span class="lineno"> 539</span> </div> |
| <div class="line"><a name="l00540"></a><span class="lineno"> 540</span>  <span class="keywordtype">bool</span> new_file;</div> |
| <div class="line"><a name="l00541"></a><span class="lineno"> 541</span>  <span class="keywordflow">do</span> {</div> |
| <div class="line"><a name="l00542"></a><span class="lineno"> 542</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->AppendRowBatch(</div> |
| <div class="line"><a name="l00543"></a><span class="lineno"> 543</span>  batch, partition->second.second, &new_file));</div> |
| <div class="line"><a name="l00544"></a><span class="lineno"> 544</span>  <span class="keywordflow">if</span> (new_file) {</div> |
| <div class="line"><a name="l00545"></a><span class="lineno"> 545</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a97034a6afc292d4a26a238d402c32012">FinalizePartitionFile</a>(state, output_partition));</div> |
| <div class="line"><a name="l00546"></a><span class="lineno"> 546</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#ad05565061b0102660efcd17cfaeb56e1">CreateNewTmpFile</a>(state, output_partition));</div> |
| <div class="line"><a name="l00547"></a><span class="lineno"> 547</span>  }</div> |
| <div class="line"><a name="l00548"></a><span class="lineno"> 548</span>  } <span class="keywordflow">while</span> (new_file);</div> |
| <div class="line"><a name="l00549"></a><span class="lineno"> 549</span>  partition->second.second.clear();</div> |
| <div class="line"><a name="l00550"></a><span class="lineno"> 550</span>  }</div> |
| <div class="line"><a name="l00551"></a><span class="lineno"> 551</span>  }</div> |
| <div class="line"><a name="l00552"></a><span class="lineno"> 552</span> </div> |
| <div class="line"><a name="l00553"></a><span class="lineno"> 553</span>  <span class="keywordflow">if</span> (eos) {</div> |
| <div class="line"><a name="l00554"></a><span class="lineno"> 554</span>  <span class="comment">// Close Hdfs files, and update stats in runtime state.</span></div> |
| <div class="line"><a name="l00555"></a><span class="lineno"> 555</span>  <span class="keywordflow">for</span> (PartitionMap::iterator cur_partition =</div> |
| <div class="line"><a name="l00556"></a><span class="lineno"> 556</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.begin();</div> |
| <div class="line"><a name="l00557"></a><span class="lineno"> 557</span>  cur_partition != <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.end();</div> |
| <div class="line"><a name="l00558"></a><span class="lineno"> 558</span>  ++cur_partition) {</div> |
| <div class="line"><a name="l00559"></a><span class="lineno"> 559</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a97034a6afc292d4a26a238d402c32012">FinalizePartitionFile</a>(state, cur_partition->second.first));</div> |
| <div class="line"><a name="l00560"></a><span class="lineno"> 560</span>  }</div> |
| <div class="line"><a name="l00561"></a><span class="lineno"> 561</span>  }</div> |
| <div class="line"><a name="l00562"></a><span class="lineno"> 562</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00563"></a><span class="lineno"> 563</span> }</div> |
| <div class="line"><a name="l00564"></a><span class="lineno"> 564</span> </div> |
| <div class="line"><a name="l00565"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a97034a6afc292d4a26a238d402c32012"> 565</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#a97034a6afc292d4a26a238d402c32012">HdfsTableSink::FinalizePartitionFile</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state,</div> |
| <div class="line"><a name="l00566"></a><span class="lineno"> 566</span>  <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* partition) {</div> |
| <div class="line"><a name="l00567"></a><span class="lineno"> 567</span>  <span class="keywordflow">if</span> (partition-><a class="code" href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">tmp_hdfs_file</a> == NULL && !<a class="code" href="classimpala_1_1HdfsTableSink.html#af355351518defbe5eac0ef7e4641d654">overwrite_</a>) <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00568"></a><span class="lineno"> 568</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">profile</a>(), <span class="stringliteral">"FinalizePartitionFileTimer"</span>));</div> |
| <div class="line"><a name="l00569"></a><span class="lineno"> 569</span> </div> |
| <div class="line"><a name="l00570"></a><span class="lineno"> 570</span>  <span class="comment">// OutputPartition writer could be NULL if there is no row to output.</span></div> |
| <div class="line"><a name="l00571"></a><span class="lineno"> 571</span>  <span class="keywordflow">if</span> (partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>.get() != NULL) {</div> |
| <div class="line"><a name="l00572"></a><span class="lineno"> 572</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->Finalize());</div> |
| <div class="line"><a name="l00573"></a><span class="lineno"> 573</span> </div> |
| <div class="line"><a name="l00574"></a><span class="lineno"> 574</span>  <span class="comment">// Track total number of appended rows per partition in runtime</span></div> |
| <div class="line"><a name="l00575"></a><span class="lineno"> 575</span>  <span class="comment">// state. partition->num_rows counts number of rows appended is per-file.</span></div> |
| <div class="line"><a name="l00576"></a><span class="lineno"> 576</span>  PartitionStatusMap::iterator it =</div> |
| <div class="line"><a name="l00577"></a><span class="lineno"> 577</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#ad319395860c8405b365598838c89bd1b">per_partition_status</a>()->find(partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">partition_name</a>);</div> |
| <div class="line"><a name="l00578"></a><span class="lineno"> 578</span> </div> |
| <div class="line"><a name="l00579"></a><span class="lineno"> 579</span>  <span class="comment">// Should have been created in GetOutputPartition() when the partition was initialised.</span></div> |
| <div class="line"><a name="l00580"></a><span class="lineno"> 580</span>  DCHECK(it != state-><a class="code" href="classimpala_1_1RuntimeState.html#ad319395860c8405b365598838c89bd1b">per_partition_status</a>()->end());</div> |
| <div class="line"><a name="l00581"></a><span class="lineno"> 581</span>  it->second.num_appended_rows += partition-><a class="code" href="structimpala_1_1OutputPartition.html#a6c5c9cee1f82ec607f54eda9bc339543">num_rows</a>;</div> |
| <div class="line"><a name="l00582"></a><span class="lineno"> 582</span>  <a class="code" href="classimpala_1_1DataSink.html#ab95c1680a2c851bed71977e502139b32">DataSink::MergeInsertStats</a>(partition-><a class="code" href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">writer</a>->stats(), &it->second.stats);</div> |
| <div class="line"><a name="l00583"></a><span class="lineno"> 583</span>  }</div> |
| <div class="line"><a name="l00584"></a><span class="lineno"> 584</span> </div> |
| <div class="line"><a name="l00585"></a><span class="lineno"> 585</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ad50de8f7c4cfc89e7dc141b8dabbcc44">ClosePartitionFile</a>(state, partition);</div> |
| <div class="line"><a name="l00586"></a><span class="lineno"> 586</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00587"></a><span class="lineno"> 587</span> }</div> |
| <div class="line"><a name="l00588"></a><span class="lineno"> 588</span> </div> |
| <div class="line"><a name="l00589"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#ad50de8f7c4cfc89e7dc141b8dabbcc44"> 589</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsTableSink.html#ad50de8f7c4cfc89e7dc141b8dabbcc44">HdfsTableSink::ClosePartitionFile</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state, <a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* partition) {</div> |
| <div class="line"><a name="l00590"></a><span class="lineno"> 590</span>  <span class="keywordflow">if</span> (partition-><a class="code" href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">tmp_hdfs_file</a> == NULL) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00591"></a><span class="lineno"> 591</span>  <span class="keywordtype">int</span> hdfs_ret = hdfsCloseFile(<a class="code" href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">hdfs_connection_</a>, partition-><a class="code" href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">tmp_hdfs_file</a>);</div> |
| <div class="line"><a name="l00592"></a><span class="lineno"> 592</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"hdfsCloseFile() file="</span> << partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>;</div> |
| <div class="line"><a name="l00593"></a><span class="lineno"> 593</span>  <span class="keywordflow">if</span> (hdfs_ret != 0) {</div> |
| <div class="line"><a name="l00594"></a><span class="lineno"> 594</span>  state-><a class="code" href="classimpala_1_1RuntimeState.html#aaf076b1879d9e712b08ba3a6e24607b9">LogError</a>(<a class="code" href="classimpala_1_1ErrorMsg.html">ErrorMsg</a>(TErrorCode::GENERAL,</div> |
| <div class="line"><a name="l00595"></a><span class="lineno"> 595</span>  <a class="code" href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">GetHdfsErrorMsg</a>(<span class="stringliteral">"Failed to close HDFS file: "</span>,</div> |
| <div class="line"><a name="l00596"></a><span class="lineno"> 596</span>  partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>)));</div> |
| <div class="line"><a name="l00597"></a><span class="lineno"> 597</span>  }</div> |
| <div class="line"><a name="l00598"></a><span class="lineno"> 598</span>  partition-><a class="code" href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">tmp_hdfs_file</a> = NULL;</div> |
| <div class="line"><a name="l00599"></a><span class="lineno"> 599</span>  <a class="code" href="classimpala_1_1ImpaladMetrics.html#ac7ac80ffa02fb127c775f88400bb6f8e">ImpaladMetrics::NUM_FILES_OPEN_FOR_INSERT</a>->Increment(-1);</div> |
| <div class="line"><a name="l00600"></a><span class="lineno"> 600</span> }</div> |
| <div class="line"><a name="l00601"></a><span class="lineno"> 601</span> </div> |
| <div class="line"><a name="l00602"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a2ca4efce0b0f420ca113819620f023c7"> 602</a></span> <span class="keywordtype">void</span> <a class="code" href="classimpala_1_1HdfsTableSink.html#a2ca4efce0b0f420ca113819620f023c7">HdfsTableSink::Close</a>(<a class="code" href="classimpala_1_1RuntimeState.html">RuntimeState</a>* state) {</div> |
| <div class="line"><a name="l00603"></a><span class="lineno"> 603</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1DataSink.html#ae28cb708ca3252c487c446c5d6d683d4">closed_</a>) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00604"></a><span class="lineno"> 604</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a32d00c3c1b4eb90fcae15c9ffbb5a3bb">runtime_profile_</a>-><a class="code" href="classimpala_1_1RuntimeProfile.html#abd28edd77ba3928dbac3af735102fb3b">total_time_counter</a>());</div> |
| <div class="line"><a name="l00605"></a><span class="lineno"> 605</span>  <span class="keywordflow">for</span> (PartitionMap::iterator cur_partition =</div> |
| <div class="line"><a name="l00606"></a><span class="lineno"> 606</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.begin();</div> |
| <div class="line"><a name="l00607"></a><span class="lineno"> 607</span>  cur_partition != <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.end();</div> |
| <div class="line"><a name="l00608"></a><span class="lineno"> 608</span>  ++cur_partition) {</div> |
| <div class="line"><a name="l00609"></a><span class="lineno"> 609</span>  <span class="keywordflow">if</span> (cur_partition->second.first->writer.get() != NULL) {</div> |
| <div class="line"><a name="l00610"></a><span class="lineno"> 610</span>  cur_partition->second.first->writer->Close();</div> |
| <div class="line"><a name="l00611"></a><span class="lineno"> 611</span>  }</div> |
| <div class="line"><a name="l00612"></a><span class="lineno"> 612</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#ad50de8f7c4cfc89e7dc141b8dabbcc44">ClosePartitionFile</a>(state, cur_partition->second.first);</div> |
| <div class="line"><a name="l00613"></a><span class="lineno"> 613</span>  }</div> |
| <div class="line"><a name="l00614"></a><span class="lineno"> 614</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">partition_keys_to_output_partitions_</a>.clear();</div> |
| <div class="line"><a name="l00615"></a><span class="lineno"> 615</span> </div> |
| <div class="line"><a name="l00616"></a><span class="lineno"> 616</span>  <span class="comment">// Close literal partition key exprs</span></div> |
| <div class="line"><a name="l00617"></a><span class="lineno"> 617</span>  BOOST_FOREACH(</div> |
| <div class="line"><a name="l00618"></a><span class="lineno"> 618</span>  <span class="keyword">const</span> HdfsTableDescriptor::PartitionIdToDescriptorMap::value_type& id_to_desc,</div> |
| <div class="line"><a name="l00619"></a><span class="lineno"> 619</span>  <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#a9c3f56805e3c2577ff77baf3abf49670">partition_descriptors</a>()) {</div> |
| <div class="line"><a name="l00620"></a><span class="lineno"> 620</span>  <a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html">HdfsPartitionDescriptor</a>* partition = id_to_desc.second;</div> |
| <div class="line"><a name="l00621"></a><span class="lineno"> 621</span>  partition-><a class="code" href="classimpala_1_1HdfsPartitionDescriptor.html#ac07d03794cffd542bda8c2f49a0658e5">CloseExprs</a>(state);</div> |
| <div class="line"><a name="l00622"></a><span class="lineno"> 622</span>  }</div> |
| <div class="line"><a name="l00623"></a><span class="lineno"> 623</span>  <a class="code" href="classimpala_1_1Expr.html#a70300b0ba7c91fddb9adfaeb0dfc09a5">Expr::Close</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>, state);</div> |
| <div class="line"><a name="l00624"></a><span class="lineno"> 624</span>  <a class="code" href="classimpala_1_1Expr.html#a70300b0ba7c91fddb9adfaeb0dfc09a5">Expr::Close</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>, state);</div> |
| <div class="line"><a name="l00625"></a><span class="lineno"> 625</span>  <a class="code" href="classimpala_1_1DataSink.html#ae28cb708ca3252c487c446c5d6d683d4">closed_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00626"></a><span class="lineno"> 626</span> }</div> |
| <div class="line"><a name="l00627"></a><span class="lineno"> 627</span> </div> |
| <div class="line"><a name="l00628"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#abf1f8a799d6f3fe3c1d8539537791a50"> 628</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1HdfsTableSink.html#abf1f8a799d6f3fe3c1d8539537791a50">HdfsTableSink::GetFileBlockSize</a>(<a class="code" href="structimpala_1_1OutputPartition.html">OutputPartition</a>* output_partition, int64_t* size) {</div> |
| <div class="line"><a name="l00629"></a><span class="lineno"> 629</span>  hdfsFileInfo* info = hdfsGetPathInfo(output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a86e51e4750dce9b115b88a61eccef63b">hdfs_connection</a>,</div> |
| <div class="line"><a name="l00630"></a><span class="lineno"> 630</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>.c_str());</div> |
| <div class="line"><a name="l00631"></a><span class="lineno"> 631</span> </div> |
| <div class="line"><a name="l00632"></a><span class="lineno"> 632</span>  <span class="keywordflow">if</span> (info == NULL) {</div> |
| <div class="line"><a name="l00633"></a><span class="lineno"> 633</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<a class="code" href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">GetHdfsErrorMsg</a>(<span class="stringliteral">"Failed to get info on temporary HDFS file: "</span>,</div> |
| <div class="line"><a name="l00634"></a><span class="lineno"> 634</span>  output_partition-><a class="code" href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">current_file_name</a>));</div> |
| <div class="line"><a name="l00635"></a><span class="lineno"> 635</span>  }</div> |
| <div class="line"><a name="l00636"></a><span class="lineno"> 636</span> </div> |
| <div class="line"><a name="l00637"></a><span class="lineno"> 637</span>  *size = info->mBlockSize;</div> |
| <div class="line"><a name="l00638"></a><span class="lineno"> 638</span>  hdfsFreeFileInfo(info, 1);</div> |
| <div class="line"><a name="l00639"></a><span class="lineno"> 639</span> </div> |
| <div class="line"><a name="l00640"></a><span class="lineno"> 640</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00641"></a><span class="lineno"> 641</span> }</div> |
| <div class="line"><a name="l00642"></a><span class="lineno"> 642</span> </div> |
| <div class="line"><a name="l00643"></a><span class="lineno"><a class="line" href="classimpala_1_1HdfsTableSink.html#a938c70d0fcf84b3c606761be535918ac"> 643</a></span> <span class="keywordtype">string</span> <a class="code" href="classimpala_1_1HdfsTableSink.html#a938c70d0fcf84b3c606761be535918ac">HdfsTableSink::DebugString</a>()<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l00644"></a><span class="lineno"> 644</span>  stringstream out;</div> |
| <div class="line"><a name="l00645"></a><span class="lineno"> 645</span>  out << <span class="stringliteral">"HdfsTableSink(overwrite="</span> << (<a class="code" href="classimpala_1_1HdfsTableSink.html#af355351518defbe5eac0ef7e4641d654">overwrite_</a> ? <span class="stringliteral">"true"</span> : <span class="stringliteral">"false"</span>)</div> |
| <div class="line"><a name="l00646"></a><span class="lineno"> 646</span>  << <span class="stringliteral">" table_desc="</span> << <a class="code" href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">table_desc_</a>-><a class="code" href="classimpala_1_1HdfsTableDescriptor.html#af5dca3aa2b21c8ce35bd5150d4723380">DebugString</a>()</div> |
| <div class="line"><a name="l00647"></a><span class="lineno"> 647</span>  << <span class="stringliteral">" partition_key_exprs="</span> << <a class="code" href="classimpala_1_1Expr.html#ab8ae88546f4ce923b4d187e0d7d96fa8">Expr::DebugString</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">partition_key_expr_ctxs_</a>)</div> |
| <div class="line"><a name="l00648"></a><span class="lineno"> 648</span>  << <span class="stringliteral">" output_exprs="</span> << <a class="code" href="classimpala_1_1Expr.html#ab8ae88546f4ce923b4d187e0d7d96fa8">Expr::DebugString</a>(<a class="code" href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">output_expr_ctxs_</a>)</div> |
| <div class="line"><a name="l00649"></a><span class="lineno"> 649</span>  << <span class="stringliteral">")"</span>;</div> |
| <div class="line"><a name="l00650"></a><span class="lineno"> 650</span>  <span class="keywordflow">return</span> out.str();</div> |
| <div class="line"><a name="l00651"></a><span class="lineno"> 651</span> }</div> |
| <div class="line"><a name="l00652"></a><span class="lineno"> 652</span> </div> |
| <div class="line"><a name="l00653"></a><span class="lineno"> 653</span> }</div> |
| <div class="ttc" id="row-batch_8h_html"><div class="ttname"><a href="row-batch_8h.html">row-batch.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableDescriptor_html_af5dca3aa2b21c8ce35bd5150d4723380"><div class="ttname"><a href="classimpala_1_1HdfsTableDescriptor.html#af5dca3aa2b21c8ce35bd5150d4723380">impala::HdfsTableDescriptor::DebugString</a></div><div class="ttdeci">virtual std::string DebugString() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8cc_source.html#l00190">descriptors.cc:190</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_abf1f8a799d6f3fe3c1d8539537791a50"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#abf1f8a799d6f3fe3c1d8539537791a50">impala::HdfsTableSink::GetFileBlockSize</a></div><div class="ttdeci">static Status GetFileBlockSize(OutputPartition *output_partition, int64_t *size)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00628">hdfs-table-sink.cc:628</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a27bed76f727f93f1d75fbb2762cb0fa6"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a27bed76f727f93f1d75fbb2762cb0fa6">impala::RuntimeState::CheckQueryState</a></div><div class="ttdeci">Status CheckQueryState()</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8cc_source.html#l00286">runtime-state.cc:286</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a4ca2934de11e4bde6e068d576e295f43"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a4ca2934de11e4bde6e068d576e295f43">impala::HdfsTableSink::table_id_</a></div><div class="ttdeci">TableId table_id_</div><div class="ttdoc">Table id resolved in Prepare() to set tuple_desc_;. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00226">hdfs-table-sink.h:226</a></div></div> |
| <div class="ttc" id="classimpala_1_1DataSink_html_a0a447d23b1d3b6df7b2720439f3e486d"><div class="ttname"><a href="classimpala_1_1DataSink.html#a0a447d23b1d3b6df7b2720439f3e486d">impala::DataSink::expr_mem_tracker_</a></div><div class="ttdeci">boost::scoped_ptr< MemTracker > expr_mem_tracker_</div><div class="ttdef"><b>Definition:</b> <a href="data-sink_8h_source.html#l00085">data-sink.h:85</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowBatch_html_ac695df3b85ee416b3d99844813ae813d"><div class="ttname"><a href="classimpala_1_1RowBatch.html#ac695df3b85ee416b3d99844813ae813d">impala::RowBatch::num_rows</a></div><div class="ttdeci">int num_rows() const </div><div class="ttdef"><b>Definition:</b> <a href="row-batch_8h_source.html#l00215">row-batch.h:215</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_ab4a79c2d0c772e6ecafdae14569bfde4"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#ab4a79c2d0c772e6ecafdae14569bfde4">impala::HdfsPartitionDescriptor::file_format</a></div><div class="ttdeci">THdfsFileFormat::type file_format() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00184">descriptors.h:184</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a267db75ab7c5c0a26296071c31b5369d"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a267db75ab7c5c0a26296071c31b5369d">impala::HdfsTableSink::output_expr_ctxs_</a></div><div class="ttdeci">std::vector< ExprContext * > output_expr_ctxs_</div><div class="ttdoc">Exprs that materialize output values. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00214">hdfs-table-sink.h:214</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_a3f6d43477d551e065f039c7b074bf4b1"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#a3f6d43477d551e065f039c7b074bf4b1">impala::HdfsPartitionDescriptor::OpenExprs</a></div><div class="ttdeci">Status OpenExprs(RuntimeState *state)</div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8cc_source.html#l00145">descriptors.cc:145</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a6648f284f4c7a5c44f694f363b2a0d88"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a6648f284f4c7a5c44f694f363b2a0d88">impala::RuntimeState::query_id</a></div><div class="ttdeci">const TUniqueId & query_id() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00125">runtime-state.h:125</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_aad8b5d001c2558ffc6b6e0b6f09723d8"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#aad8b5d001c2558ffc6b6e0b6f09723d8">impala::OutputPartition::final_hdfs_file_name_prefix</a></div><div class="ttdeci">std::string final_hdfs_file_name_prefix</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00049">hdfs-table-sink.h:49</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a7710498d89015e8b80b60ef21eeb93f7"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a7710498d89015e8b80b60ef21eeb93f7">impala::HdfsTableSink::profile</a></div><div class="ttdeci">virtual RuntimeProfile * profile()</div><div class="ttdoc">Returns the runtime profile for the sink. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00147">hdfs-table-sink.h:147</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ae6da527d105f3b2380948e1b4fbc37ce"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ae6da527d105f3b2380948e1b4fbc37ce">impala::HdfsTableSink::BuildHdfsFileNames</a></div><div class="ttdeci">void BuildHdfsFileNames(const HdfsPartitionDescriptor &partition_descriptor, OutputPartition *output)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00240">hdfs-table-sink.cc:240</a></div></div> |
| <div class="ttc" id="classimpala_1_1RawValue_html_a15b702837a5957a127b04676261a7ba9"><div class="ttname"><a href="classimpala_1_1RawValue.html#a15b702837a5957a127b04676261a7ba9">impala::RawValue::PrintValueAsBytes</a></div><div class="ttdeci">static void PrintValueAsBytes(const void *value, const ColumnType &type, std::stringstream *stream)</div><div class="ttdoc">Writes the byte representation of a value to a stringstream character-by-character. </div><div class="ttdef"><b>Definition:</b> <a href="raw-value_8cc_source.html#l00028">raw-value.cc:28</a></div></div> |
| <div class="ttc" id="classimpala_1_1RawValue_html_af526a6e3729755632d191f2bf08628e7"><div class="ttname"><a href="classimpala_1_1RawValue.html#af526a6e3729755632d191f2bf08628e7">impala::RawValue::Eq</a></div><div class="ttdeci">static bool Eq(const void *v1, const void *v2, const ColumnType &type)</div><div class="ttdef"><b>Definition:</b> <a href="raw-value_8h_source.html#l00106">raw-value.h:106</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_abe51d04af9ad683ef8bc891a89da0300"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#abe51d04af9ad683ef8bc891a89da0300">impala::HdfsPartitionDescriptor::PrepareExprs</a></div><div class="ttdeci">Status PrepareExprs(RuntimeState *state)</div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8cc_source.html#l00134">descriptors.cc:134</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsParquetTableWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsParquetTableWriter.html">impala::HdfsParquetTableWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-parquet-table-writer_8h_source.html#l00049">hdfs-parquet-table-writer.h:49</a></div></div> |
| <div class="ttc" id="impalad-metrics_8h_html"><div class="ttname"><a href="impalad-metrics_8h.html">impalad-metrics.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTextTableWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsTextTableWriter.html">impala::HdfsTextTableWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-text-table-writer_8h_source.html#l00040">hdfs-text-table-writer.h:40</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_aad1111f87951868de4d8f8b2dfe87c5f"><div class="ttname"><a href="classimpala_1_1Expr.html#aad1111f87951868de4d8f8b2dfe87c5f">impala::Expr::Open</a></div><div class="ttdeci">static Status Open(const std::vector< ExprContext * > &ctxs, RuntimeState *state)</div><div class="ttdoc">Convenience function for opening multiple expr trees. </div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a7e3e024159837a80f864c62713d91005"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a7e3e024159837a80f864c62713d91005">impala::RuntimeState::hdfs_files_to_move</a></div><div class="ttdeci">FileMoveMap * hdfs_files_to_move()</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00144">runtime-state.h:144</a></div></div> |
| <div class="ttc" id="mem-tracker_8h_html"><div class="ttname"><a href="mem-tracker_8h.html">mem-tracker.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableDescriptor_html"><div class="ttname"><a href="classimpala_1_1HdfsTableDescriptor.html">impala::HdfsTableDescriptor</a></div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00226">descriptors.h:226</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ad99889b28415d19eae2164112e6f507c"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ad99889b28415d19eae2164112e6f507c">impala::HdfsTableSink::PrepareExprs</a></div><div class="ttdeci">Status PrepareExprs(RuntimeState *state)</div><div class="ttdoc">Initialise and prepare select and partition key expressions. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00072">hdfs-table-sink.cc:72</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a443f6bb9e09cdfcc5268e66dc87673c6"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a443f6bb9e09cdfcc5268e66dc87673c6">impala::OutputPartition::tmp_hdfs_dir_name</a></div><div class="ttdeci">std::string tmp_hdfs_dir_name</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00060">hdfs-table-sink.h:60</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a9d0676f3ddac66add883e7f636be908b"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a9d0676f3ddac66add883e7f636be908b">impala::OutputPartition::current_file_name</a></div><div class="ttdeci">std::string current_file_name</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00055">hdfs-table-sink.h:55</a></div></div> |
| <div class="ttc" id="classimpala_1_1DescriptorTbl_html_a2e67f43279256b454184fea3daa9c8da"><div class="ttname"><a href="classimpala_1_1DescriptorTbl.html#a2e67f43279256b454184fea3daa9c8da">impala::DescriptorTbl::GetTableDescriptor</a></div><div class="ttdeci">TableDescriptor * GetTableDescriptor(TableId id) const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8cc_source.html#l00427">descriptors.cc:427</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_acc5427a2a031ccd7e40250a6e3aa59c7"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#acc5427a2a031ccd7e40250a6e3aa59c7">impala::HdfsTableSink::partition_descriptor_map_</a></div><div class="ttdeci">PartitionDescriptorMap partition_descriptor_map_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00267">hdfs-table-sink.h:267</a></div></div> |
| <div class="ttc" id="status_8h_html_a85f7d0e774e15eb35b74f53264305e16"><div class="ttname"><a href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a></div><div class="ttdeci">#define RETURN_IF_ERROR(stmt)</div><div class="ttdoc">some generally useful macros </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00242">status.h:242</a></div></div> |
| <div class="ttc" id="hdfs-util_8h_html"><div class="ttname"><a href="hdfs-util_8h.html">hdfs-util.h</a></div></div> |
| <div class="ttc" id="raw-value_8h_html"><div class="ttname"><a href="raw-value_8h.html">raw-value.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a2ca4efce0b0f420ca113819620f023c7"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a2ca4efce0b0f420ca113819620f023c7">impala::HdfsTableSink::Close</a></div><div class="ttdeci">virtual void Close(RuntimeState *state)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00602">hdfs-table-sink.cc:602</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowBatch_html_a9320ed986b0717ef26d73d871f3b3a42"><div class="ttname"><a href="classimpala_1_1RowBatch.html#a9320ed986b0717ef26d73d871f3b3a42">impala::RowBatch::GetRow</a></div><div class="ttdeci">TupleRow * GetRow(int row_idx)</div><div class="ttdef"><b>Definition:</b> <a href="row-batch_8h_source.html#l00140">row-batch.h:140</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a06230db843c80fc6f4ca6fe2fd756a75"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a06230db843c80fc6f4ca6fe2fd756a75">impala::HdfsTableSink::partitions_created_counter_</a></div><div class="ttdeci">RuntimeProfile::Counter * partitions_created_counter_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00273">hdfs-table-sink.h:273</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a33eacd5217233d7dc3e54293c989d52e"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a33eacd5217233d7dc3e54293c989d52e">impala::HdfsTableSink::GetOutputPartition</a></div><div class="ttdeci">Status GetOutputPartition(RuntimeState *state, const std::string &key, PartitionPair **partition_pair)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00447">hdfs-table-sink.cc:447</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_a8841ca205b2a05d608e82d443bad3a77"><div class="ttname"><a href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a></div><div class="ttdeci">#define ADD_TIMER(profile, name)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00050">runtime-profile.h:50</a></div></div> |
| <div class="ttc" id="expr-context_8h_html"><div class="ttname"><a href="expr-context_8h.html">expr-context.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableDescriptor_html_abaa1ebf5914c72c1454f43b1836ff6dd"><div class="ttname"><a href="classimpala_1_1HdfsTableDescriptor.html#abaa1ebf5914c72c1454f43b1836ff6dd">impala::HdfsTableDescriptor::hdfs_base_dir</a></div><div class="ttdeci">const std::string & hdfs_base_dir() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00229">descriptors.h:229</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_ad319395860c8405b365598838c89bd1b"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#ad319395860c8405b365598838c89bd1b">impala::RuntimeState::per_partition_status</a></div><div class="ttdeci">PartitionStatusMap * per_partition_status()</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00177">runtime-state.h:177</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowDescriptor_html"><div class="ttname"><a href="classimpala_1_1RowDescriptor.html">impala::RowDescriptor</a></div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00373">descriptors.h:373</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_aef01083969b4a1506a4c7d3332f42fea"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#aef01083969b4a1506a4c7d3332f42fea">impala::HdfsPartitionDescriptor::id</a></div><div class="ttdeci">int64_t id() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00190">descriptors.h:190</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_abfb3777eb9bb647e071f72b3755dfa37"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#abfb3777eb9bb647e071f72b3755dfa37">impala::OutputPartition::num_files</a></div><div class="ttdeci">int32_t num_files</div><div class="ttdoc">Number of files created in this partition. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00080">hdfs-table-sink.h:80</a></div></div> |
| <div class="ttc" id="classimpala_1_1TableDescriptor_html_a665ceaf9c184ec39544c3ecc448c0d11"><div class="ttname"><a href="classimpala_1_1TableDescriptor.html#a665ceaf9c184ec39544c3ecc448c0d11">impala::TableDescriptor::num_cols</a></div><div class="ttdeci">int num_cols() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00152">descriptors.h:152</a></div></div> |
| <div class="ttc" id="hdfs-sequence-table-writer_8h_html"><div class="ttname"><a href="hdfs-sequence-table-writer_8h.html">hdfs-sequence-table-writer.h</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html"><div class="ttname"><a href="structimpala_1_1OutputPartition.html">impala::OutputPartition</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00040">hdfs-table-sink.h:40</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_add5f44b15e0231c50eba4437a85bc627"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#add5f44b15e0231c50eba4437a85bc627">impala::HdfsTableSink::staging_dir_</a></div><div class="ttdeci">std::string staging_dir_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00244">hdfs-table-sink.h:244</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a19e01fc90bc956ac7e1a735d2bea5a5e"><div class="ttname"><a href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">impala::PrintId</a></div><div class="ttdeci">string PrintId(const TUniqueId &id, const string &separator)</div><div class="ttdef"><b>Definition:</b> <a href="debug-util_8cc_source.html#l00097">debug-util.cc:97</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_a0842f330c18b3e4e53e13655add0310a"><div class="ttname"><a href="runtime-profile_8h.html#a0842f330c18b3e4e53e13655add0310a">COUNTER_ADD</a></div><div class="ttdeci">#define COUNTER_ADD(c, v)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00055">runtime-profile.h:55</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a6f348763ea77e2c5c03e476976081fd1"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a6f348763ea77e2c5c03e476976081fd1">impala::OutputPartition::partition_name</a></div><div class="ttdeci">std::string partition_name</div><div class="ttdoc">key1=val1/key2=val2/ etc. Used to identify partitions to the metastore. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00068">hdfs-table-sink.h:68</a></div></div> |
| <div class="ttc" id="hdfs-avro-table-writer_8h_html"><div class="ttname"><a href="hdfs-avro-table-writer_8h.html">hdfs-avro-table-writer.h</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_aaa9a2971c6368e3ddd3f5140a0295eb7"><div class="ttname"><a href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a></div><div class="ttdeci">#define SCOPED_TIMER(c)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00053">runtime-profile.h:53</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a938c70d0fcf84b3c606761be535918ac"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a938c70d0fcf84b3c606761be535918ac">impala::HdfsTableSink::DebugString</a></div><div class="ttdeci">std::string DebugString() const </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00643">hdfs-table-sink.cc:643</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a3346404566a7395694377bab8cd22678"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a3346404566a7395694377bab8cd22678">impala::OutputPartition::partition_descriptor</a></div><div class="ttdeci">const HdfsPartitionDescriptor * partition_descriptor</div><div class="ttdoc">The descriptor for this partition. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00086">hdfs-table-sink.h:86</a></div></div> |
| <div class="ttc" id="hdfs-fs-cache_8h_html"><div class="ttname"><a href="hdfs-fs-cache_8h.html">hdfs-fs-cache.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_a70300b0ba7c91fddb9adfaeb0dfc09a5"><div class="ttname"><a href="classimpala_1_1Expr.html#a70300b0ba7c91fddb9adfaeb0dfc09a5">impala::Expr::Close</a></div><div class="ttdeci">static void Close(const std::vector< ExprContext * > &ctxs, RuntimeState *state)</div><div class="ttdoc">Convenience function for closing multiple expr trees. </div></div> |
| <div class="ttc" id="namespaceimpala_html_a21643b3d264e5669ec62886358cb5341"><div class="ttname"><a href="namespaceimpala.html#a21643b3d264e5669ec62886358cb5341">impala::UrlEncode</a></div><div class="ttdeci">static void UrlEncode(const char *in, int in_len, string *out, bool hive_compat)</div><div class="ttdef"><b>Definition:</b> <a href="url-coding_8cc_source.html#l00048">url-coding.cc:48</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsFsCache_html_af7b18daf8c4da6bfce2c7612a515bf64"><div class="ttname"><a href="classimpala_1_1HdfsFsCache.html#af7b18daf8c4da6bfce2c7612a515bf64">impala::HdfsFsCache::instance</a></div><div class="ttdeci">static HdfsFsCache * instance()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-fs-cache_8h_source.html#l00043">hdfs-fs-cache.h:43</a></div></div> |
| <div class="ttc" id="classimpala_1_1ImpaladMetrics_html_ac7ac80ffa02fb127c775f88400bb6f8e"><div class="ttname"><a href="classimpala_1_1ImpaladMetrics.html#ac7ac80ffa02fb127c775f88400bb6f8e">impala::ImpaladMetrics::NUM_FILES_OPEN_FOR_INSERT</a></div><div class="ttdeci">static IntGauge * NUM_FILES_OPEN_FOR_INSERT</div><div class="ttdef"><b>Definition:</b> <a href="impalad-metrics_8h_source.html#l00149">impalad-metrics.h:149</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html"><div class="ttname"><a href="classimpala_1_1Status.html">impala::Status</a></div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00081">status.h:81</a></div></div> |
| <div class="ttc" id="logging_8h_html_a916e38366cee7c32fd5fae7e2ee5e4a9"><div class="ttname"><a href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a></div><div class="ttdeci">#define VLOG_QUERY</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00057">logging.h:57</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a54bdc0c70b33283ca488c5d5c8619361"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a54bdc0c70b33283ca488c5d5c8619361">impala::HdfsTableSink::rows_inserted_counter_</a></div><div class="ttdeci">RuntimeProfile::Counter * rows_inserted_counter_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00275">hdfs-table-sink.h:275</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_aaf0abb7b8d271e272e23399bb1e110ad"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#aaf0abb7b8d271e272e23399bb1e110ad">impala::HdfsTableSink::default_partition_</a></div><div class="ttdeci">const HdfsPartitionDescriptor * default_partition_</div><div class="ttdoc">Currently this is the default partition since we don't support multi-format sinks. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00211">hdfs-table-sink.h:211</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a86e51e4750dce9b115b88a61eccef63b"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a86e51e4750dce9b115b88a61eccef63b">impala::OutputPartition::hdfs_connection</a></div><div class="ttdeci">hdfsFS hdfs_connection</div><div class="ttdoc">Connection to hdfs. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00071">hdfs-table-sink.h:71</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a4ea610c1ebbb6e0aa9667e3f0ef983aa"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a4ea610c1ebbb6e0aa9667e3f0ef983aa">impala::HdfsTableSink::hdfs_connection_</a></div><div class="ttdeci">hdfsFS hdfs_connection_</div><div class="ttdoc">Connection to hdfs, established in Open() and closed in Close(). </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00220">hdfs-table-sink.h:220</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_afded447223c40f6ad8cb5b501875d568"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#afded447223c40f6ad8cb5b501875d568">impala::HdfsTableSink::table_desc_</a></div><div class="ttdeci">const HdfsTableDescriptor * table_desc_</div><div class="ttdoc">Descriptor of target table. Set in Prepare(). </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00208">hdfs-table-sink.h:208</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_aaf076b1879d9e712b08ba3a6e24607b9"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#aaf076b1879d9e712b08ba3a6e24607b9">impala::RuntimeState::LogError</a></div><div class="ttdeci">bool LogError(const ErrorMsg &msg)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8cc_source.html#l00224">runtime-state.cc:224</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_adc231e17933fbc4385f968d7eecc5ee2"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#adc231e17933fbc4385f968d7eecc5ee2">impala::RuntimeState::query_options</a></div><div class="ttdeci">const TQueryOptions & query_options() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00095">runtime-state.h:95</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_ac07d03794cffd542bda8c2f49a0658e5"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#ac07d03794cffd542bda8c2f49a0658e5">impala::HdfsPartitionDescriptor::CloseExprs</a></div><div class="ttdeci">void CloseExprs(RuntimeState *state)</div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8cc_source.html#l00151">descriptors.cc:151</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html"><div class="ttname"><a href="classimpala_1_1RuntimeState.html">impala::RuntimeState</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00069">runtime-state.h:69</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ad50de8f7c4cfc89e7dc141b8dabbcc44"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ad50de8f7c4cfc89e7dc141b8dabbcc44">impala::HdfsTableSink::ClosePartitionFile</a></div><div class="ttdeci">void ClosePartitionFile(RuntimeState *state, OutputPartition *partition)</div><div class="ttdoc">Closes the hdfs file for this partition as well as the writer. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00589">hdfs-table-sink.cc:589</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_ae1e66f0a122f72d42cda498511e63c0c"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#ae1e66f0a122f72d42cda498511e63c0c">impala::HdfsPartitionDescriptor::location</a></div><div class="ttdeci">const std::string & location() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00189">descriptors.h:189</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ad05565061b0102660efcd17cfaeb56e1"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ad05565061b0102660efcd17cfaeb56e1">impala::HdfsTableSink::CreateNewTmpFile</a></div><div class="ttdeci">Status CreateNewTmpFile(RuntimeState *state, OutputPartition *output_partition)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00279">hdfs-table-sink.cc:279</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a75e3842c1ebc3a199e3b16618808d459"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a75e3842c1ebc3a199e3b16618808d459">impala::HdfsTableSink::partition_key_expr_ctxs_</a></div><div class="ttdeci">std::vector< ExprContext * > partition_key_expr_ctxs_</div><div class="ttdoc">Exprs of partition keys. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00237">hdfs-table-sink.h:237</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a4c01f227dc12d859d5824e7c91d6d1aa"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">impala::RuntimeState::obj_pool</a></div><div class="ttdeci">ObjectPool * obj_pool() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00092">runtime-state.h:92</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a2170cd3291d4d58c5cc93431d5d3e1d4"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a2170cd3291d4d58c5cc93431d5d3e1d4">impala::HdfsTableSink::encode_timer_</a></div><div class="ttdeci">RuntimeProfile::Counter * encode_timer_</div><div class="ttdoc">Time spent converting tuple to on disk format. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00279">hdfs-table-sink.h:279</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsAvroTableWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsAvroTableWriter.html">impala::HdfsAvroTableWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-avro-table-writer_8h_source.html#l00056">hdfs-avro-table-writer.h:56</a></div></div> |
| <div class="ttc" id="classimpala_1_1TableDescriptor_html_aa5b2e225d0d68fa5ef0313db79d0ffb9"><div class="ttname"><a href="classimpala_1_1TableDescriptor.html#aa5b2e225d0d68fa5ef0313db79d0ffb9">impala::TableDescriptor::num_clustering_cols</a></div><div class="ttdeci">int num_clustering_cols() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00153">descriptors.h:153</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableDescriptor_html_a9c3f56805e3c2577ff77baf3abf49670"><div class="ttname"><a href="classimpala_1_1HdfsTableDescriptor.html#a9c3f56805e3c2577ff77baf3abf49670">impala::HdfsTableDescriptor::partition_descriptors</a></div><div class="ttdeci">const PartitionIdToDescriptorMap & partition_descriptors() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00245">descriptors.h:245</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a3f2e0429bebe4586bc9140eafd8a21f1"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a3f2e0429bebe4586bc9140eafd8a21f1">impala::HdfsTableSink::Prepare</a></div><div class="ttdeci">virtual Status Prepare(RuntimeState *state)</div><div class="ttdoc">Prepares output_exprs and partition_key_exprs, and connects to HDFS. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00107">hdfs-table-sink.cc:107</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a87c2d62b1ea6cc0a4a6c0fb43918c860"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a87c2d62b1ea6cc0a4a6c0fb43918c860">impala::OutputPartition::tmp_hdfs_file</a></div><div class="ttdeci">hdfsFile tmp_hdfs_file</div><div class="ttdoc">Hdfs file at tmp_hdfs_file_name. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00074">hdfs-table-sink.h:74</a></div></div> |
| <div class="ttc" id="string-value_8inline_8h_html"><div class="ttname"><a href="string-value_8inline_8h.html">string-value.inline.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_af0847904c2bf82facb74614fc46a1edc"><div class="ttname"><a href="namespaceimpala.html#af0847904c2bf82facb74614fc46a1edc">impala::ROOT_PARTITION_KEY</a></div><div class="ttdeci">static const string & ROOT_PARTITION_KEY</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00052">hdfs-table-sink.cc:52</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_af355351518defbe5eac0ef7e4641d654"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#af355351518defbe5eac0ef7e4641d654">impala::HdfsTableSink::overwrite_</a></div><div class="ttdeci">bool overwrite_</div><div class="ttdoc">Indicates whether the existing partitions should be overwritten. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00240">hdfs-table-sink.h:240</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_aac035c52016117af399543521f069c51"><div class="ttname"><a href="runtime-profile_8h.html#aac035c52016117af399543521f069c51">ADD_COUNTER</a></div><div class="ttdeci">#define ADD_COUNTER(profile, name, unit)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00047">runtime-profile.h:47</a></div></div> |
| <div class="ttc" id="classuint64__t_html"><div class="ttname"><a href="classuint64__t.html">uint64_t</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ab39cf9643eeb89dc752711b5e9437035"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ab39cf9643eeb89dc752711b5e9437035">impala::HdfsTableSink::files_created_counter_</a></div><div class="ttdeci">RuntimeProfile::Counter * files_created_counter_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00274">hdfs-table-sink.h:274</a></div></div> |
| <div class="ttc" id="hdfs-table-sink_8h_html"><div class="ttname"><a href="hdfs-table-sink_8h.html">hdfs-table-sink.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a32d00c3c1b4eb90fcae15c9ffbb5a3bb"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a32d00c3c1b4eb90fcae15c9ffbb5a3bb">impala::HdfsTableSink::runtime_profile_</a></div><div class="ttdeci">RuntimeProfile * runtime_profile_</div><div class="ttdoc">Allocated from runtime state's pool. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00272">hdfs-table-sink.h:272</a></div></div> |
| <div class="ttc" id="exec-node_8h_html"><div class="ttname"><a href="exec-node_8h.html">exec-node.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_ac985744be040bdbe5b1d4d4967c9c0e0"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#ac985744be040bdbe5b1d4d4967c9c0e0">impala::RuntimeState::fragment_instance_id</a></div><div class="ttdeci">const TUniqueId & fragment_instance_id() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00126">runtime-state.h:126</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a4fab224a06b3d7b7e38fefe6a0089615"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a4fab224a06b3d7b7e38fefe6a0089615">impala::HdfsTableSink::PartitionPair</a></div><div class="ttdeci">std::pair< OutputPartition *, std::vector< int32_t > > PartitionPair</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00176">hdfs-table-sink.h:176</a></div></div> |
| <div class="ttc" id="classimpala_1_1DataSink_html_ab95c1680a2c851bed71977e502139b32"><div class="ttname"><a href="classimpala_1_1DataSink.html#ab95c1680a2c851bed71977e502139b32">impala::DataSink::MergeInsertStats</a></div><div class="ttdeci">static void MergeInsertStats(const TInsertStats &src_stats, TInsertStats *dst_stats)</div><div class="ttdef"><b>Definition:</b> <a href="data-sink_8cc_source.html#l00090">data-sink.cc:90</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_abab3a5213f629bebdc266a858dc1d45d"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#abab3a5213f629bebdc266a858dc1d45d">impala::RuntimeState::desc_tbl</a></div><div class="ttdeci">const DescriptorTbl & desc_tbl() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00093">runtime-state.h:93</a></div></div> |
| <div class="ttc" id="classimpala_1_1MemTracker_html"><div class="ttname"><a href="classimpala_1_1MemTracker.html">impala::MemTracker</a></div><div class="ttdoc">This class is thread-safe. </div><div class="ttdef"><b>Definition:</b> <a href="mem-tracker_8h_source.html#l00061">mem-tracker.h:61</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a5e6d1761306e721e0e8284fe08ec1229"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a5e6d1761306e721e0e8284fe08ec1229">impala::HdfsTableSink::GetHashTblKey</a></div><div class="ttdeci">void GetHashTblKey(const std::vector< ExprContext * > &ctxs, std::string *key)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00436">hdfs-table-sink.cc:436</a></div></div> |
| <div class="ttc" id="runtime-state_8h_html"><div class="ttname"><a href="runtime-state_8h.html">runtime-state.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1TableDescriptor_html_adbb3a74120d7bddf6d56eee5364aea2d"><div class="ttname"><a href="classimpala_1_1TableDescriptor.html#adbb3a74120d7bddf6d56eee5364aea2d">impala::TableDescriptor::col_names</a></div><div class="ttdeci">const std::vector< std::string > & col_names() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00165">descriptors.h:165</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a6c5c9cee1f82ec607f54eda9bc339543"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a6c5c9cee1f82ec607f54eda9bc339543">impala::OutputPartition::num_rows</a></div><div class="ttdeci">int64_t num_rows</div><div class="ttdoc">Records number of rows appended to the current file in this partition. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00077">hdfs-table-sink.h:77</a></div></div> |
| <div class="ttc" id="classimpala_1_1RowBatch_html"><div class="ttname"><a href="classimpala_1_1RowBatch.html">impala::RowBatch</a></div><div class="ttdef"><b>Definition:</b> <a href="row-batch_8h_source.html#l00066">row-batch.h:66</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_a8a722c1d7f411feee573a73e32757dc6"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#a8a722c1d7f411feee573a73e32757dc6">impala::OutputPartition::writer</a></div><div class="ttdeci">boost::scoped_ptr< HdfsTableWriter > writer</div><div class="ttdoc">Table format specific writer functions. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00083">hdfs-table-sink.h:83</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_aa301171de526d983f26ad15b6254c4ad"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#aa301171de526d983f26ad15b6254c4ad">impala::HdfsTableSink::current_row_</a></div><div class="ttdeci">TupleRow * current_row_</div><div class="ttdoc">Current row from the current RowBatch to output. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00217">hdfs-table-sink.h:217</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa98447566dd6700a2faaaaf3059f4d95"><div class="ttname"><a href="namespaceimpala.html#aa98447566dd6700a2faaaaf3059f4d95">impala::row_desc</a></div><div class="ttdeci">const RowDescriptor & row_desc() const </div></div> |
| <div class="ttc" id="url-coding_8h_html"><div class="ttname"><a href="url-coding_8h.html">url-coding.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a171a32851f95cfd9f4f4283b7859fec9"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a171a32851f95cfd9f4f4283b7859fec9">impala::HdfsTableSink::select_list_texprs_</a></div><div class="ttdeci">const std::vector< TExpr > & select_list_texprs_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00230">hdfs-table-sink.h:230</a></div></div> |
| <div class="ttc" id="hdfs_8h_html"><div class="ttname"><a href="hdfs_8h.html">hdfs.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_ad6c837d1a8e2d4a995aa00a4df7c5af5"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#ad6c837d1a8e2d4a995aa00a4df7c5af5">impala::RuntimeState::instance_mem_tracker</a></div><div class="ttdeci">MemTracker * instance_mem_tracker()</div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00140">runtime-state.h:140</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ae21c4cf3b20e10ae6b0a6c8cdac92298"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ae21c4cf3b20e10ae6b0a6c8cdac92298">impala::HdfsTableSink::compress_timer_</a></div><div class="ttdeci">RuntimeProfile::Counter * compress_timer_</div><div class="ttdoc">Time spent compressing data. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00283">hdfs-table-sink.h:283</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_html"><div class="ttname"><a href="classimpala_1_1RuntimeProfile.html">impala::RuntimeProfile</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00083">runtime-profile.h:83</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a7cf72bcb73a038dbe3365c229a4fffac"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a7cf72bcb73a038dbe3365c229a4fffac">impala::HdfsTableSink::has_empty_input_batch_</a></div><div class="ttdeci">bool has_empty_input_batch_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00287">hdfs-table-sink.h:287</a></div></div> |
| <div class="ttc" id="classimpala_1_1ErrorMsg_html"><div class="ttname"><a href="classimpala_1_1ErrorMsg.html">impala::ErrorMsg</a></div><div class="ttdef"><b>Definition:</b> <a href="error-util_8h_source.html#l00047">error-util.h:47</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExprContext_html_a92f9a380825bf39ff33b5b3168bc377a"><div class="ttname"><a href="classimpala_1_1ExprContext.html#a92f9a380825bf39ff33b5b3168bc377a">impala::ExprContext::FreeLocalAllocations</a></div><div class="ttdeci">void FreeLocalAllocations()</div><div class="ttdef"><b>Definition:</b> <a href="expr-context_8cc_source.html#l00109">expr-context.cc:109</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a99f2add9dbe23c8fceaabcccfb2ee0fc"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a99f2add9dbe23c8fceaabcccfb2ee0fc">impala::HdfsTableSink::hdfs_write_timer_</a></div><div class="ttdeci">RuntimeProfile::Counter * hdfs_write_timer_</div><div class="ttdoc">Time spent writing to hdfs. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00281">hdfs-table-sink.h:281</a></div></div> |
| <div class="ttc" id="hdfs-text-table-writer_8h_html"><div class="ttname"><a href="hdfs-text-table-writer_8h.html">hdfs-text-table-writer.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a280d7bde9f6b5180794281d066c24ba6"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a280d7bde9f6b5180794281d066c24ba6">impala::HdfsTableSink::Send</a></div><div class="ttdeci">virtual Status Send(RuntimeState *state, RowBatch *batch, bool eos)</div><div class="ttdoc">Append all rows in batch to the temporary Hdfs files corresponding to partitions. ...</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00495">hdfs-table-sink.cc:495</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a580565665ea944eb64f3f495b1bee1e0"><div class="ttname"><a href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">impala::Status::OK</a></div><div class="ttdeci">static const Status OK</div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00087">status.h:87</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a97034a6afc292d4a26a238d402c32012"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a97034a6afc292d4a26a238d402c32012">impala::HdfsTableSink::FinalizePartitionFile</a></div><div class="ttdeci">Status FinalizePartitionFile(RuntimeState *state, OutputPartition *partition)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00565">hdfs-table-sink.cc:565</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ad393a2093952c5b6f3a61bd3e1302e61"><div class="ttname"><a href="namespaceimpala.html#ad393a2093952c5b6f3a61bd3e1302e61">impala::row_desc_</a></div><div class="ttdeci">const RowDescriptor * row_desc_</div><div class="ttdoc">owned by plan root, which resides in runtime_state_'s pool </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00255">coordinator.h:255</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsSequenceTableWriter_html"><div class="ttname"><a href="classimpala_1_1HdfsSequenceTableWriter.html">impala::HdfsSequenceTableWriter</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-sequence-table-writer_8h_source.html#l00038">hdfs-sequence-table-writer.h:38</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a0e7f566d373b853ce6b7d7b382f05df8"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a0e7f566d373b853ce6b7d7b382f05df8">impala::HdfsTableSink::dynamic_partition_key_expr_ctxs_</a></div><div class="ttdeci">std::vector< ExprContext * > dynamic_partition_key_expr_ctxs_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00259">hdfs-table-sink.h:259</a></div></div> |
| <div class="ttc" id="expr_8h_html"><div class="ttname"><a href="expr_8h.html">expr.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html">impala::HdfsPartitionDescriptor</a></div><div class="ttdoc">Metadata for a single partition inside an Hdfs table. </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00177">descriptors.h:177</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a117bc8c377e5da4b5424a8a1615d7b82"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a117bc8c377e5da4b5424a8a1615d7b82">impala::HdfsTableSink::unique_id_str_</a></div><div class="ttdeci">std::string unique_id_str_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00248">hdfs-table-sink.h:248</a></div></div> |
| <div class="ttc" id="names_8h_html"><div class="ttname"><a href="names_8h.html">names.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ObjectPool_html_ab191078c99825682a0a9915ca1e0c05c"><div class="ttname"><a href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">impala::ObjectPool::Add</a></div><div class="ttdeci">T * Add(T *t)</div><div class="ttdef"><b>Definition:</b> <a href="object-pool_8h_source.html#l00042">object-pool.h:42</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_a6b9c01a432ae49b57482c9fc42a73681"><div class="ttname"><a href="classimpala_1_1Expr.html#a6b9c01a432ae49b57482c9fc42a73681">impala::Expr::CreateExprTrees</a></div><div class="ttdeci">static Status CreateExprTrees(ObjectPool *pool, const std::vector< TExpr > &texprs, std::vector< ExprContext * > *ctxs)</div><div class="ttdef"><b>Definition:</b> <a href="expr_8cc_source.html#l00149">expr.cc:149</a></div></div> |
| <div class="ttc" id="classimpala_1_1DataSink_html_ae28cb708ca3252c487c446c5d6d683d4"><div class="ttname"><a href="classimpala_1_1DataSink.html#ae28cb708ca3252c487c446c5d6d683d4">impala::DataSink::closed_</a></div><div class="ttdeci">bool closed_</div><div class="ttdef"><b>Definition:</b> <a href="data-sink_8h_source.html#l00083">data-sink.h:83</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_ab4065ec85d296b6aa9a4f1106ce4ef7a"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#ab4065ec85d296b6aa9a4f1106ce4ef7a">impala::HdfsTableSink::mem_tracker_</a></div><div class="ttdeci">boost::scoped_ptr< MemTracker > mem_tracker_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00269">hdfs-table-sink.h:269</a></div></div> |
| <div class="ttc" id="classimpala_1_1DataSink_html_ad1d8007e6e079be3695d5db545ecc446"><div class="ttname"><a href="classimpala_1_1DataSink.html#ad1d8007e6e079be3695d5db545ecc446">impala::DataSink::Prepare</a></div><div class="ttdeci">virtual Status Prepare(RuntimeState *state)</div><div class="ttdef"><b>Definition:</b> <a href="data-sink_8cc_source.html#l00136">data-sink.cc:136</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_a3ae02e50debba50ac5c7b6dd9f8016e7"><div class="ttname"><a href="classimpala_1_1Expr.html#a3ae02e50debba50ac5c7b6dd9f8016e7">impala::Expr::Prepare</a></div><div class="ttdeci">static Status Prepare(const std::vector< ExprContext * > &ctxs, RuntimeState *state, const RowDescriptor &row_desc, MemTracker *tracker)</div></div> |
| <div class="ttc" id="logging_8h_html_a0bdc3628c740982e847cd24114c07994"><div class="ttname"><a href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a></div><div class="ttdeci">#define VLOG_FILE</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00058">logging.h:58</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a95ba859e42fe93445b340533220836ac"><div class="ttname"><a href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">impala::Status::ok</a></div><div class="ttdeci">bool ok() const </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00172">status.h:172</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a97f9241ddab33d28853fe12df52c7e3b"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a97f9241ddab33d28853fe12df52c7e3b">impala::HdfsTableSink::InitOutputPartition</a></div><div class="ttdeci">Status InitOutputPartition(RuntimeState *state, const HdfsPartitionDescriptor &partition_descriptor, OutputPartition *output_partition)</div><div class="ttdoc">Initialises the filenames of a given output partition, and opens the temporary file. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00325">hdfs-table-sink.cc:325</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aed2594aa39638a14b42554a4e3229dcf"><div class="ttname"><a href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">impala::GetHdfsErrorMsg</a></div><div class="ttdeci">string GetHdfsErrorMsg(const string &prefix, const string &file)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-util_8cc_source.html#l00026">hdfs-util.cc:26</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_a084e0f1f5e87833cc943567f31d9813f"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#a084e0f1f5e87833cc943567f31d9813f">impala::HdfsPartitionDescriptor::block_size</a></div><div class="ttdeci">int block_size() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00188">descriptors.h:188</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aaebb9aadfa9e5aec064bc26565df06d3"><div class="ttname"><a href="namespaceimpala.html#aaebb9aadfa9e5aec064bc26565df06d3">impala::name</a></div><div class="ttdeci">string name</div><div class="ttdef"><b>Definition:</b> <a href="cpu-info_8cc_source.html#l00050">cpu-info.cc:50</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_ab8ae88546f4ce923b4d187e0d7d96fa8"><div class="ttname"><a href="classimpala_1_1Expr.html#ab8ae88546f4ce923b4d187e0d7d96fa8">impala::Expr::DebugString</a></div><div class="ttdeci">virtual std::string DebugString() const </div><div class="ttdef"><b>Definition:</b> <a href="expr_8cc_source.html#l00385">expr.cc:385</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a4e5f342bd92c77e85c90ede98dda8923"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a4e5f342bd92c77e85c90ede98dda8923">impala::HdfsTableSink::bytes_written_counter_</a></div><div class="ttdeci">RuntimeProfile::Counter * bytes_written_counter_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00276">hdfs-table-sink.h:276</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableDescriptor_html_a353f9c28c66a50e66b31e48a77161be3"><div class="ttname"><a href="classimpala_1_1HdfsTableDescriptor.html#a353f9c28c66a50e66b31e48a77161be3">impala::HdfsTableDescriptor::null_partition_key_value</a></div><div class="ttdeci">const std::string & null_partition_key_value() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00230">descriptors.h:230</a></div></div> |
| <div class="ttc" id="structimpala_1_1OutputPartition_html_aae4c6c45ad22865df963002c3f74d25e"><div class="ttname"><a href="structimpala_1_1OutputPartition.html#aae4c6c45ad22865df963002c3f74d25e">impala::OutputPartition::tmp_hdfs_file_name_prefix</a></div><div class="ttdeci">std::string tmp_hdfs_file_name_prefix</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00065">hdfs-table-sink.h:65</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a97ae2a720cc57b784f2bd23ec4ee9a15"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a97ae2a720cc57b784f2bd23ec4ee9a15">impala::HdfsTableSink::Open</a></div><div class="ttdeci">virtual Status Open(RuntimeState *state)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8cc_source.html#l00160">hdfs-table-sink.cc:160</a></div></div> |
| <div class="ttc" id="hdfs-parquet-table-writer_8h_html"><div class="ttname"><a href="hdfs-parquet-table-writer_8h.html">hdfs-parquet-table-writer.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a743942ea55982b76101671cd20d00807"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a743942ea55982b76101671cd20d00807">impala::HdfsTableSink::partition_keys_to_output_partitions_</a></div><div class="ttdeci">PartitionMap partition_keys_to_output_partitions_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00255">hdfs-table-sink.h:255</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a00633ae3b4ac61a6419d543852e67dff"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a00633ae3b4ac61a6419d543852e67dff">impala::HdfsTableSink::partition_key_texprs_</a></div><div class="ttdeci">const std::vector< TExpr > & partition_key_texprs_</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00234">hdfs-table-sink.h:234</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsPartitionDescriptor_html_a87507069eae1cbb5686df05d723f98b9"><div class="ttname"><a href="classimpala_1_1HdfsPartitionDescriptor.html#a87507069eae1cbb5686df05d723f98b9">impala::HdfsPartitionDescriptor::partition_key_value_ctxs</a></div><div class="ttdeci">const std::vector< ExprContext * > & partition_key_value_ctxs() const </div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8h_source.html#l00185">descriptors.h:185</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsTableSink_html_a09ff5ccfa2aed127b0e8a267fa085c94"><div class="ttname"><a href="classimpala_1_1HdfsTableSink.html#a09ff5ccfa2aed127b0e8a267fa085c94">impala::HdfsTableSink::row_desc_</a></div><div class="ttdeci">const RowDescriptor & row_desc_</div><div class="ttdoc">Row descriptor of row batches passed in Send(). Set in c'tor. </div><div class="ttdef"><b>Definition:</b> <a href="hdfs-table-sink_8h_source.html#l00223">hdfs-table-sink.h:223</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_html_abd28edd77ba3928dbac3af735102fb3b"><div class="ttname"><a href="classimpala_1_1RuntimeProfile.html#abd28edd77ba3928dbac3af735102fb3b">impala::RuntimeProfile::total_time_counter</a></div><div class="ttdeci">Counter * total_time_counter()</div><div class="ttdoc">Returns the counter for the total elapsed time. </div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00453">runtime-profile.h:453</a></div></div> |
| </div><!-- fragment --></div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_e5d120be6b5e8a44336cbfd013b25604.html">be</a></li><li class="navelem"><a class="el" href="dir_68cf7cafb51a962d5bc4848b83cab0de.html">src</a></li><li class="navelem"><a class="el" href="dir_479336c0a15f2c0737bcafcf969f884c.html">exec</a></li><li class="navelem"><a class="el" href="hdfs-table-sink_8cc.html">hdfs-table-sink.cc</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:35 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
