| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: be/src/runtime/coordinator.cc Source File</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('coordinator_8cc_source.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">coordinator.cc</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| <a href="coordinator_8cc.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">// Copyright 2012 Cloudera Inc.</span></div> |
| <div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment">// Licensed under the Apache License, Version 2.0 (the "License");</span></div> |
| <div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment">// you may not use this file except in compliance with the License.</span></div> |
| <div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment">// You may obtain a copy of the License at</span></div> |
| <div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment">// http://www.apache.org/licenses/LICENSE-2.0</span></div> |
| <div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment">// Unless required by applicable law or agreed to in writing, software</span></div> |
| <div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment">// distributed under the License is distributed on an "AS IS" BASIS,</span></div> |
| <div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment">// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div> |
| <div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment">// See the License for the specific language governing permissions and</span></div> |
| <div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment">// limitations under the License.</span></div> |
| <div class="line"><a name="l00014"></a><span class="lineno"> 14</span> </div> |
| <div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="preprocessor">#include "<a class="code" href="coordinator_8h.html">runtime/coordinator.h</a>"</span></div> |
| <div class="line"><a name="l00016"></a><span class="lineno"> 16</span> </div> |
| <div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="preprocessor">#include <limits></span></div> |
| <div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="preprocessor">#include <map></span></div> |
| <div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="preprocessor">#include <thrift/protocol/TDebugProtocol.h></span></div> |
| <div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="preprocessor">#include <boost/algorithm/string/join.hpp></span></div> |
| <div class="line"><a name="l00021"></a><span class="lineno"> 21</span> <span class="preprocessor">#include <boost/accumulators/accumulators.hpp></span></div> |
| <div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="preprocessor">#include <boost/accumulators/statistics/stats.hpp></span></div> |
| <div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="preprocessor">#include <boost/accumulators/statistics/min.hpp></span></div> |
| <div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="preprocessor">#include <boost/accumulators/statistics/mean.hpp></span></div> |
| <div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="preprocessor">#include <boost/accumulators/statistics/median.hpp></span></div> |
| <div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="preprocessor">#include <boost/accumulators/statistics/max.hpp></span></div> |
| <div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="preprocessor">#include <boost/accumulators/statistics/variance.hpp></span></div> |
| <div class="line"><a name="l00028"></a><span class="lineno"> 28</span> <span class="preprocessor">#include <boost/bind.hpp></span></div> |
| <div class="line"><a name="l00029"></a><span class="lineno"> 29</span> <span class="preprocessor">#include <boost/filesystem.hpp></span></div> |
| <div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="preprocessor">#include <boost/foreach.hpp></span></div> |
| <div class="line"><a name="l00031"></a><span class="lineno"> 31</span> <span class="preprocessor">#include <boost/lexical_cast.hpp></span></div> |
| <div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="preprocessor">#include <boost/unordered_set.hpp></span></div> |
| <div class="line"><a name="l00033"></a><span class="lineno"> 33</span> <span class="preprocessor">#include <boost/algorithm/string/split.hpp></span></div> |
| <div class="line"><a name="l00034"></a><span class="lineno"> 34</span> <span class="preprocessor">#include <boost/algorithm/string.hpp></span></div> |
| <div class="line"><a name="l00035"></a><span class="lineno"> 35</span> <span class="preprocessor">#include <gutil/strings/substitute.h></span></div> |
| <div class="line"><a name="l00036"></a><span class="lineno"> 36</span> </div> |
| <div class="line"><a name="l00037"></a><span class="lineno"> 37</span> <span class="preprocessor">#include "<a class="code" href="logging_8h.html">common/logging.h</a>"</span></div> |
| <div class="line"><a name="l00038"></a><span class="lineno"> 38</span> <span class="preprocessor">#include "<a class="code" href="expr_8h.html">exprs/expr.h</a>"</span></div> |
| <div class="line"><a name="l00039"></a><span class="lineno"> 39</span> <span class="preprocessor">#include "<a class="code" href="data-sink_8h.html">exec/data-sink.h</a>"</span></div> |
| <div class="line"><a name="l00040"></a><span class="lineno"> 40</span> <span class="preprocessor">#include "<a class="code" href="client-cache_8h.html">runtime/client-cache.h</a>"</span></div> |
| <div class="line"><a name="l00041"></a><span class="lineno"> 41</span> <span class="preprocessor">#include "<a class="code" href="data-stream-sender_8h.html">runtime/data-stream-sender.h</a>"</span></div> |
| <div class="line"><a name="l00042"></a><span class="lineno"> 42</span> <span class="preprocessor">#include "<a class="code" href="data-stream-mgr_8h.html">runtime/data-stream-mgr.h</a>"</span></div> |
| <div class="line"><a name="l00043"></a><span class="lineno"> 43</span> <span class="preprocessor">#include "<a class="code" href="exec-env_8h.html">runtime/exec-env.h</a>"</span></div> |
| <div class="line"><a name="l00044"></a><span class="lineno"> 44</span> <span class="preprocessor">#include "<a class="code" href="hdfs-fs-cache_8h.html">runtime/hdfs-fs-cache.h</a>"</span></div> |
| <div class="line"><a name="l00045"></a><span class="lineno"> 45</span> <span class="preprocessor">#include "<a class="code" href="plan-fragment-executor_8h.html">runtime/plan-fragment-executor.h</a>"</span></div> |
| <div class="line"><a name="l00046"></a><span class="lineno"> 46</span> <span class="preprocessor">#include "<a class="code" href="row-batch_8h.html">runtime/row-batch.h</a>"</span></div> |
| <div class="line"><a name="l00047"></a><span class="lineno"> 47</span> <span class="preprocessor">#include "<a class="code" href="parallel-executor_8h.html">runtime/parallel-executor.h</a>"</span></div> |
| <div class="line"><a name="l00048"></a><span class="lineno"> 48</span> <span class="preprocessor">#include "<a class="code" href="scheduler_8h.html">statestore/scheduler.h</a>"</span></div> |
| <div class="line"><a name="l00049"></a><span class="lineno"> 49</span> <span class="preprocessor">#include "<a class="code" href="data-sink_8h.html">exec/data-sink.h</a>"</span></div> |
| <div class="line"><a name="l00050"></a><span class="lineno"> 50</span> <span class="preprocessor">#include "<a class="code" href="scan-node_8h.html">exec/scan-node.h</a>"</span></div> |
| <div class="line"><a name="l00051"></a><span class="lineno"> 51</span> <span class="preprocessor">#include "<a class="code" href="container-util_8h.html">util/container-util.h</a>"</span></div> |
| <div class="line"><a name="l00052"></a><span class="lineno"> 52</span> <span class="preprocessor">#include "<a class="code" href="debug-util_8h.html">util/debug-util.h</a>"</span></div> |
| <div class="line"><a name="l00053"></a><span class="lineno"> 53</span> <span class="preprocessor">#include "<a class="code" href="error-util_8h.html">util/error-util.h</a>"</span></div> |
| <div class="line"><a name="l00054"></a><span class="lineno"> 54</span> <span class="preprocessor">#include "<a class="code" href="hdfs-bulk-ops_8h.html">util/hdfs-bulk-ops.h</a>"</span></div> |
| <div class="line"><a name="l00055"></a><span class="lineno"> 55</span> <span class="preprocessor">#include "<a class="code" href="hdfs-util_8h.html">util/hdfs-util.h</a>"</span></div> |
| <div class="line"><a name="l00056"></a><span class="lineno"> 56</span> <span class="preprocessor">#include "<a class="code" href="llama-util_8h.html">util/llama-util.h</a>"</span></div> |
| <div class="line"><a name="l00057"></a><span class="lineno"> 57</span> <span class="preprocessor">#include "<a class="code" href="network-util_8h.html">util/network-util.h</a>"</span></div> |
| <div class="line"><a name="l00058"></a><span class="lineno"> 58</span> <span class="preprocessor">#include "<a class="code" href="pretty-printer_8h.html">util/pretty-printer.h</a>"</span></div> |
| <div class="line"><a name="l00059"></a><span class="lineno"> 59</span> <span class="preprocessor">#include "<a class="code" href="summary-util_8h.html">util/summary-util.h</a>"</span></div> |
| <div class="line"><a name="l00060"></a><span class="lineno"> 60</span> <span class="preprocessor">#include "gen-cpp/ImpalaInternalService.h"</span></div> |
| <div class="line"><a name="l00061"></a><span class="lineno"> 61</span> <span class="preprocessor">#include "gen-cpp/ImpalaInternalService_types.h"</span></div> |
| <div class="line"><a name="l00062"></a><span class="lineno"> 62</span> <span class="preprocessor">#include "gen-cpp/Frontend_types.h"</span></div> |
| <div class="line"><a name="l00063"></a><span class="lineno"> 63</span> <span class="preprocessor">#include "gen-cpp/PlanNodes_types.h"</span></div> |
| <div class="line"><a name="l00064"></a><span class="lineno"> 64</span> <span class="preprocessor">#include "gen-cpp/Partitions_types.h"</span></div> |
| <div class="line"><a name="l00065"></a><span class="lineno"> 65</span> <span class="preprocessor">#include "gen-cpp/ImpalaInternalService_constants.h"</span></div> |
| <div class="line"><a name="l00066"></a><span class="lineno"> 66</span> </div> |
| <div class="line"><a name="l00067"></a><span class="lineno"> 67</span> <span class="preprocessor">#include "<a class="code" href="names_8h.html">common/names.h</a>"</span></div> |
| <div class="line"><a name="l00068"></a><span class="lineno"> 68</span> </div> |
| <div class="line"><a name="l00069"></a><span class="lineno"> 69</span> <span class="keyword">using namespace </span>apache::thrift;</div> |
| <div class="line"><a name="l00070"></a><span class="lineno"> 70</span> <span class="keyword">using namespace </span>strings;</div> |
| <div class="line"><a name="l00071"></a><span class="lineno"> 71</span> <span class="keyword">namespace </span>accumulators = boost::accumulators;</div> |
| <div class="line"><a name="l00072"></a><span class="lineno"> 72</span> <span class="keyword">using</span> boost::algorithm::iequals;</div> |
| <div class="line"><a name="l00073"></a><span class="lineno"> 73</span> <span class="keyword">using</span> boost::algorithm::is_any_of;</div> |
| <div class="line"><a name="l00074"></a><span class="lineno"> 74</span> <span class="keyword">using</span> boost::algorithm::join;</div> |
| <div class="line"><a name="l00075"></a><span class="lineno"> 75</span> <span class="keyword">using</span> boost::algorithm::token_compress_on;</div> |
| <div class="line"><a name="l00076"></a><span class="lineno"> 76</span> <span class="keyword">using</span> boost::algorithm::split;</div> |
| <div class="line"><a name="l00077"></a><span class="lineno"> 77</span> <span class="keyword">using</span> <a class="code" href="sasl-test_8cc.html#a2db66f7b463c0e190f8933e51f43d91e">boost::filesystem::path</a>;</div> |
| <div class="line"><a name="l00078"></a><span class="lineno"> 78</span> </div> |
| <div class="line"><a name="l00079"></a><span class="lineno"> 79</span> <a class="code" href="coordinator_8cc.html#a0bc8067794763b22cd794e6cac87ad6b">DECLARE_int32</a>(be_port);</div> |
| <div class="line"><a name="l00080"></a><span class="lineno"> 80</span> <a class="code" href="coordinator_8cc.html#a3bcc9693782d97a40316b227514bcd06">DECLARE_string</a>(hostname);</div> |
| <div class="line"><a name="l00081"></a><span class="lineno"> 81</span> </div> |
| <div class="line"><a name="l00082"></a><span class="lineno"> 82</span> <a class="code" href="coordinator_8cc.html#a8586080ec51402cfb11689fea6660a39">DEFINE_bool</a>(insert_inherit_permissions, <span class="keyword">false</span>, <span class="stringliteral">"If true, new directories created by "</span></div> |
| <div class="line"><a name="l00083"></a><span class="lineno"> 83</span>  <span class="stringliteral">"INSERTs will inherit the permissions of their parent directories"</span>);</div> |
| <div class="line"><a name="l00084"></a><span class="lineno"> 84</span> </div> |
| <div class="line"><a name="l00085"></a><span class="lineno"> 85</span> <span class="keyword">namespace </span>impala {</div> |
| <div class="line"><a name="l00086"></a><span class="lineno"> 86</span> </div> |
| <div class="line"><a name="l00087"></a><span class="lineno"> 87</span> <span class="comment">// container for debug options in TPlanFragmentExecParams (debug_node, debug_action,</span></div> |
| <div class="line"><a name="l00088"></a><span class="lineno"> 88</span> <span class="comment">// debug_phase)</span></div> |
| <div class="line"><a name="l00089"></a><span class="lineno"><a class="line" href="structimpala_1_1DebugOptions.html"> 89</a></span> <span class="keyword">struct </span><a class="code" href="structimpala_1_1DebugOptions.html">DebugOptions</a> {</div> |
| <div class="line"><a name="l00090"></a><span class="lineno"><a class="line" href="structimpala_1_1DebugOptions.html#ab0848258c00b410e8178975ece709db8"> 90</a></span>  <span class="keywordtype">int</span> <a class="code" href="structimpala_1_1DebugOptions.html#ab0848258c00b410e8178975ece709db8">backend_num</a>;</div> |
| <div class="line"><a name="l00091"></a><span class="lineno"><a class="line" href="structimpala_1_1DebugOptions.html#a585e0f226f2c13e409d89203f50f43ae"> 91</a></span>  <span class="keywordtype">int</span> <a class="code" href="structimpala_1_1DebugOptions.html#a585e0f226f2c13e409d89203f50f43ae">node_id</a>;</div> |
| <div class="line"><a name="l00092"></a><span class="lineno"><a class="line" href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b"> 92</a></span>  TDebugAction::type <a class="code" href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b">action</a>;</div> |
| <div class="line"><a name="l00093"></a><span class="lineno"><a class="line" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73"> 93</a></span>  TExecNodePhase::type <a class="code" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">phase</a>; <span class="comment">// INVALID: debug options invalid</span></div> |
| <div class="line"><a name="l00094"></a><span class="lineno"> 94</span> </div> |
| <div class="line"><a name="l00095"></a><span class="lineno"><a class="line" href="structimpala_1_1DebugOptions.html#aa704942f14835910654194513bdfbd85"> 95</a></span>  <a class="code" href="structimpala_1_1DebugOptions.html#aa704942f14835910654194513bdfbd85">DebugOptions</a>()</div> |
| <div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  : backend_num(-1), node_id(-1), action(TDebugAction::WAIT),</div> |
| <div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  phase(TExecNodePhase::INVALID) {}</div> |
| <div class="line"><a name="l00098"></a><span class="lineno"> 98</span> };</div> |
| <div class="line"><a name="l00099"></a><span class="lineno"> 99</span> </div> |
| <div class="line"><a name="l00100"></a><span class="lineno"> 100</span> <span class="comment">// Execution state of a particular fragment.</span></div> |
| <div class="line"><a name="l00101"></a><span class="lineno"> 101</span> <span class="comment">// Concurrent accesses:</span></div> |
| <div class="line"><a name="l00102"></a><span class="lineno"> 102</span> <span class="comment">// - GetNodeThroughput() called when coordinator's profile is printed</span></div> |
| <div class="line"><a name="l00103"></a><span class="lineno"> 103</span> <span class="comment">// - updates through UpdateFragmentExecStatus()</span></div> |
| <div class="line"><a name="l00104"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html"> 104</a></span> <span class="keyword">class </span><a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html">Coordinator::BackendExecState</a> {</div> |
| <div class="line"><a name="l00105"></a><span class="lineno"> 105</span>  <span class="keyword">public</span>:</div> |
| <div class="line"><a name="l00106"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a0de3913d5153b5256a357db2328e650d"> 106</a></span>  TUniqueId <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a0de3913d5153b5256a357db2328e650d">fragment_instance_id</a>;</div> |
| <div class="line"><a name="l00107"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a21a85a27441062e63b8858c1edde1886"> 107</a></span>  <a class="code" href="classimpala_1_1MonotonicStopWatch.html">MonotonicStopWatch</a> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a21a85a27441062e63b8858c1edde1886">stopwatch</a>; <span class="comment">// wall clock timer for this fragment</span></div> |
| <div class="line"><a name="l00108"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ac75dbc9b6de0c2c1616eb0d56a9602db"> 108</a></span>  <span class="keyword">const</span> TNetworkAddress <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ac75dbc9b6de0c2c1616eb0d56a9602db">backend_address</a>; <span class="comment">// of ImpalaInternalService</span></div> |
| <div class="line"><a name="l00109"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ab9ac162db9bba61a67ef0dc1db24ff91"> 109</a></span>  int64_t <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ab9ac162db9bba61a67ef0dc1db24ff91">total_split_size</a>; <span class="comment">// summed up across all splits; in bytes</span></div> |
| <div class="line"><a name="l00110"></a><span class="lineno"> 110</span> </div> |
| <div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <span class="comment">// assembled in c'tor</span></div> |
| <div class="line"><a name="l00112"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a947d4f7961a00d4b92cb3061d4b51f67"> 112</a></span>  TExecPlanFragmentParams <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a947d4f7961a00d4b92cb3061d4b51f67">rpc_params</a>;</div> |
| <div class="line"><a name="l00113"></a><span class="lineno"> 113</span> </div> |
| <div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  <span class="comment">// Fragment idx for this ExecState</span></div> |
| <div class="line"><a name="l00115"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#af1fe611bdd6d25b0e8ba4938b8efb4e2"> 115</a></span>  <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#af1fe611bdd6d25b0e8ba4938b8efb4e2">fragment_idx</a>;</div> |
| <div class="line"><a name="l00116"></a><span class="lineno"> 116</span> </div> |
| <div class="line"><a name="l00117"></a><span class="lineno"> 117</span>  <span class="comment">// The 0-based instance idx.</span></div> |
| <div class="line"><a name="l00118"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#af612858a21f2acad7ff0ee8f58dadf94"> 118</a></span>  <span class="keywordtype">int</span> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#af612858a21f2acad7ff0ee8f58dadf94">instance_idx</a>;</div> |
| <div class="line"><a name="l00119"></a><span class="lineno"> 119</span> </div> |
| <div class="line"><a name="l00120"></a><span class="lineno"> 120</span>  <span class="comment">// protects fields below</span></div> |
| <div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  <span class="comment">// lock ordering: Coordinator::lock_ can only get obtained *prior*</span></div> |
| <div class="line"><a name="l00122"></a><span class="lineno"> 122</span>  <span class="comment">// to lock</span></div> |
| <div class="line"><a name="l00123"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a94595ccb59d6b9ee3bf56ebb49097c4d"> 123</a></span>  boost::mutex <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a94595ccb59d6b9ee3bf56ebb49097c4d">lock</a>;</div> |
| <div class="line"><a name="l00124"></a><span class="lineno"> 124</span> </div> |
| <div class="line"><a name="l00125"></a><span class="lineno"> 125</span>  <span class="comment">// if the status indicates an error status, execution of this fragment</span></div> |
| <div class="line"><a name="l00126"></a><span class="lineno"> 126</span>  <span class="comment">// has either been aborted by the remote backend (which then reported the error)</span></div> |
| <div class="line"><a name="l00127"></a><span class="lineno"> 127</span>  <span class="comment">// or cancellation has been initiated; either way, execution must not be cancelled</span></div> |
| <div class="line"><a name="l00128"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a21ef571aa3ec060718ac944c5ef47001"> 128</a></span>  <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a21ef571aa3ec060718ac944c5ef47001">status</a>;</div> |
| <div class="line"><a name="l00129"></a><span class="lineno"> 129</span> </div> |
| <div class="line"><a name="l00130"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#aed65f272a2672410ec13d5fa02d2b7a3"> 130</a></span>  <span class="keywordtype">bool</span> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#aed65f272a2672410ec13d5fa02d2b7a3">initiated</a>; <span class="comment">// if true, TPlanExecRequest rpc has been sent</span></div> |
| <div class="line"><a name="l00131"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a9baeffdce3e7221d0c3e644adcb910bd"> 131</a></span>  <span class="keywordtype">bool</span> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a9baeffdce3e7221d0c3e644adcb910bd">done</a>; <span class="comment">// if true, execution terminated; do not cancel in that case</span></div> |
| <div class="line"><a name="l00132"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a0da064b03c3877a662a1830117222c1a"> 132</a></span>  <span class="keywordtype">bool</span> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a0da064b03c3877a662a1830117222c1a">profile_created</a>; <span class="comment">// true after the first call to profile->Update()</span></div> |
| <div class="line"><a name="l00133"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a759b34e1712af6039b9c6e81ed83a798"> 133</a></span>  <a class="code" href="classimpala_1_1RuntimeProfile.html">RuntimeProfile</a>* <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a759b34e1712af6039b9c6e81ed83a798">profile</a>; <span class="comment">// owned by obj_pool()</span></div> |
| <div class="line"><a name="l00134"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ae011202240d67890415790622b7ea5c2"> 134</a></span>  <a class="code" href="namespaceimpala.html#a940125179d49c976f146709006813049">ErrorLogMap</a> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ae011202240d67890415790622b7ea5c2">error_log</a>; <span class="comment">// errors reported by this backend</span></div> |
| <div class="line"><a name="l00135"></a><span class="lineno"> 135</span> </div> |
| <div class="line"><a name="l00136"></a><span class="lineno"> 136</span>  <span class="comment">// Total scan ranges complete across all scan nodes</span></div> |
| <div class="line"><a name="l00137"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#af31d42eeb3f5ce24b0e4548b7593a99a"> 137</a></span>  int64_t <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#af31d42eeb3f5ce24b0e4548b7593a99a">total_ranges_complete</a>;</div> |
| <div class="line"><a name="l00138"></a><span class="lineno"> 138</span> </div> |
| <div class="line"><a name="l00139"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a95a1797a63017a4504f27118b984adf2"> 139</a></span>  <a class="code" href="structimpala_1_1FragmentInstanceCounters.html">FragmentInstanceCounters</a> <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a95a1797a63017a4504f27118b984adf2">aggregate_counters</a>;</div> |
| <div class="line"><a name="l00140"></a><span class="lineno"> 140</span> </div> |
| <div class="line"><a name="l00141"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a90b9637d6b3e8b12fdefc25fde7786a8"> 141</a></span>  <a class="code" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a90b9637d6b3e8b12fdefc25fde7786a8">BackendExecState</a>(<a class="code" href="classimpala_1_1QuerySchedule.html">QuerySchedule</a>& schedule, Coordinator* coord,</div> |
| <div class="line"><a name="l00142"></a><span class="lineno"> 142</span>  <span class="keyword">const</span> TNetworkAddress& coord_address,</div> |
| <div class="line"><a name="l00143"></a><span class="lineno"> 143</span>  <span class="keywordtype">int</span> backend_num, <span class="keyword">const</span> TPlanFragment& fragment, <span class="keywordtype">int</span> fragment_idx,</div> |
| <div class="line"><a name="l00144"></a><span class="lineno"> 144</span>  <span class="keyword">const</span> <a class="code" href="structimpala_1_1FragmentExecParams.html">FragmentExecParams</a>& params, <span class="keywordtype">int</span> instance_idx,</div> |
| <div class="line"><a name="l00145"></a><span class="lineno"> 145</span>  <a class="code" href="structimpala_1_1DebugOptions.html">DebugOptions</a>* debug_options, <a class="code" href="classimpala_1_1ObjectPool.html">ObjectPool</a>* <a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>)</div> |
| <div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  : fragment_instance_id(params.instance_ids[instance_idx]),</div> |
| <div class="line"><a name="l00147"></a><span class="lineno"> 147</span>  backend_address(params.hosts[instance_idx]),</div> |
| <div class="line"><a name="l00148"></a><span class="lineno"> 148</span>  total_split_size(0),</div> |
| <div class="line"><a name="l00149"></a><span class="lineno"> 149</span>  fragment_idx(fragment_idx),</div> |
| <div class="line"><a name="l00150"></a><span class="lineno"> 150</span>  instance_idx(instance_idx),</div> |
| <div class="line"><a name="l00151"></a><span class="lineno"> 151</span>  initiated(false),</div> |
| <div class="line"><a name="l00152"></a><span class="lineno"> 152</span>  done(false),</div> |
| <div class="line"><a name="l00153"></a><span class="lineno"> 153</span>  profile_created(false),</div> |
| <div class="line"><a name="l00154"></a><span class="lineno"> 154</span>  total_ranges_complete(0) {</div> |
| <div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  stringstream ss;</div> |
| <div class="line"><a name="l00156"></a><span class="lineno"> 156</span>  ss << <span class="stringliteral">"Instance "</span> << <a class="code" href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">PrintId</a>(fragment_instance_id)</div> |
| <div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  << <span class="stringliteral">" (host="</span> << backend_address << <span class="stringliteral">")"</span>;</div> |
| <div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  profile = obj_pool-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<span class="keyword">new</span> <a class="code" href="classimpala_1_1RuntimeProfile.html">RuntimeProfile</a>(obj_pool, ss.str()));</div> |
| <div class="line"><a name="l00159"></a><span class="lineno"> 159</span>  coord->SetExecPlanFragmentParams(schedule, backend_num, fragment, fragment_idx,</div> |
| <div class="line"><a name="l00160"></a><span class="lineno"> 160</span>  params, instance_idx, coord_address, &rpc_params);</div> |
| <div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <span class="keywordflow">if</span> (debug_options != NULL) {</div> |
| <div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  rpc_params.params.__set_debug_node_id(debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#a585e0f226f2c13e409d89203f50f43ae">node_id</a>);</div> |
| <div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  rpc_params.params.__set_debug_action(debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b">action</a>);</div> |
| <div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  rpc_params.params.__set_debug_phase(debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">phase</a>);</div> |
| <div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  }</div> |
| <div class="line"><a name="l00166"></a><span class="lineno"> 166</span>  ComputeTotalSplitSize();</div> |
| <div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  }</div> |
| <div class="line"><a name="l00168"></a><span class="lineno"> 168</span> </div> |
| <div class="line"><a name="l00169"></a><span class="lineno"> 169</span>  <span class="comment">// Computes sum of split sizes of leftmost scan. Call only after setting</span></div> |
| <div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  <span class="comment">// exec_params.</span></div> |
| <div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  <span class="keywordtype">void</span> ComputeTotalSplitSize();</div> |
| <div class="line"><a name="l00172"></a><span class="lineno"> 172</span> </div> |
| <div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  <span class="comment">// Return value of throughput counter for given plan_node_id, or 0 if that node</span></div> |
| <div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  <span class="comment">// doesn't exist.</span></div> |
| <div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  <span class="comment">// Thread-safe.</span></div> |
| <div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  int64_t GetNodeThroughput(<span class="keywordtype">int</span> plan_node_id);</div> |
| <div class="line"><a name="l00177"></a><span class="lineno"> 177</span> </div> |
| <div class="line"><a name="l00178"></a><span class="lineno"> 178</span>  <span class="comment">// Return number of completed scan ranges for plan_node_id, or 0 if that node</span></div> |
| <div class="line"><a name="l00179"></a><span class="lineno"> 179</span>  <span class="comment">// doesn't exist.</span></div> |
| <div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  <span class="comment">// Thread-safe.</span></div> |
| <div class="line"><a name="l00181"></a><span class="lineno"> 181</span>  int64_t GetNumScanRangesCompleted(<span class="keywordtype">int</span> plan_node_id);</div> |
| <div class="line"><a name="l00182"></a><span class="lineno"> 182</span> </div> |
| <div class="line"><a name="l00183"></a><span class="lineno"> 183</span>  <span class="comment">// Updates the total number of scan ranges complete for this fragment. Returns</span></div> |
| <div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  <span class="comment">// the delta since the last time this was called.</span></div> |
| <div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  <span class="comment">// lock must be taken before calling this.</span></div> |
| <div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  int64_t UpdateNumScanRangesCompleted();</div> |
| <div class="line"><a name="l00187"></a><span class="lineno"> 187</span> };</div> |
| <div class="line"><a name="l00188"></a><span class="lineno"> 188</span> </div> |
| <div class="line"><a name="l00189"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a36694b2204f19835f96e27121ce790a5"> 189</a></span> <span class="keywordtype">void</span> Coordinator::BackendExecState::ComputeTotalSplitSize() {</div> |
| <div class="line"><a name="l00190"></a><span class="lineno"> 190</span>  <span class="keyword">const</span> <a class="code" href="namespaceimpala.html#a48e0e1c5d4b43432450c92750fb35cb3">PerNodeScanRanges</a>& per_node_scan_ranges = rpc_params.params.per_node_scan_ranges;</div> |
| <div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  total_split_size = 0;</div> |
| <div class="line"><a name="l00192"></a><span class="lineno"> 192</span>  BOOST_FOREACH(<span class="keyword">const</span> PerNodeScanRanges::value_type& entry, per_node_scan_ranges) {</div> |
| <div class="line"><a name="l00193"></a><span class="lineno"> 193</span>  BOOST_FOREACH(<span class="keyword">const</span> TScanRangeParams& scan_range_params, entry.second) {</div> |
| <div class="line"><a name="l00194"></a><span class="lineno"> 194</span>  <span class="keywordflow">if</span> (!scan_range_params.scan_range.__isset.hdfs_file_split) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  total_split_size += scan_range_params.scan_range.hdfs_file_split.length;</div> |
| <div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  }</div> |
| <div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  }</div> |
| <div class="line"><a name="l00198"></a><span class="lineno"> 198</span> }</div> |
| <div class="line"><a name="l00199"></a><span class="lineno"> 199</span> </div> |
| <div class="line"><a name="l00200"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#a2d25aa970bc4e657dbd1b06d3c3dd261"> 200</a></span> int64_t Coordinator::BackendExecState::GetNodeThroughput(<span class="keywordtype">int</span> plan_node_id) {</div> |
| <div class="line"><a name="l00201"></a><span class="lineno"> 201</span>  <a class="code" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a>* counter = NULL;</div> |
| <div class="line"><a name="l00202"></a><span class="lineno"> 202</span>  {</div> |
| <div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  lock_guard<mutex> l(lock);</div> |
| <div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  <a class="code" href="namespaceimpala.html#a47d8255dea5b52dc4c93a11ed02b29b5">CounterMap</a>& throughput_counters = aggregate_counters.throughput_counters;</div> |
| <div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  CounterMap::iterator i = throughput_counters.find(plan_node_id);</div> |
| <div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  <span class="keywordflow">if</span> (i == throughput_counters.end()) <span class="keywordflow">return</span> 0;</div> |
| <div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  counter = i->second;</div> |
| <div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  }</div> |
| <div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  DCHECK(counter != NULL);</div> |
| <div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  <span class="comment">// make sure not to hold lock when calling value() to avoid potential deadlocks</span></div> |
| <div class="line"><a name="l00211"></a><span class="lineno"> 211</span>  <span class="keywordflow">return</span> counter-><a class="code" href="classimpala_1_1RuntimeProfile_1_1Counter.html#a6db04a7861baf700e025bea45de5a2e1">value</a>();</div> |
| <div class="line"><a name="l00212"></a><span class="lineno"> 212</span> }</div> |
| <div class="line"><a name="l00213"></a><span class="lineno"> 213</span> </div> |
| <div class="line"><a name="l00214"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#acff4531b8f1e90899948d0338af9ff31"> 214</a></span> int64_t Coordinator::BackendExecState::GetNumScanRangesCompleted(<span class="keywordtype">int</span> plan_node_id) {</div> |
| <div class="line"><a name="l00215"></a><span class="lineno"> 215</span>  <a class="code" href="classimpala_1_1RuntimeProfile_1_1Counter.html">RuntimeProfile::Counter</a>* counter = NULL;</div> |
| <div class="line"><a name="l00216"></a><span class="lineno"> 216</span>  {</div> |
| <div class="line"><a name="l00217"></a><span class="lineno"> 217</span>  lock_guard<mutex> l(lock);</div> |
| <div class="line"><a name="l00218"></a><span class="lineno"> 218</span>  <a class="code" href="namespaceimpala.html#a47d8255dea5b52dc4c93a11ed02b29b5">CounterMap</a>& ranges_complete = aggregate_counters.scan_ranges_complete_counters;</div> |
| <div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  CounterMap::iterator i = ranges_complete.find(plan_node_id);</div> |
| <div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="keywordflow">if</span> (i == ranges_complete.end()) <span class="keywordflow">return</span> 0;</div> |
| <div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  counter = i->second;</div> |
| <div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  }</div> |
| <div class="line"><a name="l00223"></a><span class="lineno"> 223</span>  DCHECK(counter != NULL);</div> |
| <div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="comment">// make sure not to hold lock when calling value() to avoid potential deadlocks</span></div> |
| <div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  <span class="keywordflow">return</span> counter-><a class="code" href="classimpala_1_1RuntimeProfile_1_1Counter.html#a6db04a7861baf700e025bea45de5a2e1">value</a>();</div> |
| <div class="line"><a name="l00226"></a><span class="lineno"> 226</span> }</div> |
| <div class="line"><a name="l00227"></a><span class="lineno"> 227</span> </div> |
| <div class="line"><a name="l00228"></a><span class="lineno"><a class="line" href="classimpala_1_1Coordinator_1_1BackendExecState.html#ae8d847caeb3764ee61ff39c73c4abaf3"> 228</a></span> int64_t Coordinator::BackendExecState::UpdateNumScanRangesCompleted() {</div> |
| <div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  int64_t total = 0;</div> |
| <div class="line"><a name="l00230"></a><span class="lineno"> 230</span>  <a class="code" href="namespaceimpala.html#a47d8255dea5b52dc4c93a11ed02b29b5">CounterMap</a>& complete = aggregate_counters.scan_ranges_complete_counters;</div> |
| <div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  <span class="keywordflow">for</span> (CounterMap::iterator i = complete.begin(); i != complete.end(); ++i) {</div> |
| <div class="line"><a name="l00232"></a><span class="lineno"> 232</span>  total += i->second->value();</div> |
| <div class="line"><a name="l00233"></a><span class="lineno"> 233</span>  }</div> |
| <div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  int64_t delta = total - total_ranges_complete;</div> |
| <div class="line"><a name="l00235"></a><span class="lineno"> 235</span>  total_ranges_complete = total;</div> |
| <div class="line"><a name="l00236"></a><span class="lineno"> 236</span>  DCHECK_GE(delta, 0);</div> |
| <div class="line"><a name="l00237"></a><span class="lineno"> 237</span>  <span class="keywordflow">return</span> delta;</div> |
| <div class="line"><a name="l00238"></a><span class="lineno"> 238</span> }</div> |
| <div class="line"><a name="l00239"></a><span class="lineno"> 239</span> </div> |
| <div class="line"><a name="l00240"></a><span class="lineno"> 240</span> Coordinator::Coordinator(<a class="code" href="classimpala_1_1ExecEnv.html">ExecEnv</a>* exec_env, <a class="code" href="classimpala_1_1RuntimeProfile_1_1EventSequence.html">RuntimeProfile::EventSequence</a>* <a class="code" href="namespaceimpala.html#a5368f0a1443b0bd404d98b27107678a4">events</a>)</div> |
| <div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  : <a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>(exec_env),</div> |
| <div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  <a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a>(false),</div> |
| <div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  <a class="code" href="namespaceimpala.html#aea293e10bddb1360a019cd1e9abf45f2">returned_all_results_</a>(false),</div> |
| <div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>(NULL), <span class="comment">// Set in Prepare()</span></div> |
| <div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  <a class="code" href="namespaceimpala.html#a5219328e34f90ec0679655b8e66bd31f">query_mem_tracker_</a>(), <span class="comment">// Set in Exec()</span></div> |
| <div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  <a class="code" href="namespaceimpala.html#a20991e967dbb80151f6500eb14145943">num_remaining_backends_</a>(0),</div> |
| <div class="line"><a name="l00247"></a><span class="lineno"> 247</span>  <a class="code" href="namespaceimpala.html#a1c183dc242f42291fbf439e78bebc8fa">obj_pool_</a>(new <a class="code" href="classimpala_1_1ObjectPool.html">ObjectPool</a>()),</div> |
| <div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  <a class="code" href="namespaceimpala.html#a316415821307d27237bed61e0be1ef0e">query_events_</a>(events) {</div> |
| <div class="line"><a name="l00249"></a><span class="lineno"> 249</span> }</div> |
| <div class="line"><a name="l00250"></a><span class="lineno"> 250</span> </div> |
| <div class="line"><a name="l00251"></a><span class="lineno"> 251</span> <a class="code" href="namespaceimpala.html#a33916d0527ad4e5ed5fb5b8089e90888">Coordinator::~Coordinator</a>() {</div> |
| <div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  <a class="code" href="namespaceimpala.html#a5219328e34f90ec0679655b8e66bd31f">query_mem_tracker_</a>.reset();</div> |
| <div class="line"><a name="l00253"></a><span class="lineno"> 253</span> }</div> |
| <div class="line"><a name="l00254"></a><span class="lineno"> 254</span> </div> |
| <div class="line"><a name="l00255"></a><span class="lineno"><a class="line" href="namespaceimpala.html#a427ff48858b169ea3a12329b72d022b5"> 255</a></span> TExecNodePhase::type <a class="code" href="namespaceimpala.html#a427ff48858b169ea3a12329b72d022b5">GetExecNodePhase</a>(<span class="keyword">const</span> <span class="keywordtype">string</span>& key) {</div> |
| <div class="line"><a name="l00256"></a><span class="lineno"> 256</span>  map<int, const char*>::const_iterator entry =</div> |
| <div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  _TExecNodePhase_VALUES_TO_NAMES.begin();</div> |
| <div class="line"><a name="l00258"></a><span class="lineno"> 258</span>  <span class="keywordflow">for</span> (; entry != _TExecNodePhase_VALUES_TO_NAMES.end(); ++entry) {</div> |
| <div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  <span class="keywordflow">if</span> (iequals(key, (*entry).second)) {</div> |
| <div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  <span class="keywordflow">return</span> <span class="keyword">static_cast<</span>TExecNodePhase::type<span class="keyword">></span>(entry->first);</div> |
| <div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  }</div> |
| <div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  }</div> |
| <div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  <span class="keywordflow">return</span> TExecNodePhase::INVALID;</div> |
| <div class="line"><a name="l00264"></a><span class="lineno"> 264</span> }</div> |
| <div class="line"><a name="l00265"></a><span class="lineno"> 265</span> </div> |
| <div class="line"><a name="l00266"></a><span class="lineno"> 266</span> <span class="comment">// TODO: templatize this</span></div> |
| <div class="line"><a name="l00267"></a><span class="lineno"><a class="line" href="namespaceimpala.html#a87523263d912f04770414ae7dbe55d01"> 267</a></span> TDebugAction::type <a class="code" href="namespaceimpala.html#a87523263d912f04770414ae7dbe55d01">GetDebugAction</a>(<span class="keyword">const</span> <span class="keywordtype">string</span>& key) {</div> |
| <div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  map<int, const char*>::const_iterator entry =</div> |
| <div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  _TDebugAction_VALUES_TO_NAMES.begin();</div> |
| <div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  <span class="keywordflow">for</span> (; entry != _TDebugAction_VALUES_TO_NAMES.end(); ++entry) {</div> |
| <div class="line"><a name="l00271"></a><span class="lineno"> 271</span>  <span class="keywordflow">if</span> (iequals(key, (*entry).second)) {</div> |
| <div class="line"><a name="l00272"></a><span class="lineno"> 272</span>  <span class="keywordflow">return</span> <span class="keyword">static_cast<</span>TDebugAction::type<span class="keyword">></span>(entry->first);</div> |
| <div class="line"><a name="l00273"></a><span class="lineno"> 273</span>  }</div> |
| <div class="line"><a name="l00274"></a><span class="lineno"> 274</span>  }</div> |
| <div class="line"><a name="l00275"></a><span class="lineno"> 275</span>  <span class="keywordflow">return</span> TDebugAction::WAIT;</div> |
| <div class="line"><a name="l00276"></a><span class="lineno"> 276</span> }</div> |
| <div class="line"><a name="l00277"></a><span class="lineno"> 277</span> </div> |
| <div class="line"><a name="l00278"></a><span class="lineno"><a class="line" href="namespaceimpala.html#a7ce4ff5974d4d8b4a5c20b36f061af7a"> 278</a></span> <span class="keyword">static</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a7ce4ff5974d4d8b4a5c20b36f061af7a">ProcessQueryOptions</a>(</div> |
| <div class="line"><a name="l00279"></a><span class="lineno"> 279</span>  <span class="keyword">const</span> TQueryOptions& query_options, <a class="code" href="structimpala_1_1DebugOptions.html">DebugOptions</a>* debug_options) {</div> |
| <div class="line"><a name="l00280"></a><span class="lineno"> 280</span>  DCHECK(debug_options != NULL);</div> |
| <div class="line"><a name="l00281"></a><span class="lineno"> 281</span>  <span class="keywordflow">if</span> (!query_options.__isset.debug_action || query_options.debug_action.empty()) {</div> |
| <div class="line"><a name="l00282"></a><span class="lineno"> 282</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">phase</a> = TExecNodePhase::INVALID; <span class="comment">// signal not set</span></div> |
| <div class="line"><a name="l00283"></a><span class="lineno"> 283</span>  <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00284"></a><span class="lineno"> 284</span>  }</div> |
| <div class="line"><a name="l00285"></a><span class="lineno"> 285</span>  vector<string> components;</div> |
| <div class="line"><a name="l00286"></a><span class="lineno"> 286</span>  split(components, query_options.debug_action, is_any_of(<span class="stringliteral">":"</span>), token_compress_on);</div> |
| <div class="line"><a name="l00287"></a><span class="lineno"> 287</span>  <span class="keywordflow">if</span> (components.size() < 3 || components.size() > 4) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  <span class="keywordflow">if</span> (components.size() == 3) {</div> |
| <div class="line"><a name="l00289"></a><span class="lineno"> 289</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#ab0848258c00b410e8178975ece709db8">backend_num</a> = -1;</div> |
| <div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#a585e0f226f2c13e409d89203f50f43ae">node_id</a> = atoi(components[0].c_str());</div> |
| <div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">phase</a> = <a class="code" href="namespaceimpala.html#a427ff48858b169ea3a12329b72d022b5">GetExecNodePhase</a>(components[1]);</div> |
| <div class="line"><a name="l00292"></a><span class="lineno"> 292</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b">action</a> = <a class="code" href="namespaceimpala.html#a87523263d912f04770414ae7dbe55d01">GetDebugAction</a>(components[2]);</div> |
| <div class="line"><a name="l00293"></a><span class="lineno"> 293</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00294"></a><span class="lineno"> 294</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#ab0848258c00b410e8178975ece709db8">backend_num</a> = atoi(components[0].c_str());</div> |
| <div class="line"><a name="l00295"></a><span class="lineno"> 295</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#a585e0f226f2c13e409d89203f50f43ae">node_id</a> = atoi(components[1].c_str());</div> |
| <div class="line"><a name="l00296"></a><span class="lineno"> 296</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">phase</a> = <a class="code" href="namespaceimpala.html#a427ff48858b169ea3a12329b72d022b5">GetExecNodePhase</a>(components[2]);</div> |
| <div class="line"><a name="l00297"></a><span class="lineno"> 297</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b">action</a> = <a class="code" href="namespaceimpala.html#a87523263d912f04770414ae7dbe55d01">GetDebugAction</a>(components[3]);</div> |
| <div class="line"><a name="l00298"></a><span class="lineno"> 298</span>  }</div> |
| <div class="line"><a name="l00299"></a><span class="lineno"> 299</span>  DCHECK(!(debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">phase</a> == TExecNodePhase::CLOSE &&</div> |
| <div class="line"><a name="l00300"></a><span class="lineno"> 300</span>  debug_options-><a class="code" href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b">action</a> == TDebugAction::WAIT))</div> |
| <div class="line"><a name="l00301"></a><span class="lineno"> 301</span>  << <span class="stringliteral">"Do not use CLOSE:WAIT debug actions "</span></div> |
| <div class="line"><a name="l00302"></a><span class="lineno"> 302</span>  << <span class="stringliteral">"because nodes cannot be cancelled in Close()"</span>;</div> |
| <div class="line"><a name="l00303"></a><span class="lineno"> 303</span> }</div> |
| <div class="line"><a name="l00304"></a><span class="lineno"> 304</span> </div> |
| <div class="line"><a name="l00305"></a><span class="lineno"> 305</span> Status <a class="code" href="namespaceimpala.html#a8173907db7b6f1785b33db9d4e253be7">Coordinator::Exec</a>(QuerySchedule& schedule,</div> |
| <div class="line"><a name="l00306"></a><span class="lineno"> 306</span>  vector<ExprContext*>* output_expr_ctxs) {</div> |
| <div class="line"><a name="l00307"></a><span class="lineno"> 307</span>  <span class="keyword">const</span> TQueryExecRequest& request = schedule.request();</div> |
| <div class="line"><a name="l00308"></a><span class="lineno"> 308</span>  DCHECK_GT(request.fragments.size(), 0);</div> |
| <div class="line"><a name="l00309"></a><span class="lineno"> 309</span>  <a class="code" href="namespaceimpala.html#aae8267be6175d1f4208f54913a68ea7c">needs_finalization_</a> = request.__isset.finalize_params;</div> |
| <div class="line"><a name="l00310"></a><span class="lineno"> 310</span>  <span class="keywordflow">if</span> (needs_finalization_) {</div> |
| <div class="line"><a name="l00311"></a><span class="lineno"> 311</span>  <a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a> = request.finalize_params;</div> |
| <div class="line"><a name="l00312"></a><span class="lineno"> 312</span>  }</div> |
| <div class="line"><a name="l00313"></a><span class="lineno"> 313</span> </div> |
| <div class="line"><a name="l00314"></a><span class="lineno"> 314</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Exec() query_id="</span> << schedule.query_id();</div> |
| <div class="line"><a name="l00315"></a><span class="lineno"> 315</span>  <a class="code" href="namespaceimpala.html#a4f604de2fe46ab0a3969d06f4126bc77">stmt_type_</a> = request.stmt_type;</div> |
| <div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a> = schedule.query_id();</div> |
| <div class="line"><a name="l00317"></a><span class="lineno"> 317</span>  <a class="code" href="namespaceimpala.html#a0a06145acd27005a1c1f482dc2375742">desc_tbl_</a> = request.desc_tbl;</div> |
| <div class="line"><a name="l00318"></a><span class="lineno"> 318</span>  <a class="code" href="namespaceimpala.html#ab296eb278342117dde26e2e28bbe8190">query_ctx_</a> = request.query_ctx;</div> |
| <div class="line"><a name="l00319"></a><span class="lineno"> 319</span> </div> |
| <div class="line"><a name="l00320"></a><span class="lineno"> 320</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>.reset(</div> |
| <div class="line"><a name="l00321"></a><span class="lineno"> 321</span>  <span class="keyword">new</span> RuntimeProfile(<a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>(), <span class="stringliteral">"Execution Profile "</span> + <a class="code" href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">PrintId</a>(<a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>)));</div> |
| <div class="line"><a name="l00322"></a><span class="lineno"> 322</span>  <a class="code" href="namespaceimpala.html#a7d75818564bd9d86830ca57bbe71116d">finalization_timer_</a> = <a class="code" href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>, <span class="stringliteral">"FinalizationTimer"</span>);</div> |
| <div class="line"><a name="l00323"></a><span class="lineno"> 323</span> </div> |
| <div class="line"><a name="l00324"></a><span class="lineno"> 324</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->total_time_counter());</div> |
| <div class="line"><a name="l00325"></a><span class="lineno"> 325</span> </div> |
| <div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  vector<FragmentExecParams>* fragment_exec_params = schedule.exec_params();</div> |
| <div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  TNetworkAddress coord = <a class="code" href="namespaceimpala.html#a309108c8b3eaf4e5b154bc4eb4624880">MakeNetworkAddress</a>(FLAGS_hostname, FLAGS_be_port);</div> |
| <div class="line"><a name="l00328"></a><span class="lineno"> 328</span> </div> |
| <div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  <span class="comment">// to keep things simple, make async Cancel() calls wait until plan fragment</span></div> |
| <div class="line"><a name="l00330"></a><span class="lineno"> 330</span>  <span class="comment">// execution has been initiated, otherwise we might try to cancel fragment</span></div> |
| <div class="line"><a name="l00331"></a><span class="lineno"> 331</span>  <span class="comment">// execution at backends where it hasn't even started</span></div> |
| <div class="line"><a name="l00332"></a><span class="lineno"> 332</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l00333"></a><span class="lineno"> 333</span> </div> |
| <div class="line"><a name="l00334"></a><span class="lineno"> 334</span>  <span class="comment">// we run the root fragment ourselves if it is unpartitioned</span></div> |
| <div class="line"><a name="l00335"></a><span class="lineno"> 335</span>  <span class="keywordtype">bool</span> has_coordinator_fragment =</div> |
| <div class="line"><a name="l00336"></a><span class="lineno"> 336</span>  request.fragments[0].partition.type == TPartitionType::UNPARTITIONED;</div> |
| <div class="line"><a name="l00337"></a><span class="lineno"> 337</span> </div> |
| <div class="line"><a name="l00338"></a><span class="lineno"> 338</span>  <span class="keywordflow">if</span> (has_coordinator_fragment) {</div> |
| <div class="line"><a name="l00339"></a><span class="lineno"> 339</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.reset(<span class="keyword">new</span> PlanFragmentExecutor(</div> |
| <div class="line"><a name="l00340"></a><span class="lineno"> 340</span>  <a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>, <a class="code" href="classimpala_1_1PlanFragmentExecutor.html#a155af468fa604258d08f8dab01b309e7">PlanFragmentExecutor::ReportStatusCallback</a>()));</div> |
| <div class="line"><a name="l00341"></a><span class="lineno"> 341</span>  <span class="comment">// If a coordinator fragment is requested (for most queries this</span></div> |
| <div class="line"><a name="l00342"></a><span class="lineno"> 342</span>  <span class="comment">// will be the case, the exception is parallel INSERT queries), start</span></div> |
| <div class="line"><a name="l00343"></a><span class="lineno"> 343</span>  <span class="comment">// this before starting any more plan fragments in backend threads,</span></div> |
| <div class="line"><a name="l00344"></a><span class="lineno"> 344</span>  <span class="comment">// otherwise they start sending data before the local exchange node</span></div> |
| <div class="line"><a name="l00345"></a><span class="lineno"> 345</span>  <span class="comment">// had a chance to register with the stream mgr.</span></div> |
| <div class="line"><a name="l00346"></a><span class="lineno"> 346</span>  TExecPlanFragmentParams rpc_params;</div> |
| <div class="line"><a name="l00347"></a><span class="lineno"> 347</span>  <a class="code" href="namespaceimpala.html#ad9d7363b9825635e741e2d45e1d944fa">SetExecPlanFragmentParams</a>(schedule, 0, request.fragments[0], 0,</div> |
| <div class="line"><a name="l00348"></a><span class="lineno"> 348</span>  (*fragment_exec_params)[0], 0, coord, &rpc_params);</div> |
| <div class="line"><a name="l00349"></a><span class="lineno"> 349</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->Prepare(rpc_params));</div> |
| <div class="line"><a name="l00350"></a><span class="lineno"> 350</span> </div> |
| <div class="line"><a name="l00351"></a><span class="lineno"> 351</span>  <span class="comment">// Prepare output_expr_ctxs before optimizing the LLVM module. The other exprs of this</span></div> |
| <div class="line"><a name="l00352"></a><span class="lineno"> 352</span>  <span class="comment">// coordinator fragment have been prepared in executor_->Prepare().</span></div> |
| <div class="line"><a name="l00353"></a><span class="lineno"> 353</span>  DCHECK(output_expr_ctxs != NULL);</div> |
| <div class="line"><a name="l00354"></a><span class="lineno"> 354</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1Expr.html#a6b9c01a432ae49b57482c9fc42a73681">Expr::CreateExprTrees</a>(</div> |
| <div class="line"><a name="l00355"></a><span class="lineno"> 355</span>  <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>()-><a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>(), request.fragments[0].output_exprs,</div> |
| <div class="line"><a name="l00356"></a><span class="lineno"> 356</span>  output_expr_ctxs));</div> |
| <div class="line"><a name="l00357"></a><span class="lineno"> 357</span>  MemTracker* output_expr_tracker = <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>()-><a class="code" href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<span class="keyword">new</span> MemTracker(</div> |
| <div class="line"><a name="l00358"></a><span class="lineno"> 358</span>  -1, -1, <span class="stringliteral">"Output exprs"</span>, <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>()->instance_mem_tracker(), <span class="keyword">false</span>));</div> |
| <div class="line"><a name="l00359"></a><span class="lineno"> 359</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1Expr.html#a3ae02e50debba50ac5c7b6dd9f8016e7">Expr::Prepare</a>(</div> |
| <div class="line"><a name="l00360"></a><span class="lineno"> 360</span>  *output_expr_ctxs, <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>(), <a class="code" href="namespaceimpala.html#aa98447566dd6700a2faaaaf3059f4d95">row_desc</a>(), output_expr_tracker));</div> |
| <div class="line"><a name="l00361"></a><span class="lineno"> 361</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00362"></a><span class="lineno"> 362</span>  <span class="comment">// The coordinator instance may require a query mem tracker even if there is no</span></div> |
| <div class="line"><a name="l00363"></a><span class="lineno"> 363</span>  <span class="comment">// coordinator fragment. For example, result-caching tracks memory via the query mem</span></div> |
| <div class="line"><a name="l00364"></a><span class="lineno"> 364</span>  <span class="comment">// tracker.</span></div> |
| <div class="line"><a name="l00365"></a><span class="lineno"> 365</span>  <span class="comment">// If there is a fragment, the fragment executor created above initializes the query</span></div> |
| <div class="line"><a name="l00366"></a><span class="lineno"> 366</span>  <span class="comment">// mem tracker. If not, the query mem tracker is created here.</span></div> |
| <div class="line"><a name="l00367"></a><span class="lineno"> 367</span>  int64_t query_limit = -1;</div> |
| <div class="line"><a name="l00368"></a><span class="lineno"> 368</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#ab296eb278342117dde26e2e28bbe8190">query_ctx_</a>.request.query_options.__isset.mem_limit &&</div> |
| <div class="line"><a name="l00369"></a><span class="lineno"> 369</span>  <a class="code" href="namespaceimpala.html#ab296eb278342117dde26e2e28bbe8190">query_ctx_</a>.request.query_options.mem_limit > 0) {</div> |
| <div class="line"><a name="l00370"></a><span class="lineno"> 370</span>  query_limit = <a class="code" href="namespaceimpala.html#ab296eb278342117dde26e2e28bbe8190">query_ctx_</a>.request.query_options.mem_limit;</div> |
| <div class="line"><a name="l00371"></a><span class="lineno"> 371</span>  }</div> |
| <div class="line"><a name="l00372"></a><span class="lineno"> 372</span>  MemTracker* pool_tracker = <a class="code" href="classimpala_1_1MemTracker.html#a22fbc3febb516f0f5edd1d40699e6991">MemTracker::GetRequestPoolMemTracker</a>(</div> |
| <div class="line"><a name="l00373"></a><span class="lineno"> 373</span>  schedule.request_pool(), <a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a609be125f8de92848139a7cae6e975bc">process_mem_tracker</a>());</div> |
| <div class="line"><a name="l00374"></a><span class="lineno"> 374</span>  <a class="code" href="namespaceimpala.html#a5219328e34f90ec0679655b8e66bd31f">query_mem_tracker_</a> =</div> |
| <div class="line"><a name="l00375"></a><span class="lineno"> 375</span>  <a class="code" href="classimpala_1_1MemTracker.html#a5b39ede97dce30ef1c7c00b117887a56">MemTracker::GetQueryMemTracker</a>(<a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>, query_limit, -1, pool_tracker, NULL);</div> |
| <div class="line"><a name="l00376"></a><span class="lineno"> 376</span> </div> |
| <div class="line"><a name="l00377"></a><span class="lineno"> 377</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.reset(NULL);</div> |
| <div class="line"><a name="l00378"></a><span class="lineno"> 378</span>  }</div> |
| <div class="line"><a name="l00379"></a><span class="lineno"> 379</span> </div> |
| <div class="line"><a name="l00380"></a><span class="lineno"> 380</span>  <span class="comment">// Initialize the execution profile structures.</span></div> |
| <div class="line"><a name="l00381"></a><span class="lineno"> 381</span>  <a class="code" href="namespaceimpala.html#aecf5ef62c6e2482353b0f41f80528245">InitExecProfile</a>(request);</div> |
| <div class="line"><a name="l00382"></a><span class="lineno"> 382</span> </div> |
| <div class="line"><a name="l00383"></a><span class="lineno"> 383</span>  DebugOptions debug_options;</div> |
| <div class="line"><a name="l00384"></a><span class="lineno"> 384</span>  <a class="code" href="namespaceimpala.html#a7ce4ff5974d4d8b4a5c20b36f061af7a">ProcessQueryOptions</a>(schedule.query_options(), &debug_options);</div> |
| <div class="line"><a name="l00385"></a><span class="lineno"> 385</span> </div> |
| <div class="line"><a name="l00386"></a><span class="lineno"> 386</span>  <span class="comment">// start fragment instances from left to right, so that receivers have</span></div> |
| <div class="line"><a name="l00387"></a><span class="lineno"> 387</span>  <span class="comment">// Prepare()'d before senders start sending</span></div> |
| <div class="line"><a name="l00388"></a><span class="lineno"> 388</span>  <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.resize(schedule.num_backends());</div> |
| <div class="line"><a name="l00389"></a><span class="lineno"> 389</span>  <a class="code" href="namespaceimpala.html#a20991e967dbb80151f6500eb14145943">num_remaining_backends_</a> = schedule.num_backends();</div> |
| <div class="line"><a name="l00390"></a><span class="lineno"> 390</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"starting "</span> << schedule.num_backends()</div> |
| <div class="line"><a name="l00391"></a><span class="lineno"> 391</span>  << <span class="stringliteral">" backends for query "</span> << <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>;</div> |
| <div class="line"><a name="l00392"></a><span class="lineno"> 392</span> </div> |
| <div class="line"><a name="l00393"></a><span class="lineno"> 393</span>  <a class="code" href="namespaceimpala.html#a316415821307d27237bed61e0be1ef0e">query_events_</a>-><a class="code" href="classimpala_1_1RuntimeProfile_1_1EventSequence.html#a7bd1a59efa58c0fbf69f41180f6dbfa1">MarkEvent</a>(<span class="stringliteral">"Ready to start remote fragments"</span>);</div> |
| <div class="line"><a name="l00394"></a><span class="lineno"> 394</span>  <span class="keywordtype">int</span> backend_num = 0;</div> |
| <div class="line"><a name="l00395"></a><span class="lineno"> 395</span>  StatsMetric<double> latencies(<span class="stringliteral">"fragment-latencies"</span>, TUnit::TIME_NS);</div> |
| <div class="line"><a name="l00396"></a><span class="lineno"> 396</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> fragment_idx = (has_coordinator_fragment ? 1 : 0);</div> |
| <div class="line"><a name="l00397"></a><span class="lineno"> 397</span>  fragment_idx < request.fragments.size(); ++fragment_idx) {</div> |
| <div class="line"><a name="l00398"></a><span class="lineno"> 398</span>  <span class="keyword">const</span> FragmentExecParams& params = (*fragment_exec_params)[fragment_idx];</div> |
| <div class="line"><a name="l00399"></a><span class="lineno"> 399</span> </div> |
| <div class="line"><a name="l00400"></a><span class="lineno"> 400</span>  <span class="comment">// set up exec states</span></div> |
| <div class="line"><a name="l00401"></a><span class="lineno"> 401</span>  <span class="keywordtype">int</span> num_hosts = params.hosts.size();</div> |
| <div class="line"><a name="l00402"></a><span class="lineno"> 402</span>  DCHECK_GT(num_hosts, 0);</div> |
| <div class="line"><a name="l00403"></a><span class="lineno"> 403</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> instance_idx = 0; instance_idx < num_hosts; ++instance_idx) {</div> |
| <div class="line"><a name="l00404"></a><span class="lineno"> 404</span>  DebugOptions* backend_debug_options =</div> |
| <div class="line"><a name="l00405"></a><span class="lineno"> 405</span>  (debug_options.phase != TExecNodePhase::INVALID</div> |
| <div class="line"><a name="l00406"></a><span class="lineno"> 406</span>  && (debug_options.backend_num == -1</div> |
| <div class="line"><a name="l00407"></a><span class="lineno"> 407</span>  || debug_options.backend_num == backend_num)</div> |
| <div class="line"><a name="l00408"></a><span class="lineno"> 408</span>  ? &debug_options</div> |
| <div class="line"><a name="l00409"></a><span class="lineno"> 409</span>  : NULL);</div> |
| <div class="line"><a name="l00410"></a><span class="lineno"> 410</span>  <span class="comment">// TODO: pool of pre-formatted BackendExecStates?</span></div> |
| <div class="line"><a name="l00411"></a><span class="lineno"> 411</span>  BackendExecState* exec_state =</div> |
| <div class="line"><a name="l00412"></a><span class="lineno"> 412</span>  <a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<span class="keyword">new</span> BackendExecState(schedule, <span class="keyword">this</span>, coord, backend_num,</div> |
| <div class="line"><a name="l00413"></a><span class="lineno"> 413</span>  request.fragments[fragment_idx], fragment_idx,</div> |
| <div class="line"><a name="l00414"></a><span class="lineno"> 414</span>  params, instance_idx, backend_debug_options, <a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>()));</div> |
| <div class="line"><a name="l00415"></a><span class="lineno"> 415</span>  <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[backend_num] = exec_state;</div> |
| <div class="line"><a name="l00416"></a><span class="lineno"> 416</span>  ++backend_num;</div> |
| <div class="line"><a name="l00417"></a><span class="lineno"> 417</span>  VLOG(2) << <span class="stringliteral">"Exec(): starting instance: fragment_idx="</span> << fragment_idx</div> |
| <div class="line"><a name="l00418"></a><span class="lineno"> 418</span>  << <span class="stringliteral">" instance_id="</span> << params.instance_ids[instance_idx];</div> |
| <div class="line"><a name="l00419"></a><span class="lineno"> 419</span>  }</div> |
| <div class="line"><a name="l00420"></a><span class="lineno"> 420</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[fragment_idx].num_instances = num_hosts;</div> |
| <div class="line"><a name="l00421"></a><span class="lineno"> 421</span> </div> |
| <div class="line"><a name="l00422"></a><span class="lineno"> 422</span>  <span class="comment">// Issue all rpcs in parallel</span></div> |
| <div class="line"><a name="l00423"></a><span class="lineno"> 423</span>  Status fragments_exec_status = <a class="code" href="classimpala_1_1ParallelExecutor.html#abad564e6b67909483e30f0ac1c0f30cc">ParallelExecutor::Exec</a>(</div> |
| <div class="line"><a name="l00424"></a><span class="lineno"> 424</span>  bind<Status>(mem_fn(&<a class="code" href="namespaceimpala.html#a4abf2193b29fb5607bb924bd14dfa347">Coordinator::ExecRemoteFragment</a>), <span class="keyword">this</span>, _1),</div> |
| <div class="line"><a name="l00425"></a><span class="lineno"> 425</span>  reinterpret_cast<void**>(&<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[backend_num - num_hosts]),</div> |
| <div class="line"><a name="l00426"></a><span class="lineno"> 426</span>  num_hosts, &latencies);</div> |
| <div class="line"><a name="l00427"></a><span class="lineno"> 427</span> </div> |
| <div class="line"><a name="l00428"></a><span class="lineno"> 428</span>  <span class="keywordflow">if</span> (!fragments_exec_status.ok()) {</div> |
| <div class="line"><a name="l00429"></a><span class="lineno"> 429</span>  DCHECK(<a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()); <span class="comment">// nobody should have been able to cancel</span></div> |
| <div class="line"><a name="l00430"></a><span class="lineno"> 430</span>  <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a> = fragments_exec_status;</div> |
| <div class="line"><a name="l00431"></a><span class="lineno"> 431</span>  <span class="comment">// tear down running fragments and return</span></div> |
| <div class="line"><a name="l00432"></a><span class="lineno"> 432</span>  <a class="code" href="namespaceimpala.html#a721f135364b8f28d9cf09cd45e95745d">CancelInternal</a>();</div> |
| <div class="line"><a name="l00433"></a><span class="lineno"> 433</span>  <span class="keywordflow">return</span> fragments_exec_status;</div> |
| <div class="line"><a name="l00434"></a><span class="lineno"> 434</span>  }</div> |
| <div class="line"><a name="l00435"></a><span class="lineno"> 435</span>  }</div> |
| <div class="line"><a name="l00436"></a><span class="lineno"> 436</span> </div> |
| <div class="line"><a name="l00437"></a><span class="lineno"> 437</span>  <a class="code" href="namespaceimpala.html#a316415821307d27237bed61e0be1ef0e">query_events_</a>-><a class="code" href="classimpala_1_1RuntimeProfile_1_1EventSequence.html#a7bd1a59efa58c0fbf69f41180f6dbfa1">MarkEvent</a>(<span class="stringliteral">"Remote fragments started"</span>);</div> |
| <div class="line"><a name="l00438"></a><span class="lineno"> 438</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddInfoString(<span class="stringliteral">"Fragment start latencies"</span>,</div> |
| <div class="line"><a name="l00439"></a><span class="lineno"> 439</span>  latencies.ToHumanReadable());</div> |
| <div class="line"><a name="l00440"></a><span class="lineno"> 440</span> </div> |
| <div class="line"><a name="l00441"></a><span class="lineno"> 441</span>  <span class="comment">// If we have a coordinator fragment and remote fragments (the common case),</span></div> |
| <div class="line"><a name="l00442"></a><span class="lineno"> 442</span>  <span class="comment">// release the thread token on the coordinator fragment. This fragment</span></div> |
| <div class="line"><a name="l00443"></a><span class="lineno"> 443</span>  <span class="comment">// spends most of the time waiting and doing very little work. Holding on to</span></div> |
| <div class="line"><a name="l00444"></a><span class="lineno"> 444</span>  <span class="comment">// the token causes underutilization of the machine. If there are 12 queries</span></div> |
| <div class="line"><a name="l00445"></a><span class="lineno"> 445</span>  <span class="comment">// on this node, that's 12 tokens reserved for no reason.</span></div> |
| <div class="line"><a name="l00446"></a><span class="lineno"> 446</span>  <span class="keywordflow">if</span> (has_coordinator_fragment && request.fragments.size() > 1) {</div> |
| <div class="line"><a name="l00447"></a><span class="lineno"> 447</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->ReleaseThreadToken();</div> |
| <div class="line"><a name="l00448"></a><span class="lineno"> 448</span>  }</div> |
| <div class="line"><a name="l00449"></a><span class="lineno"> 449</span> </div> |
| <div class="line"><a name="l00450"></a><span class="lineno"> 450</span>  <a class="code" href="namespaceimpala.html#a7dff4fbd53ca26eb055bc173471c7be3">PrintBackendInfo</a>();</div> |
| <div class="line"><a name="l00451"></a><span class="lineno"> 451</span> </div> |
| <div class="line"><a name="l00452"></a><span class="lineno"> 452</span>  stringstream ss;</div> |
| <div class="line"><a name="l00453"></a><span class="lineno"> 453</span>  ss << <span class="stringliteral">"Query "</span> << <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>;</div> |
| <div class="line"><a name="l00454"></a><span class="lineno"> 454</span>  <a class="code" href="namespaceimpala.html#ab3b57d53bc4b6adfed05b760c72e0faf">progress_</a> = ProgressUpdater(ss.str(), schedule.num_scan_ranges());</div> |
| <div class="line"><a name="l00455"></a><span class="lineno"> 455</span> </div> |
| <div class="line"><a name="l00456"></a><span class="lineno"> 456</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00457"></a><span class="lineno"> 457</span> }</div> |
| <div class="line"><a name="l00458"></a><span class="lineno"> 458</span> </div> |
| <div class="line"><a name="l00459"></a><span class="lineno"> 459</span> Status <a class="code" href="namespaceimpala.html#a5d05598507a4b64b2288e3522cf38c58">Coordinator::GetStatus</a>() {</div> |
| <div class="line"><a name="l00460"></a><span class="lineno"> 460</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l00461"></a><span class="lineno"> 461</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>;</div> |
| <div class="line"><a name="l00462"></a><span class="lineno"> 462</span> }</div> |
| <div class="line"><a name="l00463"></a><span class="lineno"> 463</span> </div> |
| <div class="line"><a name="l00464"></a><span class="lineno"> 464</span> Status <a class="code" href="namespaceimpala.html#aba23271e9c88b75efea05a2afb25856b">Coordinator::UpdateStatus</a>(<span class="keyword">const</span> Status& status, <span class="keyword">const</span> TUniqueId* instance_id) {</div> |
| <div class="line"><a name="l00465"></a><span class="lineno"> 465</span>  {</div> |
| <div class="line"><a name="l00466"></a><span class="lineno"> 466</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l00467"></a><span class="lineno"> 467</span> </div> |
| <div class="line"><a name="l00468"></a><span class="lineno"> 468</span>  <span class="comment">// The query is done and we are just waiting for remote fragments to clean up.</span></div> |
| <div class="line"><a name="l00469"></a><span class="lineno"> 469</span>  <span class="comment">// Ignore their cancelled updates.</span></div> |
| <div class="line"><a name="l00470"></a><span class="lineno"> 470</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#aea293e10bddb1360a019cd1e9abf45f2">returned_all_results_</a> && status.IsCancelled()) <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>;</div> |
| <div class="line"><a name="l00471"></a><span class="lineno"> 471</span> </div> |
| <div class="line"><a name="l00472"></a><span class="lineno"> 472</span>  <span class="comment">// nothing to update</span></div> |
| <div class="line"><a name="l00473"></a><span class="lineno"> 473</span>  <span class="keywordflow">if</span> (status.ok()) <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>;</div> |
| <div class="line"><a name="l00474"></a><span class="lineno"> 474</span> </div> |
| <div class="line"><a name="l00475"></a><span class="lineno"> 475</span>  <span class="comment">// don't override an error status; also, cancellation has already started</span></div> |
| <div class="line"><a name="l00476"></a><span class="lineno"> 476</span>  <span class="keywordflow">if</span> (!<a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>;</div> |
| <div class="line"><a name="l00477"></a><span class="lineno"> 477</span> </div> |
| <div class="line"><a name="l00478"></a><span class="lineno"> 478</span>  <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a> = status;</div> |
| <div class="line"><a name="l00479"></a><span class="lineno"> 479</span>  <a class="code" href="namespaceimpala.html#a721f135364b8f28d9cf09cd45e95745d">CancelInternal</a>();</div> |
| <div class="line"><a name="l00480"></a><span class="lineno"> 480</span>  }</div> |
| <div class="line"><a name="l00481"></a><span class="lineno"> 481</span> </div> |
| <div class="line"><a name="l00482"></a><span class="lineno"> 482</span>  <span class="comment">// Log the id of the fragment that first failed so we can track it down easier.</span></div> |
| <div class="line"><a name="l00483"></a><span class="lineno"> 483</span>  <span class="keywordflow">if</span> (instance_id != NULL) {</div> |
| <div class="line"><a name="l00484"></a><span class="lineno"> 484</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Query id="</span> << query_id_ << <span class="stringliteral">" failed because fragment id="</span></div> |
| <div class="line"><a name="l00485"></a><span class="lineno"> 485</span>  << *instance_id << <span class="stringliteral">" failed."</span>;</div> |
| <div class="line"><a name="l00486"></a><span class="lineno"> 486</span>  }</div> |
| <div class="line"><a name="l00487"></a><span class="lineno"> 487</span> </div> |
| <div class="line"><a name="l00488"></a><span class="lineno"> 488</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>;</div> |
| <div class="line"><a name="l00489"></a><span class="lineno"> 489</span> }</div> |
| <div class="line"><a name="l00490"></a><span class="lineno"> 490</span> </div> |
| <div class="line"><a name="l00491"></a><span class="lineno"> 491</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#aa62513ecce3dc2a2c7fc202d31a9a1f2">Coordinator::PopulatePathPermissionCache</a>(hdfsFS fs, <span class="keyword">const</span> <span class="keywordtype">string</span>& path_str,</div> |
| <div class="line"><a name="l00492"></a><span class="lineno"> 492</span>  <a class="code" href="namespaceimpala.html#a72ec1818c46664b276a7a29ed73d4fdb">PermissionCache</a>* permissions_cache) {</div> |
| <div class="line"><a name="l00493"></a><span class="lineno"> 493</span>  <span class="comment">// Find out if the path begins with a hdfs:// -style prefix, and remove it and the</span></div> |
| <div class="line"><a name="l00494"></a><span class="lineno"> 494</span>  <span class="comment">// location (e.g. host:port) if so.</span></div> |
| <div class="line"><a name="l00495"></a><span class="lineno"> 495</span>  <span class="keywordtype">int</span> scheme_end = path_str.find(<span class="stringliteral">"://"</span>);</div> |
| <div class="line"><a name="l00496"></a><span class="lineno"> 496</span>  <span class="keywordtype">string</span> stripped_str;</div> |
| <div class="line"><a name="l00497"></a><span class="lineno"> 497</span>  <span class="keywordflow">if</span> (scheme_end != string::npos) {</div> |
| <div class="line"><a name="l00498"></a><span class="lineno"> 498</span>  <span class="comment">// Skip past the subsequent location:port/ prefix.</span></div> |
| <div class="line"><a name="l00499"></a><span class="lineno"> 499</span>  stripped_str = path_str.substr(path_str.find(<span class="stringliteral">"/"</span>, scheme_end + 3));</div> |
| <div class="line"><a name="l00500"></a><span class="lineno"> 500</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00501"></a><span class="lineno"> 501</span>  stripped_str = path_str;</div> |
| <div class="line"><a name="l00502"></a><span class="lineno"> 502</span>  }</div> |
| <div class="line"><a name="l00503"></a><span class="lineno"> 503</span> </div> |
| <div class="line"><a name="l00504"></a><span class="lineno"> 504</span>  <span class="comment">// Get the list of path components, used to build all path prefixes.</span></div> |
| <div class="line"><a name="l00505"></a><span class="lineno"> 505</span>  vector<string> components;</div> |
| <div class="line"><a name="l00506"></a><span class="lineno"> 506</span>  split(components, stripped_str, is_any_of(<span class="stringliteral">"/"</span>));</div> |
| <div class="line"><a name="l00507"></a><span class="lineno"> 507</span> </div> |
| <div class="line"><a name="l00508"></a><span class="lineno"> 508</span>  <span class="comment">// Build a set of all prefixes (including the complete string) of stripped_path. So</span></div> |
| <div class="line"><a name="l00509"></a><span class="lineno"> 509</span>  <span class="comment">// /a/b/c/d leads to a vector of: /a, /a/b, /a/b/c, /a/b/c/d</span></div> |
| <div class="line"><a name="l00510"></a><span class="lineno"> 510</span>  vector<string> prefixes;</div> |
| <div class="line"><a name="l00511"></a><span class="lineno"> 511</span>  <span class="comment">// Stores the current prefix</span></div> |
| <div class="line"><a name="l00512"></a><span class="lineno"> 512</span>  stringstream accumulator;</div> |
| <div class="line"><a name="l00513"></a><span class="lineno"> 513</span>  BOOST_FOREACH(<span class="keyword">const</span> <span class="keywordtype">string</span>& component, components) {</div> |
| <div class="line"><a name="l00514"></a><span class="lineno"> 514</span>  <span class="keywordflow">if</span> (component.empty()) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00515"></a><span class="lineno"> 515</span>  accumulator << <span class="stringliteral">"/"</span> << component;</div> |
| <div class="line"><a name="l00516"></a><span class="lineno"> 516</span>  prefixes.push_back(accumulator.str());</div> |
| <div class="line"><a name="l00517"></a><span class="lineno"> 517</span>  }</div> |
| <div class="line"><a name="l00518"></a><span class="lineno"> 518</span> </div> |
| <div class="line"><a name="l00519"></a><span class="lineno"> 519</span>  <span class="comment">// Now for each prefix, stat() it to see if a) it exists and b) if so what its</span></div> |
| <div class="line"><a name="l00520"></a><span class="lineno"> 520</span>  <span class="comment">// permissions are. When we meet a directory that doesn't exist, we record the fact that</span></div> |
| <div class="line"><a name="l00521"></a><span class="lineno"> 521</span>  <span class="comment">// we need to create it, and the permissions of its parent dir to inherit.</span></div> |
| <div class="line"><a name="l00522"></a><span class="lineno"> 522</span>  <span class="comment">//</span></div> |
| <div class="line"><a name="l00523"></a><span class="lineno"> 523</span>  <span class="comment">// Every prefix is recorded in the PermissionCache so we don't do more than one stat()</span></div> |
| <div class="line"><a name="l00524"></a><span class="lineno"> 524</span>  <span class="comment">// for each path. If we need to create the directory, we record it as the pair (true,</span></div> |
| <div class="line"><a name="l00525"></a><span class="lineno"> 525</span>  <span class="comment">// perms) so that the caller can identify which directories need their permissions</span></div> |
| <div class="line"><a name="l00526"></a><span class="lineno"> 526</span>  <span class="comment">// explicitly set.</span></div> |
| <div class="line"><a name="l00527"></a><span class="lineno"> 527</span> </div> |
| <div class="line"><a name="l00528"></a><span class="lineno"> 528</span>  <span class="comment">// Set to the permission of the immediate parent (i.e. the permissions to inherit if the</span></div> |
| <div class="line"><a name="l00529"></a><span class="lineno"> 529</span>  <span class="comment">// current dir doesn't exist).</span></div> |
| <div class="line"><a name="l00530"></a><span class="lineno"> 530</span>  <span class="keywordtype">short</span> permissions = 0;</div> |
| <div class="line"><a name="l00531"></a><span class="lineno"> 531</span>  BOOST_FOREACH(<span class="keyword">const</span> <span class="keywordtype">string</span>& <a class="code" href="sasl-test_8cc.html#a2db66f7b463c0e190f8933e51f43d91e">path</a>, prefixes) {</div> |
| <div class="line"><a name="l00532"></a><span class="lineno"> 532</span>  PermissionCache::const_iterator it = permissions_cache->find(path);</div> |
| <div class="line"><a name="l00533"></a><span class="lineno"> 533</span>  <span class="keywordflow">if</span> (it == permissions_cache->end()) {</div> |
| <div class="line"><a name="l00534"></a><span class="lineno"> 534</span>  hdfsFileInfo* info = hdfsGetPathInfo(fs, path.c_str());</div> |
| <div class="line"><a name="l00535"></a><span class="lineno"> 535</span>  <span class="keywordflow">if</span> (info != NULL) {</div> |
| <div class="line"><a name="l00536"></a><span class="lineno"> 536</span>  <span class="comment">// File exists, so fill the cache with its current permissions.</span></div> |
| <div class="line"><a name="l00537"></a><span class="lineno"> 537</span>  permissions_cache->insert(</div> |
| <div class="line"><a name="l00538"></a><span class="lineno"> 538</span>  make_pair(path, make_pair(<span class="keyword">false</span>, info->mPermissions)));</div> |
| <div class="line"><a name="l00539"></a><span class="lineno"> 539</span>  permissions = info->mPermissions;</div> |
| <div class="line"><a name="l00540"></a><span class="lineno"> 540</span>  hdfsFreeFileInfo(info, 1);</div> |
| <div class="line"><a name="l00541"></a><span class="lineno"> 541</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00542"></a><span class="lineno"> 542</span>  <span class="comment">// File doesn't exist, so we need to set its permissions to its immediate parent</span></div> |
| <div class="line"><a name="l00543"></a><span class="lineno"> 543</span>  <span class="comment">// once it's been created.</span></div> |
| <div class="line"><a name="l00544"></a><span class="lineno"> 544</span>  permissions_cache->insert(make_pair(path, make_pair(<span class="keyword">true</span>, permissions)));</div> |
| <div class="line"><a name="l00545"></a><span class="lineno"> 545</span>  }</div> |
| <div class="line"><a name="l00546"></a><span class="lineno"> 546</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00547"></a><span class="lineno"> 547</span>  permissions = it->second.second;</div> |
| <div class="line"><a name="l00548"></a><span class="lineno"> 548</span>  }</div> |
| <div class="line"><a name="l00549"></a><span class="lineno"> 549</span>  }</div> |
| <div class="line"><a name="l00550"></a><span class="lineno"> 550</span> }</div> |
| <div class="line"><a name="l00551"></a><span class="lineno"> 551</span> </div> |
| <div class="line"><a name="l00552"></a><span class="lineno"> 552</span> Status <a class="code" href="namespaceimpala.html#aaedda9a1b006ac22f15b591ce15f0281">Coordinator::FinalizeSuccessfulInsert</a>() {</div> |
| <div class="line"><a name="l00553"></a><span class="lineno"> 553</span>  <a class="code" href="namespaceimpala.html#a72ec1818c46664b276a7a29ed73d4fdb">PermissionCache</a> permissions_cache;</div> |
| <div class="line"><a name="l00554"></a><span class="lineno"> 554</span>  hdfsFS hdfs_connection;</div> |
| <div class="line"><a name="l00555"></a><span class="lineno"> 555</span>  <span class="comment">// InsertStmt ensures that all partitions are on the same filesystem as the table's</span></div> |
| <div class="line"><a name="l00556"></a><span class="lineno"> 556</span>  <span class="comment">// base directory, so opening a single connection is okay.</span></div> |
| <div class="line"><a name="l00557"></a><span class="lineno"> 557</span>  <span class="comment">// TODO: modify this code so that restriction can be lifted.</span></div> |
| <div class="line"><a name="l00558"></a><span class="lineno"> 558</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsFsCache.html#af7b18daf8c4da6bfce2c7612a515bf64">HdfsFsCache::instance</a>()->GetConnection(</div> |
| <div class="line"><a name="l00559"></a><span class="lineno"> 559</span>  <a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.hdfs_base_dir, &hdfs_connection));</div> |
| <div class="line"><a name="l00560"></a><span class="lineno"> 560</span> </div> |
| <div class="line"><a name="l00561"></a><span class="lineno"> 561</span>  <span class="comment">// INSERT finalization happens in the five following steps</span></div> |
| <div class="line"><a name="l00562"></a><span class="lineno"> 562</span>  <span class="comment">// 1. If OVERWRITE, remove all the files in the target directory</span></div> |
| <div class="line"><a name="l00563"></a><span class="lineno"> 563</span>  <span class="comment">// 2. Create all the necessary partition directories.</span></div> |
| <div class="line"><a name="l00564"></a><span class="lineno"> 564</span>  HdfsOperationSet partition_create_ops(&hdfs_connection);</div> |
| <div class="line"><a name="l00565"></a><span class="lineno"> 565</span>  DescriptorTbl* descriptor_table;</div> |
| <div class="line"><a name="l00566"></a><span class="lineno"> 566</span>  <a class="code" href="classimpala_1_1DescriptorTbl.html#a7eec9a1b9f48220a92e3bdd4fdba44ba">DescriptorTbl::Create</a>(<a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>(), <a class="code" href="namespaceimpala.html#a0a06145acd27005a1c1f482dc2375742">desc_tbl_</a>, &descriptor_table);</div> |
| <div class="line"><a name="l00567"></a><span class="lineno"> 567</span>  HdfsTableDescriptor* hdfs_table = <span class="keyword">static_cast<</span>HdfsTableDescriptor*<span class="keyword">></span>(</div> |
| <div class="line"><a name="l00568"></a><span class="lineno"> 568</span>  descriptor_table->GetTableDescriptor(<a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.table_id));</div> |
| <div class="line"><a name="l00569"></a><span class="lineno"> 569</span>  DCHECK(hdfs_table != NULL) << <span class="stringliteral">"INSERT target table not known in descriptor table: "</span></div> |
| <div class="line"><a name="l00570"></a><span class="lineno"> 570</span>  << <a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.table_id;</div> |
| <div class="line"><a name="l00571"></a><span class="lineno"> 571</span> </div> |
| <div class="line"><a name="l00572"></a><span class="lineno"> 572</span>  <span class="comment">// Loop over all partitions that were updated by this insert, and create the set of</span></div> |
| <div class="line"><a name="l00573"></a><span class="lineno"> 573</span>  <span class="comment">// filesystem operations required to create the correct partition structure on disk.</span></div> |
| <div class="line"><a name="l00574"></a><span class="lineno"> 574</span>  BOOST_FOREACH(<span class="keyword">const</span> PartitionStatusMap::value_type& partition, <a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a>) {</div> |
| <div class="line"><a name="l00575"></a><span class="lineno"> 575</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="runtime-profile_8h.html#a8d817beaf6c704ff590230236d7591b1">ADD_CHILD_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>, <span class="stringliteral">"Overwrite/PartitionCreationTimer"</span>,</div> |
| <div class="line"><a name="l00576"></a><span class="lineno"> 576</span>  <span class="stringliteral">"FinalizationTimer"</span>));</div> |
| <div class="line"><a name="l00577"></a><span class="lineno"> 577</span> </div> |
| <div class="line"><a name="l00578"></a><span class="lineno"> 578</span>  <span class="comment">// Look up the partition in the descriptor table.</span></div> |
| <div class="line"><a name="l00579"></a><span class="lineno"> 579</span>  stringstream part_path_ss;</div> |
| <div class="line"><a name="l00580"></a><span class="lineno"> 580</span>  <span class="keywordflow">if</span> (partition.second.id == -1) {</div> |
| <div class="line"><a name="l00581"></a><span class="lineno"> 581</span>  <span class="comment">// If this is a non-existant partition, use the default partition location of</span></div> |
| <div class="line"><a name="l00582"></a><span class="lineno"> 582</span>  <span class="comment">// <base_dir>/part_key_1=val/part_key_2=val/...</span></div> |
| <div class="line"><a name="l00583"></a><span class="lineno"> 583</span>  part_path_ss << <a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.hdfs_base_dir << <span class="stringliteral">"/"</span> << partition.first;</div> |
| <div class="line"><a name="l00584"></a><span class="lineno"> 584</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00585"></a><span class="lineno"> 585</span>  HdfsPartitionDescriptor* part = hdfs_table->GetPartition(partition.second.id);</div> |
| <div class="line"><a name="l00586"></a><span class="lineno"> 586</span>  DCHECK(part != NULL) << <span class="stringliteral">"Partition "</span> << partition.second.id</div> |
| <div class="line"><a name="l00587"></a><span class="lineno"> 587</span>  << <span class="stringliteral">" not known in descriptor table"</span>;</div> |
| <div class="line"><a name="l00588"></a><span class="lineno"> 588</span>  part_path_ss << part->location();</div> |
| <div class="line"><a name="l00589"></a><span class="lineno"> 589</span>  }</div> |
| <div class="line"><a name="l00590"></a><span class="lineno"> 590</span>  <span class="keyword">const</span> <span class="keywordtype">string</span>& part_path = part_path_ss.str();</div> |
| <div class="line"><a name="l00591"></a><span class="lineno"> 591</span> </div> |
| <div class="line"><a name="l00592"></a><span class="lineno"> 592</span>  <span class="comment">// If this is an overwrite insert, we will need to delete any updated partitions</span></div> |
| <div class="line"><a name="l00593"></a><span class="lineno"> 593</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.is_overwrite) {</div> |
| <div class="line"><a name="l00594"></a><span class="lineno"> 594</span>  <span class="keywordflow">if</span> (partition.first.empty()) {</div> |
| <div class="line"><a name="l00595"></a><span class="lineno"> 595</span>  <span class="comment">// If the root directory is written to, then the table must not be partitioned</span></div> |
| <div class="line"><a name="l00596"></a><span class="lineno"> 596</span>  DCHECK(<a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a>.size() == 1);</div> |
| <div class="line"><a name="l00597"></a><span class="lineno"> 597</span>  <span class="comment">// We need to be a little more careful, and only delete data files in the root</span></div> |
| <div class="line"><a name="l00598"></a><span class="lineno"> 598</span>  <span class="comment">// because the tmp directories the sink(s) wrote are there also.</span></div> |
| <div class="line"><a name="l00599"></a><span class="lineno"> 599</span>  <span class="comment">// So only delete files in the table directory - all files are treated as data</span></div> |
| <div class="line"><a name="l00600"></a><span class="lineno"> 600</span>  <span class="comment">// files by Hive and Impala, but directories are ignored (and may legitimately</span></div> |
| <div class="line"><a name="l00601"></a><span class="lineno"> 601</span>  <span class="comment">// be used to store permanent non-table data by other applications).</span></div> |
| <div class="line"><a name="l00602"></a><span class="lineno"> 602</span>  <span class="keywordtype">int</span> num_files = 0;</div> |
| <div class="line"><a name="l00603"></a><span class="lineno"> 603</span>  hdfsFileInfo* existing_files =</div> |
| <div class="line"><a name="l00604"></a><span class="lineno"> 604</span>  hdfsListDirectory(hdfs_connection, part_path.c_str(), &num_files);</div> |
| <div class="line"><a name="l00605"></a><span class="lineno"> 605</span>  <span class="keywordflow">if</span> (existing_files == NULL) {</div> |
| <div class="line"><a name="l00606"></a><span class="lineno"> 606</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">GetHdfsErrorMsg</a>(<span class="stringliteral">"Could not list directory: "</span>, part_path);</div> |
| <div class="line"><a name="l00607"></a><span class="lineno"> 607</span>  }</div> |
| <div class="line"><a name="l00608"></a><span class="lineno"> 608</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < num_files; ++i) {</div> |
| <div class="line"><a name="l00609"></a><span class="lineno"> 609</span>  <span class="keyword">const</span> <span class="keywordtype">string</span> filename = <a class="code" href="sasl-test_8cc.html#a2db66f7b463c0e190f8933e51f43d91e">path</a>(existing_files[i].mName).filename().string();</div> |
| <div class="line"><a name="l00610"></a><span class="lineno"> 610</span>  <span class="keywordflow">if</span> (existing_files[i].mKind == kObjectKindFile && !<a class="code" href="namespaceimpala.html#ae11a4460de7695cbe1bcd94d8286df5c">IsHiddenFile</a>(filename)) {</div> |
| <div class="line"><a name="l00611"></a><span class="lineno"> 611</span>  partition_create_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa0331700e2301dedaf6a902d078218682">DELETE</a>, existing_files[i].mName);</div> |
| <div class="line"><a name="l00612"></a><span class="lineno"> 612</span>  }</div> |
| <div class="line"><a name="l00613"></a><span class="lineno"> 613</span>  }</div> |
| <div class="line"><a name="l00614"></a><span class="lineno"> 614</span>  hdfsFreeFileInfo(existing_files, num_files);</div> |
| <div class="line"><a name="l00615"></a><span class="lineno"> 615</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00616"></a><span class="lineno"> 616</span>  <span class="comment">// This is a partition directory, not the root directory; we can delete</span></div> |
| <div class="line"><a name="l00617"></a><span class="lineno"> 617</span>  <span class="comment">// recursively with abandon, after checking that it ever existed.</span></div> |
| <div class="line"><a name="l00618"></a><span class="lineno"> 618</span>  <span class="comment">// TODO: There's a potential race here between checking for the directory</span></div> |
| <div class="line"><a name="l00619"></a><span class="lineno"> 619</span>  <span class="comment">// and a third-party deleting it.</span></div> |
| <div class="line"><a name="l00620"></a><span class="lineno"> 620</span>  <span class="keywordflow">if</span> (FLAGS_insert_inherit_permissions) {</div> |
| <div class="line"><a name="l00621"></a><span class="lineno"> 621</span>  <a class="code" href="namespaceimpala.html#aa62513ecce3dc2a2c7fc202d31a9a1f2">PopulatePathPermissionCache</a>(hdfs_connection, part_path, &permissions_cache);</div> |
| <div class="line"><a name="l00622"></a><span class="lineno"> 622</span>  }</div> |
| <div class="line"><a name="l00623"></a><span class="lineno"> 623</span>  <span class="keywordflow">if</span> (hdfsExists(hdfs_connection, part_path.c_str()) != -1) {</div> |
| <div class="line"><a name="l00624"></a><span class="lineno"> 624</span>  partition_create_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa8dc9bcd2611acd50486660a42f7feec1">DELETE_THEN_CREATE</a>, part_path);</div> |
| <div class="line"><a name="l00625"></a><span class="lineno"> 625</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00626"></a><span class="lineno"> 626</span>  <span class="comment">// Otherwise just create the directory.</span></div> |
| <div class="line"><a name="l00627"></a><span class="lineno"> 627</span>  partition_create_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa5fb0cb0f52f73f240132738b9019cd5d">CREATE_DIR</a>, part_path);</div> |
| <div class="line"><a name="l00628"></a><span class="lineno"> 628</span>  }</div> |
| <div class="line"><a name="l00629"></a><span class="lineno"> 629</span>  }</div> |
| <div class="line"><a name="l00630"></a><span class="lineno"> 630</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00631"></a><span class="lineno"> 631</span>  <span class="keywordflow">if</span> (FLAGS_insert_inherit_permissions) {</div> |
| <div class="line"><a name="l00632"></a><span class="lineno"> 632</span>  <a class="code" href="namespaceimpala.html#aa62513ecce3dc2a2c7fc202d31a9a1f2">PopulatePathPermissionCache</a>(hdfs_connection, part_path, &permissions_cache);</div> |
| <div class="line"><a name="l00633"></a><span class="lineno"> 633</span>  }</div> |
| <div class="line"><a name="l00634"></a><span class="lineno"> 634</span>  <span class="keywordflow">if</span> (hdfsExists(hdfs_connection, part_path.c_str()) == -1) {</div> |
| <div class="line"><a name="l00635"></a><span class="lineno"> 635</span>  partition_create_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa5fb0cb0f52f73f240132738b9019cd5d">CREATE_DIR</a>, part_path);</div> |
| <div class="line"><a name="l00636"></a><span class="lineno"> 636</span>  }</div> |
| <div class="line"><a name="l00637"></a><span class="lineno"> 637</span>  }</div> |
| <div class="line"><a name="l00638"></a><span class="lineno"> 638</span>  }</div> |
| <div class="line"><a name="l00639"></a><span class="lineno"> 639</span> </div> |
| <div class="line"><a name="l00640"></a><span class="lineno"> 640</span>  {</div> |
| <div class="line"><a name="l00641"></a><span class="lineno"> 641</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="runtime-profile_8h.html#a8d817beaf6c704ff590230236d7591b1">ADD_CHILD_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>, <span class="stringliteral">"Overwrite/PartitionCreationTimer"</span>,</div> |
| <div class="line"><a name="l00642"></a><span class="lineno"> 642</span>  <span class="stringliteral">"FinalizationTimer"</span>));</div> |
| <div class="line"><a name="l00643"></a><span class="lineno"> 643</span>  <span class="keywordflow">if</span> (!partition_create_ops.Execute(<a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a1aa344745ea152e6df02c4c4f7693aab">hdfs_op_thread_pool</a>(), <span class="keyword">false</span>)) {</div> |
| <div class="line"><a name="l00644"></a><span class="lineno"> 644</span>  BOOST_FOREACH(<span class="keyword">const</span> <a class="code" href="classimpala_1_1HdfsOperationSet.html#ae5b0a59660945825eb1647236e76b537">HdfsOperationSet::Error</a>& err, partition_create_ops.errors()) {</div> |
| <div class="line"><a name="l00645"></a><span class="lineno"> 645</span>  <span class="comment">// It's ok to ignore errors creating the directories, since they may already</span></div> |
| <div class="line"><a name="l00646"></a><span class="lineno"> 646</span>  <span class="comment">// exist. If there are permission errors, we'll run into them later.</span></div> |
| <div class="line"><a name="l00647"></a><span class="lineno"> 647</span>  <span class="keywordflow">if</span> (err.first->op() != <a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa5fb0cb0f52f73f240132738b9019cd5d">CREATE_DIR</a>) {</div> |
| <div class="line"><a name="l00648"></a><span class="lineno"> 648</span>  stringstream ss;</div> |
| <div class="line"><a name="l00649"></a><span class="lineno"> 649</span>  ss << <span class="stringliteral">"Error(s) deleting partition directories. First error (of "</span></div> |
| <div class="line"><a name="l00650"></a><span class="lineno"> 650</span>  << partition_create_ops.errors().size() << <span class="stringliteral">") was: "</span> << err.second;</div> |
| <div class="line"><a name="l00651"></a><span class="lineno"> 651</span>  <span class="keywordflow">return</span> Status(ss.str());</div> |
| <div class="line"><a name="l00652"></a><span class="lineno"> 652</span>  }</div> |
| <div class="line"><a name="l00653"></a><span class="lineno"> 653</span>  }</div> |
| <div class="line"><a name="l00654"></a><span class="lineno"> 654</span>  }</div> |
| <div class="line"><a name="l00655"></a><span class="lineno"> 655</span>  }</div> |
| <div class="line"><a name="l00656"></a><span class="lineno"> 656</span> </div> |
| <div class="line"><a name="l00657"></a><span class="lineno"> 657</span>  <span class="comment">// 3. Move all tmp files</span></div> |
| <div class="line"><a name="l00658"></a><span class="lineno"> 658</span>  HdfsOperationSet move_ops(&hdfs_connection);</div> |
| <div class="line"><a name="l00659"></a><span class="lineno"> 659</span>  HdfsOperationSet dir_deletion_ops(&hdfs_connection);</div> |
| <div class="line"><a name="l00660"></a><span class="lineno"> 660</span> </div> |
| <div class="line"><a name="l00661"></a><span class="lineno"> 661</span>  BOOST_FOREACH(FileMoveMap::value_type& move, <a class="code" href="namespaceimpala.html#ab1a9829c4d817be471e86d26f1f316b8">files_to_move_</a>) {</div> |
| <div class="line"><a name="l00662"></a><span class="lineno"> 662</span>  <span class="comment">// Empty destination means delete, so this is a directory. These get deleted in a</span></div> |
| <div class="line"><a name="l00663"></a><span class="lineno"> 663</span>  <span class="comment">// separate pass to ensure that we have moved all the contents of the directory first.</span></div> |
| <div class="line"><a name="l00664"></a><span class="lineno"> 664</span>  <span class="keywordflow">if</span> (move.second.empty()) {</div> |
| <div class="line"><a name="l00665"></a><span class="lineno"> 665</span>  <a class="code" href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a> << <span class="stringliteral">"Deleting file: "</span> << move.first;</div> |
| <div class="line"><a name="l00666"></a><span class="lineno"> 666</span>  dir_deletion_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa0331700e2301dedaf6a902d078218682">DELETE</a>, move.first);</div> |
| <div class="line"><a name="l00667"></a><span class="lineno"> 667</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00668"></a><span class="lineno"> 668</span>  <a class="code" href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a> << <span class="stringliteral">"Moving tmp file: "</span> << move.first << <span class="stringliteral">" to "</span> << move.second;</div> |
| <div class="line"><a name="l00669"></a><span class="lineno"> 669</span>  move_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fad29ef8d6894805fad47239ec0e5bec7b">RENAME</a>, move.first, move.second);</div> |
| <div class="line"><a name="l00670"></a><span class="lineno"> 670</span>  }</div> |
| <div class="line"><a name="l00671"></a><span class="lineno"> 671</span>  }</div> |
| <div class="line"><a name="l00672"></a><span class="lineno"> 672</span> </div> |
| <div class="line"><a name="l00673"></a><span class="lineno"> 673</span>  {</div> |
| <div class="line"><a name="l00674"></a><span class="lineno"> 674</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="runtime-profile_8h.html#a8d817beaf6c704ff590230236d7591b1">ADD_CHILD_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>, <span class="stringliteral">"FileMoveTimer"</span>, <span class="stringliteral">"FinalizationTimer"</span>));</div> |
| <div class="line"><a name="l00675"></a><span class="lineno"> 675</span>  <span class="keywordflow">if</span> (!move_ops.Execute(<a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a1aa344745ea152e6df02c4c4f7693aab">hdfs_op_thread_pool</a>(), <span class="keyword">false</span>)) {</div> |
| <div class="line"><a name="l00676"></a><span class="lineno"> 676</span>  stringstream ss;</div> |
| <div class="line"><a name="l00677"></a><span class="lineno"> 677</span>  ss << <span class="stringliteral">"Error(s) moving partition files. First error (of "</span></div> |
| <div class="line"><a name="l00678"></a><span class="lineno"> 678</span>  << move_ops.errors().size() << <span class="stringliteral">") was: "</span> << move_ops.errors()[0].second;</div> |
| <div class="line"><a name="l00679"></a><span class="lineno"> 679</span>  <span class="keywordflow">return</span> Status(ss.str());</div> |
| <div class="line"><a name="l00680"></a><span class="lineno"> 680</span>  }</div> |
| <div class="line"><a name="l00681"></a><span class="lineno"> 681</span>  }</div> |
| <div class="line"><a name="l00682"></a><span class="lineno"> 682</span> </div> |
| <div class="line"><a name="l00683"></a><span class="lineno"> 683</span>  <span class="comment">// 4. Delete temp directories</span></div> |
| <div class="line"><a name="l00684"></a><span class="lineno"> 684</span>  {</div> |
| <div class="line"><a name="l00685"></a><span class="lineno"> 685</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="runtime-profile_8h.html#a8d817beaf6c704ff590230236d7591b1">ADD_CHILD_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>, <span class="stringliteral">"FileDeletionTimer"</span>,</div> |
| <div class="line"><a name="l00686"></a><span class="lineno"> 686</span>  <span class="stringliteral">"FinalizationTimer"</span>));</div> |
| <div class="line"><a name="l00687"></a><span class="lineno"> 687</span>  <span class="keywordflow">if</span> (!dir_deletion_ops.Execute(<a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a1aa344745ea152e6df02c4c4f7693aab">hdfs_op_thread_pool</a>(), <span class="keyword">false</span>)) {</div> |
| <div class="line"><a name="l00688"></a><span class="lineno"> 688</span>  stringstream ss;</div> |
| <div class="line"><a name="l00689"></a><span class="lineno"> 689</span>  ss << <span class="stringliteral">"Error(s) deleting staging directories. First error (of "</span></div> |
| <div class="line"><a name="l00690"></a><span class="lineno"> 690</span>  << dir_deletion_ops.errors().size() << <span class="stringliteral">") was: "</span></div> |
| <div class="line"><a name="l00691"></a><span class="lineno"> 691</span>  << dir_deletion_ops.errors()[0].second;</div> |
| <div class="line"><a name="l00692"></a><span class="lineno"> 692</span>  <span class="keywordflow">return</span> Status(ss.str());</div> |
| <div class="line"><a name="l00693"></a><span class="lineno"> 693</span>  }</div> |
| <div class="line"><a name="l00694"></a><span class="lineno"> 694</span>  }</div> |
| <div class="line"><a name="l00695"></a><span class="lineno"> 695</span> </div> |
| <div class="line"><a name="l00696"></a><span class="lineno"> 696</span>  <span class="comment">// 5. Optionally update the permissions of the created partition directories</span></div> |
| <div class="line"><a name="l00697"></a><span class="lineno"> 697</span>  <span class="comment">// Do this last in case we make the dirs unwriteable.</span></div> |
| <div class="line"><a name="l00698"></a><span class="lineno"> 698</span>  <span class="keywordflow">if</span> (FLAGS_insert_inherit_permissions) {</div> |
| <div class="line"><a name="l00699"></a><span class="lineno"> 699</span>  HdfsOperationSet chmod_ops(&hdfs_connection);</div> |
| <div class="line"><a name="l00700"></a><span class="lineno"> 700</span>  BOOST_FOREACH(<span class="keyword">const</span> PermissionCache::value_type& perm, permissions_cache) {</div> |
| <div class="line"><a name="l00701"></a><span class="lineno"> 701</span>  <span class="keywordtype">bool</span> new_dir = perm.second.first;</div> |
| <div class="line"><a name="l00702"></a><span class="lineno"> 702</span>  <span class="keywordflow">if</span> (new_dir) {</div> |
| <div class="line"><a name="l00703"></a><span class="lineno"> 703</span>  <span class="keywordtype">short</span> permissions = perm.second.second;</div> |
| <div class="line"><a name="l00704"></a><span class="lineno"> 704</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"INSERT created new directory: "</span> << perm.first</div> |
| <div class="line"><a name="l00705"></a><span class="lineno"> 705</span>  << <span class="stringliteral">", inherited permissions are: "</span> << oct << permissions;</div> |
| <div class="line"><a name="l00706"></a><span class="lineno"> 706</span>  chmod_ops.Add(<a class="code" href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa7e830d2a8a807bbb551965b429eba876">CHMOD</a>, perm.first, permissions);</div> |
| <div class="line"><a name="l00707"></a><span class="lineno"> 707</span>  }</div> |
| <div class="line"><a name="l00708"></a><span class="lineno"> 708</span>  }</div> |
| <div class="line"><a name="l00709"></a><span class="lineno"> 709</span>  <span class="keywordflow">if</span> (!chmod_ops.Execute(<a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a1aa344745ea152e6df02c4c4f7693aab">hdfs_op_thread_pool</a>(), <span class="keyword">false</span>)) {</div> |
| <div class="line"><a name="l00710"></a><span class="lineno"> 710</span>  stringstream ss;</div> |
| <div class="line"><a name="l00711"></a><span class="lineno"> 711</span>  ss << <span class="stringliteral">"Error(s) setting permissions on newly created partition directories. First"</span></div> |
| <div class="line"><a name="l00712"></a><span class="lineno"> 712</span>  << <span class="stringliteral">" error (of "</span> << chmod_ops.errors().size() << <span class="stringliteral">") was: "</span></div> |
| <div class="line"><a name="l00713"></a><span class="lineno"> 713</span>  << chmod_ops.errors()[0].second;</div> |
| <div class="line"><a name="l00714"></a><span class="lineno"> 714</span>  <span class="keywordflow">return</span> Status(ss.str());</div> |
| <div class="line"><a name="l00715"></a><span class="lineno"> 715</span>  }</div> |
| <div class="line"><a name="l00716"></a><span class="lineno"> 716</span>  }</div> |
| <div class="line"><a name="l00717"></a><span class="lineno"> 717</span> </div> |
| <div class="line"><a name="l00718"></a><span class="lineno"> 718</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00719"></a><span class="lineno"> 719</span> }</div> |
| <div class="line"><a name="l00720"></a><span class="lineno"> 720</span> </div> |
| <div class="line"><a name="l00721"></a><span class="lineno"> 721</span> Status <a class="code" href="namespaceimpala.html#a1114a1388a929cb5fcb5648310fffcd4">Coordinator::FinalizeQuery</a>() {</div> |
| <div class="line"><a name="l00722"></a><span class="lineno"> 722</span>  <span class="comment">// All backends must have reported their final statuses before finalization, which is a</span></div> |
| <div class="line"><a name="l00723"></a><span class="lineno"> 723</span>  <span class="comment">// post-condition of Wait. If the query was not successful, still try to clean up the</span></div> |
| <div class="line"><a name="l00724"></a><span class="lineno"> 724</span>  <span class="comment">// staging directory.</span></div> |
| <div class="line"><a name="l00725"></a><span class="lineno"> 725</span>  DCHECK(<a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a>);</div> |
| <div class="line"><a name="l00726"></a><span class="lineno"> 726</span>  DCHECK(needs_finalization_);</div> |
| <div class="line"><a name="l00727"></a><span class="lineno"> 727</span> </div> |
| <div class="line"><a name="l00728"></a><span class="lineno"> 728</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Finalizing query: "</span> << <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>;</div> |
| <div class="line"><a name="l00729"></a><span class="lineno"> 729</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="namespaceimpala.html#a7d75818564bd9d86830ca57bbe71116d">finalization_timer_</a>);</div> |
| <div class="line"><a name="l00730"></a><span class="lineno"> 730</span>  Status return_status = <a class="code" href="namespaceimpala.html#a5d05598507a4b64b2288e3522cf38c58">GetStatus</a>();</div> |
| <div class="line"><a name="l00731"></a><span class="lineno"> 731</span>  <span class="keywordflow">if</span> (return_status.ok()) {</div> |
| <div class="line"><a name="l00732"></a><span class="lineno"> 732</span>  return_status = <a class="code" href="namespaceimpala.html#aaedda9a1b006ac22f15b591ce15f0281">FinalizeSuccessfulInsert</a>();</div> |
| <div class="line"><a name="l00733"></a><span class="lineno"> 733</span>  }</div> |
| <div class="line"><a name="l00734"></a><span class="lineno"> 734</span> </div> |
| <div class="line"><a name="l00735"></a><span class="lineno"> 735</span>  stringstream staging_dir;</div> |
| <div class="line"><a name="l00736"></a><span class="lineno"> 736</span>  DCHECK(<a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.__isset.staging_dir);</div> |
| <div class="line"><a name="l00737"></a><span class="lineno"> 737</span>  staging_dir << <a class="code" href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">finalize_params_</a>.staging_dir << <span class="stringliteral">"/"</span> << <a class="code" href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">PrintId</a>(query_id_,<span class="stringliteral">"_"</span>) << <span class="stringliteral">"/"</span>;</div> |
| <div class="line"><a name="l00738"></a><span class="lineno"> 738</span> </div> |
| <div class="line"><a name="l00739"></a><span class="lineno"> 739</span>  hdfsFS hdfs_conn;</div> |
| <div class="line"><a name="l00740"></a><span class="lineno"> 740</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="classimpala_1_1HdfsFsCache.html#af7b18daf8c4da6bfce2c7612a515bf64">HdfsFsCache::instance</a>()->GetConnection(staging_dir.str(), &hdfs_conn));</div> |
| <div class="line"><a name="l00741"></a><span class="lineno"> 741</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Removing staging directory: "</span> << staging_dir.str();</div> |
| <div class="line"><a name="l00742"></a><span class="lineno"> 742</span>  hdfsDelete(hdfs_conn, staging_dir.str().c_str(), 1);</div> |
| <div class="line"><a name="l00743"></a><span class="lineno"> 743</span> </div> |
| <div class="line"><a name="l00744"></a><span class="lineno"> 744</span>  <span class="keywordflow">return</span> return_status;</div> |
| <div class="line"><a name="l00745"></a><span class="lineno"> 745</span> }</div> |
| <div class="line"><a name="l00746"></a><span class="lineno"> 746</span> </div> |
| <div class="line"><a name="l00747"></a><span class="lineno"> 747</span> Status <a class="code" href="namespaceimpala.html#ab71d76e113dc636fe9a9ef47c61f678a">Coordinator::WaitForAllBackends</a>() {</div> |
| <div class="line"><a name="l00748"></a><span class="lineno"> 748</span>  unique_lock<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l00749"></a><span class="lineno"> 749</span>  <span class="keywordflow">while</span> (<a class="code" href="namespaceimpala.html#a20991e967dbb80151f6500eb14145943">num_remaining_backends_</a> > 0 && <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) {</div> |
| <div class="line"><a name="l00750"></a><span class="lineno"> 750</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Coordinator waiting for backends to finish, "</span></div> |
| <div class="line"><a name="l00751"></a><span class="lineno"> 751</span>  << <a class="code" href="namespaceimpala.html#a20991e967dbb80151f6500eb14145943">num_remaining_backends_</a> << <span class="stringliteral">" remaining"</span>;</div> |
| <div class="line"><a name="l00752"></a><span class="lineno"> 752</span>  <a class="code" href="namespaceimpala.html#a6a1bc3d15242d20a00589ed2fd3c6066">backend_completion_cv_</a>.wait(l);</div> |
| <div class="line"><a name="l00753"></a><span class="lineno"> 753</span>  }</div> |
| <div class="line"><a name="l00754"></a><span class="lineno"> 754</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) {</div> |
| <div class="line"><a name="l00755"></a><span class="lineno"> 755</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"All backends finished successfully."</span>;</div> |
| <div class="line"><a name="l00756"></a><span class="lineno"> 756</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00757"></a><span class="lineno"> 757</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"All backends finished due to one or more errors."</span>;</div> |
| <div class="line"><a name="l00758"></a><span class="lineno"> 758</span>  }</div> |
| <div class="line"><a name="l00759"></a><span class="lineno"> 759</span> </div> |
| <div class="line"><a name="l00760"></a><span class="lineno"> 760</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>;</div> |
| <div class="line"><a name="l00761"></a><span class="lineno"> 761</span> }</div> |
| <div class="line"><a name="l00762"></a><span class="lineno"> 762</span> </div> |
| <div class="line"><a name="l00763"></a><span class="lineno"> 763</span> Status <a class="code" href="namespaceimpala.html#aae7232bb5a43094cb03cc189745e66b8">Coordinator::Wait</a>() {</div> |
| <div class="line"><a name="l00764"></a><span class="lineno"> 764</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#aab92a7cc1123d2a61596f47ec2fea1fa">wait_lock_</a>);</div> |
| <div class="line"><a name="l00765"></a><span class="lineno"> 765</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->total_time_counter());</div> |
| <div class="line"><a name="l00766"></a><span class="lineno"> 766</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a>) <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00767"></a><span class="lineno"> 767</span>  <a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00768"></a><span class="lineno"> 768</span>  Status return_status = <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00769"></a><span class="lineno"> 769</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL) {</div> |
| <div class="line"><a name="l00770"></a><span class="lineno"> 770</span>  <span class="comment">// Open() may block</span></div> |
| <div class="line"><a name="l00771"></a><span class="lineno"> 771</span>  return_status = <a class="code" href="namespaceimpala.html#aba23271e9c88b75efea05a2afb25856b">UpdateStatus</a>(<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->Open(), NULL);</div> |
| <div class="line"><a name="l00772"></a><span class="lineno"> 772</span> </div> |
| <div class="line"><a name="l00773"></a><span class="lineno"> 773</span>  <span class="keywordflow">if</span> (return_status.ok()) {</div> |
| <div class="line"><a name="l00774"></a><span class="lineno"> 774</span>  <span class="comment">// If the coordinator fragment has a sink, it will have finished executing at this</span></div> |
| <div class="line"><a name="l00775"></a><span class="lineno"> 775</span>  <span class="comment">// point. It's safe therefore to copy the set of files to move and updated</span></div> |
| <div class="line"><a name="l00776"></a><span class="lineno"> 776</span>  <span class="comment">// partitions into the query-wide set.</span></div> |
| <div class="line"><a name="l00777"></a><span class="lineno"> 777</span>  RuntimeState* state = <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>();</div> |
| <div class="line"><a name="l00778"></a><span class="lineno"> 778</span>  DCHECK(state != NULL);</div> |
| <div class="line"><a name="l00779"></a><span class="lineno"> 779</span> </div> |
| <div class="line"><a name="l00780"></a><span class="lineno"> 780</span>  <span class="comment">// No other backends should have updated these structures if the coordinator has a</span></div> |
| <div class="line"><a name="l00781"></a><span class="lineno"> 781</span>  <span class="comment">// fragment. (Backends have a sink only if the coordinator does not)</span></div> |
| <div class="line"><a name="l00782"></a><span class="lineno"> 782</span>  DCHECK_EQ(<a class="code" href="namespaceimpala.html#ab1a9829c4d817be471e86d26f1f316b8">files_to_move_</a>.size(), 0);</div> |
| <div class="line"><a name="l00783"></a><span class="lineno"> 783</span>  DCHECK_EQ(<a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a>.size(), 0);</div> |
| <div class="line"><a name="l00784"></a><span class="lineno"> 784</span> </div> |
| <div class="line"><a name="l00785"></a><span class="lineno"> 785</span>  <span class="comment">// Because there are no other updates, safe to copy the maps rather than merge them.</span></div> |
| <div class="line"><a name="l00786"></a><span class="lineno"> 786</span>  <a class="code" href="namespaceimpala.html#ab1a9829c4d817be471e86d26f1f316b8">files_to_move_</a> = *state->hdfs_files_to_move();</div> |
| <div class="line"><a name="l00787"></a><span class="lineno"> 787</span>  <a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a> = *state->per_partition_status();</div> |
| <div class="line"><a name="l00788"></a><span class="lineno"> 788</span>  }</div> |
| <div class="line"><a name="l00789"></a><span class="lineno"> 789</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00790"></a><span class="lineno"> 790</span>  <span class="comment">// Query finalization can only happen when all backends have reported</span></div> |
| <div class="line"><a name="l00791"></a><span class="lineno"> 791</span>  <span class="comment">// relevant state. They only have relevant state to report in the parallel</span></div> |
| <div class="line"><a name="l00792"></a><span class="lineno"> 792</span>  <span class="comment">// INSERT case, otherwise all the relevant state is from the coordinator</span></div> |
| <div class="line"><a name="l00793"></a><span class="lineno"> 793</span>  <span class="comment">// fragment which will be available after Open() returns.</span></div> |
| <div class="line"><a name="l00794"></a><span class="lineno"> 794</span>  <span class="comment">// Ignore the returned status if finalization is required., since FinalizeQuery() will</span></div> |
| <div class="line"><a name="l00795"></a><span class="lineno"> 795</span>  <span class="comment">// pick it up and needs to execute regardless.</span></div> |
| <div class="line"><a name="l00796"></a><span class="lineno"> 796</span>  Status status = <a class="code" href="namespaceimpala.html#ab71d76e113dc636fe9a9ef47c61f678a">WaitForAllBackends</a>();</div> |
| <div class="line"><a name="l00797"></a><span class="lineno"> 797</span>  <span class="keywordflow">if</span> (!needs_finalization_ && !status.ok()) <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l00798"></a><span class="lineno"> 798</span>  }</div> |
| <div class="line"><a name="l00799"></a><span class="lineno"> 799</span> </div> |
| <div class="line"><a name="l00800"></a><span class="lineno"> 800</span>  <span class="comment">// Query finalization is required only for HDFS table sinks</span></div> |
| <div class="line"><a name="l00801"></a><span class="lineno"> 801</span>  <span class="keywordflow">if</span> (needs_finalization_) {</div> |
| <div class="line"><a name="l00802"></a><span class="lineno"> 802</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="namespaceimpala.html#a1114a1388a929cb5fcb5648310fffcd4">FinalizeQuery</a>());</div> |
| <div class="line"><a name="l00803"></a><span class="lineno"> 803</span>  }</div> |
| <div class="line"><a name="l00804"></a><span class="lineno"> 804</span> </div> |
| <div class="line"><a name="l00805"></a><span class="lineno"> 805</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#a4f604de2fe46ab0a3969d06f4126bc77">stmt_type_</a> == TStmtType::DML) {</div> |
| <div class="line"><a name="l00806"></a><span class="lineno"> 806</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddInfoString(<span class="stringliteral">"Insert Stats"</span>,</div> |
| <div class="line"><a name="l00807"></a><span class="lineno"> 807</span>  <a class="code" href="classimpala_1_1DataSink.html#acb54f550a23683ce30805c392d24ebc7">DataSink::OutputInsertStats</a>(<a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a>, <span class="stringliteral">"\n"</span>));</div> |
| <div class="line"><a name="l00808"></a><span class="lineno"> 808</span>  <span class="comment">// For DML queries, when Wait is done, the query is complete. Report aggregate</span></div> |
| <div class="line"><a name="l00809"></a><span class="lineno"> 809</span>  <span class="comment">// query profiles at this point.</span></div> |
| <div class="line"><a name="l00810"></a><span class="lineno"> 810</span>  <span class="comment">// TODO: make sure ReportQuerySummary gets called on error</span></div> |
| <div class="line"><a name="l00811"></a><span class="lineno"> 811</span>  <a class="code" href="namespaceimpala.html#a0e6d46f5800bcf4765efd3c21ce535bb">ReportQuerySummary</a>();</div> |
| <div class="line"><a name="l00812"></a><span class="lineno"> 812</span>  }</div> |
| <div class="line"><a name="l00813"></a><span class="lineno"> 813</span> </div> |
| <div class="line"><a name="l00814"></a><span class="lineno"> 814</span>  <span class="keywordflow">return</span> return_status;</div> |
| <div class="line"><a name="l00815"></a><span class="lineno"> 815</span> }</div> |
| <div class="line"><a name="l00816"></a><span class="lineno"> 816</span> </div> |
| <div class="line"><a name="l00817"></a><span class="lineno"> 817</span> Status <a class="code" href="namespaceimpala.html#af279911558f793413306452d5b16e6d6">Coordinator::GetNext</a>(RowBatch** batch, RuntimeState* state) {</div> |
| <div class="line"><a name="l00818"></a><span class="lineno"> 818</span>  <a class="code" href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a> << <span class="stringliteral">"GetNext() query_id="</span> << <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>;</div> |
| <div class="line"><a name="l00819"></a><span class="lineno"> 819</span>  DCHECK(<a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a>);</div> |
| <div class="line"><a name="l00820"></a><span class="lineno"> 820</span>  <a class="code" href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a>(<a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->total_time_counter());</div> |
| <div class="line"><a name="l00821"></a><span class="lineno"> 821</span> </div> |
| <div class="line"><a name="l00822"></a><span class="lineno"> 822</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() == NULL) {</div> |
| <div class="line"><a name="l00823"></a><span class="lineno"> 823</span>  <span class="comment">// If there is no local fragment, we produce no output, and execution will</span></div> |
| <div class="line"><a name="l00824"></a><span class="lineno"> 824</span>  <span class="comment">// have finished after Wait.</span></div> |
| <div class="line"><a name="l00825"></a><span class="lineno"> 825</span>  *batch = NULL;</div> |
| <div class="line"><a name="l00826"></a><span class="lineno"> 826</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#a5d05598507a4b64b2288e3522cf38c58">GetStatus</a>();</div> |
| <div class="line"><a name="l00827"></a><span class="lineno"> 827</span>  }</div> |
| <div class="line"><a name="l00828"></a><span class="lineno"> 828</span> </div> |
| <div class="line"><a name="l00829"></a><span class="lineno"> 829</span>  <span class="comment">// do not acquire lock_ here, otherwise we could block and prevent an async</span></div> |
| <div class="line"><a name="l00830"></a><span class="lineno"> 830</span>  <span class="comment">// Cancel() from proceeding</span></div> |
| <div class="line"><a name="l00831"></a><span class="lineno"> 831</span>  Status status = <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->GetNext(batch);</div> |
| <div class="line"><a name="l00832"></a><span class="lineno"> 832</span> </div> |
| <div class="line"><a name="l00833"></a><span class="lineno"> 833</span>  <span class="comment">// if there was an error, we need to return the query's error status rather than</span></div> |
| <div class="line"><a name="l00834"></a><span class="lineno"> 834</span>  <span class="comment">// the status we just got back from the local executor (which may well be CANCELLED</span></div> |
| <div class="line"><a name="l00835"></a><span class="lineno"> 835</span>  <span class="comment">// in that case). Coordinator fragment failed in this case so we log the query_id.</span></div> |
| <div class="line"><a name="l00836"></a><span class="lineno"> 836</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="namespaceimpala.html#aba23271e9c88b75efea05a2afb25856b">UpdateStatus</a>(status, &<a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>()->fragment_instance_id()));</div> |
| <div class="line"><a name="l00837"></a><span class="lineno"> 837</span> </div> |
| <div class="line"><a name="l00838"></a><span class="lineno"> 838</span>  <span class="keywordflow">if</span> (*batch == NULL) {</div> |
| <div class="line"><a name="l00839"></a><span class="lineno"> 839</span>  <a class="code" href="namespaceimpala.html#aea293e10bddb1360a019cd1e9abf45f2">returned_all_results_</a> = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00840"></a><span class="lineno"> 840</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->ReachedLimit()) {</div> |
| <div class="line"><a name="l00841"></a><span class="lineno"> 841</span>  <span class="comment">// We've reached the query limit, cancel the remote fragments. The</span></div> |
| <div class="line"><a name="l00842"></a><span class="lineno"> 842</span>  <span class="comment">// Exchange node on our fragment is no longer receiving rows so the</span></div> |
| <div class="line"><a name="l00843"></a><span class="lineno"> 843</span>  <span class="comment">// remote fragments must be explicitly cancelled.</span></div> |
| <div class="line"><a name="l00844"></a><span class="lineno"> 844</span>  <a class="code" href="namespaceimpala.html#a693c7d4098922a8f6d32dcc0c03cc9b7">CancelRemoteFragments</a>();</div> |
| <div class="line"><a name="l00845"></a><span class="lineno"> 845</span>  RuntimeState* state = <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">runtime_state</a>();</div> |
| <div class="line"><a name="l00846"></a><span class="lineno"> 846</span>  <span class="keywordflow">if</span> (state != NULL) {</div> |
| <div class="line"><a name="l00847"></a><span class="lineno"> 847</span>  <span class="comment">// Cancel the streams receiving batches. The exchange nodes that would</span></div> |
| <div class="line"><a name="l00848"></a><span class="lineno"> 848</span>  <span class="comment">// normally read from the streams are done.</span></div> |
| <div class="line"><a name="l00849"></a><span class="lineno"> 849</span>  state->stream_mgr()->Cancel(state->fragment_instance_id());</div> |
| <div class="line"><a name="l00850"></a><span class="lineno"> 850</span>  }</div> |
| <div class="line"><a name="l00851"></a><span class="lineno"> 851</span>  }</div> |
| <div class="line"><a name="l00852"></a><span class="lineno"> 852</span> </div> |
| <div class="line"><a name="l00853"></a><span class="lineno"> 853</span>  <span class="comment">// Don't return final NULL until all backends have completed.</span></div> |
| <div class="line"><a name="l00854"></a><span class="lineno"> 854</span>  <span class="comment">// GetNext must wait for all backends to complete before</span></div> |
| <div class="line"><a name="l00855"></a><span class="lineno"> 855</span>  <span class="comment">// ultimately signalling the end of execution via a NULL</span></div> |
| <div class="line"><a name="l00856"></a><span class="lineno"> 856</span>  <span class="comment">// batch. After NULL is returned, the coordinator may tear down</span></div> |
| <div class="line"><a name="l00857"></a><span class="lineno"> 857</span>  <span class="comment">// query state, and perform post-query finalization which might</span></div> |
| <div class="line"><a name="l00858"></a><span class="lineno"> 858</span>  <span class="comment">// depend on the reports from all backends.</span></div> |
| <div class="line"><a name="l00859"></a><span class="lineno"> 859</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="namespaceimpala.html#ab71d76e113dc636fe9a9ef47c61f678a">WaitForAllBackends</a>());</div> |
| <div class="line"><a name="l00860"></a><span class="lineno"> 860</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) {</div> |
| <div class="line"><a name="l00861"></a><span class="lineno"> 861</span>  <span class="comment">// If the query completed successfully, report aggregate query profiles.</span></div> |
| <div class="line"><a name="l00862"></a><span class="lineno"> 862</span>  <a class="code" href="namespaceimpala.html#a0e6d46f5800bcf4765efd3c21ce535bb">ReportQuerySummary</a>();</div> |
| <div class="line"><a name="l00863"></a><span class="lineno"> 863</span>  }</div> |
| <div class="line"><a name="l00864"></a><span class="lineno"> 864</span>  }</div> |
| <div class="line"><a name="l00865"></a><span class="lineno"> 865</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00866"></a><span class="lineno"> 866</span> }</div> |
| <div class="line"><a name="l00867"></a><span class="lineno"> 867</span> </div> |
| <div class="line"><a name="l00868"></a><span class="lineno"> 868</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a7dff4fbd53ca26eb055bc173471c7be3">Coordinator::PrintBackendInfo</a>() {</div> |
| <div class="line"><a name="l00869"></a><span class="lineno"> 869</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00870"></a><span class="lineno"> 870</span>  <a class="code" href="namespaceimpala.html#ad4f56de7b31b5daea6ea0e854715925e">SummaryStats</a>& acc =</div> |
| <div class="line"><a name="l00871"></a><span class="lineno"> 871</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->fragment_idx].bytes_assigned;</div> |
| <div class="line"><a name="l00872"></a><span class="lineno"> 872</span>  acc(<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->total_split_size);</div> |
| <div class="line"><a name="l00873"></a><span class="lineno"> 873</span>  }</div> |
| <div class="line"><a name="l00874"></a><span class="lineno"> 874</span> </div> |
| <div class="line"><a name="l00875"></a><span class="lineno"> 875</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() == NULL ? 0 : 1); i < <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l00876"></a><span class="lineno"> 876</span>  <a class="code" href="namespaceimpala.html#ad4f56de7b31b5daea6ea0e854715925e">SummaryStats</a>& acc = <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].bytes_assigned;</div> |
| <div class="line"><a name="l00877"></a><span class="lineno"> 877</span>  <span class="keywordtype">double</span> min = accumulators::min(acc);</div> |
| <div class="line"><a name="l00878"></a><span class="lineno"> 878</span>  <span class="keywordtype">double</span> max = accumulators::max(acc);</div> |
| <div class="line"><a name="l00879"></a><span class="lineno"> 879</span>  <span class="keywordtype">double</span> mean = accumulators::mean(acc);</div> |
| <div class="line"><a name="l00880"></a><span class="lineno"> 880</span>  <span class="keywordtype">double</span> stddev = sqrt(accumulators::variance(acc));</div> |
| <div class="line"><a name="l00881"></a><span class="lineno"> 881</span>  stringstream ss;</div> |
| <div class="line"><a name="l00882"></a><span class="lineno"> 882</span>  ss << <span class="stringliteral">" min: "</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(min, TUnit::BYTES)</div> |
| <div class="line"><a name="l00883"></a><span class="lineno"> 883</span>  << <span class="stringliteral">", max: "</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(max, TUnit::BYTES)</div> |
| <div class="line"><a name="l00884"></a><span class="lineno"> 884</span>  << <span class="stringliteral">", avg: "</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(mean, TUnit::BYTES)</div> |
| <div class="line"><a name="l00885"></a><span class="lineno"> 885</span>  << <span class="stringliteral">", stddev: "</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(stddev, TUnit::BYTES);</div> |
| <div class="line"><a name="l00886"></a><span class="lineno"> 886</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile->AddInfoString(<span class="stringliteral">"split sizes"</span>, ss.str());</div> |
| <div class="line"><a name="l00887"></a><span class="lineno"> 887</span> </div> |
| <div class="line"><a name="l00888"></a><span class="lineno"> 888</span>  <span class="keywordflow">if</span> (<a class="code" href="logging_8h.html#a6f29cadf91ddc478eb9fa4e02cd8e0a0">VLOG_FILE_IS_ON</a>) {</div> |
| <div class="line"><a name="l00889"></a><span class="lineno"> 889</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"Byte split for fragment "</span> << i << <span class="stringliteral">" "</span> << ss.str();</div> |
| <div class="line"><a name="l00890"></a><span class="lineno"> 890</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> j = 0; j < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++j) {</div> |
| <div class="line"><a name="l00891"></a><span class="lineno"> 891</span>  BackendExecState* exec_state = <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[j];</div> |
| <div class="line"><a name="l00892"></a><span class="lineno"> 892</span>  <span class="keywordflow">if</span> (exec_state->fragment_idx != i) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00893"></a><span class="lineno"> 893</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"data volume for ipaddress "</span> << exec_state << <span class="stringliteral">": "</span></div> |
| <div class="line"><a name="l00894"></a><span class="lineno"> 894</span>  << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l00895"></a><span class="lineno"> 895</span>  exec_state->total_split_size, TUnit::BYTES);</div> |
| <div class="line"><a name="l00896"></a><span class="lineno"> 896</span>  }</div> |
| <div class="line"><a name="l00897"></a><span class="lineno"> 897</span>  }</div> |
| <div class="line"><a name="l00898"></a><span class="lineno"> 898</span>  }</div> |
| <div class="line"><a name="l00899"></a><span class="lineno"> 899</span> }</div> |
| <div class="line"><a name="l00900"></a><span class="lineno"> 900</span> </div> |
| <div class="line"><a name="l00901"></a><span class="lineno"> 901</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#aecf5ef62c6e2482353b0f41f80528245">Coordinator::InitExecProfile</a>(<span class="keyword">const</span> TQueryExecRequest& request) {</div> |
| <div class="line"><a name="l00902"></a><span class="lineno"> 902</span>  <span class="comment">// Initialize the structure to collect execution summary of every plan node.</span></div> |
| <div class="line"><a name="l00903"></a><span class="lineno"> 903</span>  <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.__isset.nodes = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00904"></a><span class="lineno"> 904</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < request.fragments.size(); ++i) {</div> |
| <div class="line"><a name="l00905"></a><span class="lineno"> 905</span>  <span class="keywordflow">if</span> (!request.fragments[i].__isset.plan) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00906"></a><span class="lineno"> 906</span>  <span class="keyword">const</span> TPlan& plan = request.fragments[i].plan;</div> |
| <div class="line"><a name="l00907"></a><span class="lineno"> 907</span>  <span class="keywordtype">int</span> fragment_first_node_idx = <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.nodes.size();</div> |
| <div class="line"><a name="l00908"></a><span class="lineno"> 908</span> </div> |
| <div class="line"><a name="l00909"></a><span class="lineno"> 909</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> j = 0; j < plan.nodes.size(); ++j) {</div> |
| <div class="line"><a name="l00910"></a><span class="lineno"> 910</span>  TPlanNodeExecSummary node;</div> |
| <div class="line"><a name="l00911"></a><span class="lineno"> 911</span>  node.node_id = plan.nodes[j].node_id;</div> |
| <div class="line"><a name="l00912"></a><span class="lineno"> 912</span>  node.fragment_id = i;</div> |
| <div class="line"><a name="l00913"></a><span class="lineno"> 913</span>  node.label = plan.nodes[j].label;</div> |
| <div class="line"><a name="l00914"></a><span class="lineno"> 914</span>  node.__set_label_detail(plan.nodes[j].label_detail);</div> |
| <div class="line"><a name="l00915"></a><span class="lineno"> 915</span>  node.num_children = plan.nodes[j].num_children;</div> |
| <div class="line"><a name="l00916"></a><span class="lineno"> 916</span> </div> |
| <div class="line"><a name="l00917"></a><span class="lineno"> 917</span>  <span class="keywordflow">if</span> (plan.nodes[j].__isset.estimated_stats) {</div> |
| <div class="line"><a name="l00918"></a><span class="lineno"> 918</span>  node.__set_estimated_stats(plan.nodes[j].estimated_stats);</div> |
| <div class="line"><a name="l00919"></a><span class="lineno"> 919</span>  }</div> |
| <div class="line"><a name="l00920"></a><span class="lineno"> 920</span> </div> |
| <div class="line"><a name="l00921"></a><span class="lineno"> 921</span>  <a class="code" href="namespaceimpala.html#a574dcc25c5967c10d13e4bf3fa4c9bcf">plan_node_id_to_summary_map_</a>[plan.nodes[j].node_id] = <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.nodes.size();</div> |
| <div class="line"><a name="l00922"></a><span class="lineno"> 922</span>  <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.nodes.push_back(node);</div> |
| <div class="line"><a name="l00923"></a><span class="lineno"> 923</span>  }</div> |
| <div class="line"><a name="l00924"></a><span class="lineno"> 924</span> </div> |
| <div class="line"><a name="l00925"></a><span class="lineno"> 925</span>  <span class="keywordflow">if</span> (request.fragments[i].__isset.output_sink &&</div> |
| <div class="line"><a name="l00926"></a><span class="lineno"> 926</span>  request.fragments[i].output_sink.type == TDataSinkType::DATA_STREAM_SINK) {</div> |
| <div class="line"><a name="l00927"></a><span class="lineno"> 927</span>  <span class="keyword">const</span> TDataStreamSink& sink = request.fragments[i].output_sink.stream_sink;</div> |
| <div class="line"><a name="l00928"></a><span class="lineno"> 928</span>  <span class="keywordtype">int</span> exch_idx = <a class="code" href="namespaceimpala.html#a574dcc25c5967c10d13e4bf3fa4c9bcf">plan_node_id_to_summary_map_</a>[sink.dest_node_id];</div> |
| <div class="line"><a name="l00929"></a><span class="lineno"> 929</span>  <span class="keywordflow">if</span> (sink.output_partition.type == TPartitionType::UNPARTITIONED) {</div> |
| <div class="line"><a name="l00930"></a><span class="lineno"> 930</span>  <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.nodes[exch_idx].__set_is_broadcast(<span class="keyword">true</span>);</div> |
| <div class="line"><a name="l00931"></a><span class="lineno"> 931</span>  }</div> |
| <div class="line"><a name="l00932"></a><span class="lineno"> 932</span>  <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.__isset.exch_to_sender_map = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00933"></a><span class="lineno"> 933</span>  <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.exch_to_sender_map[exch_idx] = fragment_first_node_idx;</div> |
| <div class="line"><a name="l00934"></a><span class="lineno"> 934</span>  }</div> |
| <div class="line"><a name="l00935"></a><span class="lineno"> 935</span>  }</div> |
| <div class="line"><a name="l00936"></a><span class="lineno"> 936</span> </div> |
| <div class="line"><a name="l00937"></a><span class="lineno"> 937</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL) {</div> |
| <div class="line"><a name="l00938"></a><span class="lineno"> 938</span>  <span class="comment">// register coordinator's fragment profile now, before those of the backends,</span></div> |
| <div class="line"><a name="l00939"></a><span class="lineno"> 939</span>  <span class="comment">// so it shows up at the top</span></div> |
| <div class="line"><a name="l00940"></a><span class="lineno"> 940</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddChild(<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile());</div> |
| <div class="line"><a name="l00941"></a><span class="lineno"> 941</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile()->set_name(Substitute(<span class="stringliteral">"Coordinator Fragment $0"</span>,</div> |
| <div class="line"><a name="l00942"></a><span class="lineno"> 942</span>  request.fragments[0].display_name));</div> |
| <div class="line"><a name="l00943"></a><span class="lineno"> 943</span>  <a class="code" href="namespaceimpala.html#a20d70f7b44aeb530802e3ccc198f19fa">CollectScanNodeCounters</a>(<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile(), &<a class="code" href="namespaceimpala.html#a32793557c7c9e175d833303a8f3df878">coordinator_counters_</a>);</div> |
| <div class="line"><a name="l00944"></a><span class="lineno"> 944</span>  }</div> |
| <div class="line"><a name="l00945"></a><span class="lineno"> 945</span> </div> |
| <div class="line"><a name="l00946"></a><span class="lineno"> 946</span>  <span class="comment">// Initialize the runtime profile structure. This adds the per fragment average</span></div> |
| <div class="line"><a name="l00947"></a><span class="lineno"> 947</span>  <span class="comment">// profiles followed by the per fragment instance profiles.</span></div> |
| <div class="line"><a name="l00948"></a><span class="lineno"> 948</span>  <span class="keywordtype">bool</span> has_coordinator_fragment =</div> |
| <div class="line"><a name="l00949"></a><span class="lineno"> 949</span>  request.fragments[0].partition.type == TPartitionType::UNPARTITIONED;</div> |
| <div class="line"><a name="l00950"></a><span class="lineno"> 950</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>.resize(request.fragments.size());</div> |
| <div class="line"><a name="l00951"></a><span class="lineno"> 951</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < request.fragments.size(); ++i) {</div> |
| <div class="line"><a name="l00952"></a><span class="lineno"> 952</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].num_instances = 0;</div> |
| <div class="line"><a name="l00953"></a><span class="lineno"> 953</span> </div> |
| <div class="line"><a name="l00954"></a><span class="lineno"> 954</span>  <span class="comment">// Special case fragment idx 0 if there is a coordinator. There is only one</span></div> |
| <div class="line"><a name="l00955"></a><span class="lineno"> 955</span>  <span class="comment">// instance of this profile so the average is just the coordinator profile.</span></div> |
| <div class="line"><a name="l00956"></a><span class="lineno"> 956</span>  <span class="keywordflow">if</span> (i == 0 && has_coordinator_fragment) {</div> |
| <div class="line"><a name="l00957"></a><span class="lineno"> 957</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile = <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile();</div> |
| <div class="line"><a name="l00958"></a><span class="lineno"> 958</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].num_instances = 1;</div> |
| <div class="line"><a name="l00959"></a><span class="lineno"> 959</span>  <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00960"></a><span class="lineno"> 960</span>  }</div> |
| <div class="line"><a name="l00961"></a><span class="lineno"> 961</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile =</div> |
| <div class="line"><a name="l00962"></a><span class="lineno"> 962</span>  <a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<span class="keyword">new</span> RuntimeProfile(<a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>(),</div> |
| <div class="line"><a name="l00963"></a><span class="lineno"> 963</span>  Substitute(<span class="stringliteral">"Averaged Fragment $0"</span>, request.fragments[i].display_name), <span class="keyword">true</span>));</div> |
| <div class="line"><a name="l00964"></a><span class="lineno"> 964</span>  <span class="comment">// Insert the avg profiles in ascending fragment number order. If</span></div> |
| <div class="line"><a name="l00965"></a><span class="lineno"> 965</span>  <span class="comment">// there is a coordinator fragment, it's been placed in</span></div> |
| <div class="line"><a name="l00966"></a><span class="lineno"> 966</span>  <span class="comment">// fragment_profiles_[0].averaged_profile, ensuring that this code</span></div> |
| <div class="line"><a name="l00967"></a><span class="lineno"> 967</span>  <span class="comment">// will put the first averaged profile immediately after it. If</span></div> |
| <div class="line"><a name="l00968"></a><span class="lineno"> 968</span>  <span class="comment">// there is no coordinator fragment, the first averaged profile</span></div> |
| <div class="line"><a name="l00969"></a><span class="lineno"> 969</span>  <span class="comment">// will be inserted as the first child of query_profile_, and then</span></div> |
| <div class="line"><a name="l00970"></a><span class="lineno"> 970</span>  <span class="comment">// all other averaged fragments will follow.</span></div> |
| <div class="line"><a name="l00971"></a><span class="lineno"> 971</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddChild(<a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile, <span class="keyword">true</span>,</div> |
| <div class="line"><a name="l00972"></a><span class="lineno"> 972</span>  (i > 0) ? <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i-1].averaged_profile : NULL);</div> |
| <div class="line"><a name="l00973"></a><span class="lineno"> 973</span> </div> |
| <div class="line"><a name="l00974"></a><span class="lineno"> 974</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].root_profile =</div> |
| <div class="line"><a name="l00975"></a><span class="lineno"> 975</span>  <a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>()-><a class="code" href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">Add</a>(<span class="keyword">new</span> RuntimeProfile(<a class="code" href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">obj_pool</a>(),</div> |
| <div class="line"><a name="l00976"></a><span class="lineno"> 976</span>  Substitute(<span class="stringliteral">"Fragment $0"</span>, request.fragments[i].display_name)));</div> |
| <div class="line"><a name="l00977"></a><span class="lineno"> 977</span>  <span class="comment">// Note: we don't start the wall timer here for the fragment</span></div> |
| <div class="line"><a name="l00978"></a><span class="lineno"> 978</span>  <span class="comment">// profile; it's uninteresting and misleading.</span></div> |
| <div class="line"><a name="l00979"></a><span class="lineno"> 979</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddChild(<a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].root_profile);</div> |
| <div class="line"><a name="l00980"></a><span class="lineno"> 980</span>  }</div> |
| <div class="line"><a name="l00981"></a><span class="lineno"> 981</span> }</div> |
| <div class="line"><a name="l00982"></a><span class="lineno"> 982</span> </div> |
| <div class="line"><a name="l00983"></a><span class="lineno"> 983</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a20d70f7b44aeb530802e3ccc198f19fa">Coordinator::CollectScanNodeCounters</a>(RuntimeProfile* profile,</div> |
| <div class="line"><a name="l00984"></a><span class="lineno"> 984</span>  FragmentInstanceCounters* counters) {</div> |
| <div class="line"><a name="l00985"></a><span class="lineno"> 985</span>  vector<RuntimeProfile*> children;</div> |
| <div class="line"><a name="l00986"></a><span class="lineno"> 986</span>  profile->GetAllChildren(&children);</div> |
| <div class="line"><a name="l00987"></a><span class="lineno"> 987</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < children.size(); ++i) {</div> |
| <div class="line"><a name="l00988"></a><span class="lineno"> 988</span>  RuntimeProfile* p = children[i];</div> |
| <div class="line"><a name="l00989"></a><span class="lineno"> 989</span>  <a class="code" href="namespaceimpala.html#aca80061c98b44477ea84e4332993b7e7">PlanNodeId</a> <span class="keywordtype">id</span> = <a class="code" href="classimpala_1_1ExecNode.html#a3e2d144b60f904f6b1f3219b9471aad3">ExecNode::GetNodeIdFromProfile</a>(p);</div> |
| <div class="line"><a name="l00990"></a><span class="lineno"> 990</span> </div> |
| <div class="line"><a name="l00991"></a><span class="lineno"> 991</span>  <span class="comment">// This profile is not for an exec node.</span></div> |
| <div class="line"><a name="l00992"></a><span class="lineno"> 992</span>  <span class="keywordflow">if</span> (<span class="keywordtype">id</span> == g_ImpalaInternalService_constants.INVALID_PLAN_NODE_ID) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l00993"></a><span class="lineno"> 993</span> </div> |
| <div class="line"><a name="l00994"></a><span class="lineno"> 994</span>  RuntimeProfile::Counter* throughput_counter =</div> |
| <div class="line"><a name="l00995"></a><span class="lineno"> 995</span>  p->GetCounter(<a class="code" href="classimpala_1_1ScanNode.html#ab56d0e228f9b992bf4eb15a24808be43">ScanNode::TOTAL_THROUGHPUT_COUNTER</a>);</div> |
| <div class="line"><a name="l00996"></a><span class="lineno"> 996</span>  <span class="keywordflow">if</span> (throughput_counter != NULL) {</div> |
| <div class="line"><a name="l00997"></a><span class="lineno"> 997</span>  counters->throughput_counters[id] = throughput_counter;</div> |
| <div class="line"><a name="l00998"></a><span class="lineno"> 998</span>  }</div> |
| <div class="line"><a name="l00999"></a><span class="lineno"> 999</span>  RuntimeProfile::Counter* scan_ranges_counter =</div> |
| <div class="line"><a name="l01000"></a><span class="lineno"> 1000</span>  p->GetCounter(<a class="code" href="classimpala_1_1ScanNode.html#a86d533cdcfa34c18ae02db91536e0d4c">ScanNode::SCAN_RANGES_COMPLETE_COUNTER</a>);</div> |
| <div class="line"><a name="l01001"></a><span class="lineno"> 1001</span>  <span class="keywordflow">if</span> (scan_ranges_counter != NULL) {</div> |
| <div class="line"><a name="l01002"></a><span class="lineno"> 1002</span>  counters->scan_ranges_complete_counters[id] = scan_ranges_counter;</div> |
| <div class="line"><a name="l01003"></a><span class="lineno"> 1003</span>  }</div> |
| <div class="line"><a name="l01004"></a><span class="lineno"> 1004</span>  }</div> |
| <div class="line"><a name="l01005"></a><span class="lineno"> 1005</span> }</div> |
| <div class="line"><a name="l01006"></a><span class="lineno"> 1006</span> </div> |
| <div class="line"><a name="l01007"></a><span class="lineno"> 1007</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#aadc20c1e4da4986703816cb21b3ecf44">Coordinator::CreateAggregateCounters</a>(</div> |
| <div class="line"><a name="l01008"></a><span class="lineno"> 1008</span>  <span class="keyword">const</span> vector<TPlanFragment>& fragments) {</div> |
| <div class="line"><a name="l01009"></a><span class="lineno"> 1009</span>  BOOST_FOREACH(<span class="keyword">const</span> TPlanFragment& fragment, fragments) {</div> |
| <div class="line"><a name="l01010"></a><span class="lineno"> 1010</span>  <span class="keywordflow">if</span> (!fragment.__isset.plan) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01011"></a><span class="lineno"> 1011</span>  <span class="keyword">const</span> vector<TPlanNode>& nodes = fragment.plan.nodes;</div> |
| <div class="line"><a name="l01012"></a><span class="lineno"> 1012</span>  BOOST_FOREACH(<span class="keyword">const</span> TPlanNode& node, nodes) {</div> |
| <div class="line"><a name="l01013"></a><span class="lineno"> 1013</span>  <span class="keywordflow">if</span> (node.node_type != TPlanNodeType::HDFS_SCAN_NODE</div> |
| <div class="line"><a name="l01014"></a><span class="lineno"> 1014</span>  && node.node_type != TPlanNodeType::HBASE_SCAN_NODE) {</div> |
| <div class="line"><a name="l01015"></a><span class="lineno"> 1015</span>  <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01016"></a><span class="lineno"> 1016</span>  }</div> |
| <div class="line"><a name="l01017"></a><span class="lineno"> 1017</span> </div> |
| <div class="line"><a name="l01018"></a><span class="lineno"> 1018</span>  stringstream s;</div> |
| <div class="line"><a name="l01019"></a><span class="lineno"> 1019</span>  s << <a class="code" href="namespaceimpala.html#ab2bf118ff6d9897b828f5fb8cce70ad3">PrintPlanNodeType</a>(node.node_type) << <span class="stringliteral">" (id="</span></div> |
| <div class="line"><a name="l01020"></a><span class="lineno"> 1020</span>  << node.node_id << <span class="stringliteral">") Throughput"</span>;</div> |
| <div class="line"><a name="l01021"></a><span class="lineno"> 1021</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddDerivedCounter(s.str(), TUnit::BYTES_PER_SECOND,</div> |
| <div class="line"><a name="l01022"></a><span class="lineno"> 1022</span>  bind<int64_t>(mem_fn(&<a class="code" href="namespaceimpala.html#a55cca46ac54085054a0c08d31e05d4fe">Coordinator::ComputeTotalThroughput</a>),</div> |
| <div class="line"><a name="l01023"></a><span class="lineno"> 1023</span>  <span class="keyword">this</span>, node.node_id));</div> |
| <div class="line"><a name="l01024"></a><span class="lineno"> 1024</span>  s.str(<span class="stringliteral">""</span>);</div> |
| <div class="line"><a name="l01025"></a><span class="lineno"> 1025</span>  s << <a class="code" href="namespaceimpala.html#ab2bf118ff6d9897b828f5fb8cce70ad3">PrintPlanNodeType</a>(node.node_type) << <span class="stringliteral">" (id="</span></div> |
| <div class="line"><a name="l01026"></a><span class="lineno"> 1026</span>  << node.node_id << <span class="stringliteral">") Completed scan ranges"</span>;</div> |
| <div class="line"><a name="l01027"></a><span class="lineno"> 1027</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddDerivedCounter(s.str(), TUnit::UNIT,</div> |
| <div class="line"><a name="l01028"></a><span class="lineno"> 1028</span>  bind<int64_t>(mem_fn(&<a class="code" href="namespaceimpala.html#a342149bf019a87aa2c45ec6b8a902865">Coordinator::ComputeTotalScanRangesComplete</a>),</div> |
| <div class="line"><a name="l01029"></a><span class="lineno"> 1029</span>  <span class="keyword">this</span>, node.node_id));</div> |
| <div class="line"><a name="l01030"></a><span class="lineno"> 1030</span>  }</div> |
| <div class="line"><a name="l01031"></a><span class="lineno"> 1031</span>  }</div> |
| <div class="line"><a name="l01032"></a><span class="lineno"> 1032</span> }</div> |
| <div class="line"><a name="l01033"></a><span class="lineno"> 1033</span> </div> |
| <div class="line"><a name="l01034"></a><span class="lineno"> 1034</span> int64_t <a class="code" href="namespaceimpala.html#a55cca46ac54085054a0c08d31e05d4fe">Coordinator::ComputeTotalThroughput</a>(<span class="keywordtype">int</span> node_id) {</div> |
| <div class="line"><a name="l01035"></a><span class="lineno"> 1035</span>  int64_t value = 0;</div> |
| <div class="line"><a name="l01036"></a><span class="lineno"> 1036</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01037"></a><span class="lineno"> 1037</span>  BackendExecState* exec_state = <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i];</div> |
| <div class="line"><a name="l01038"></a><span class="lineno"> 1038</span>  value += exec_state->GetNodeThroughput(node_id);</div> |
| <div class="line"><a name="l01039"></a><span class="lineno"> 1039</span>  }</div> |
| <div class="line"><a name="l01040"></a><span class="lineno"> 1040</span>  <span class="comment">// Add up the local fragment throughput counter</span></div> |
| <div class="line"><a name="l01041"></a><span class="lineno"> 1041</span>  <a class="code" href="namespaceimpala.html#a47d8255dea5b52dc4c93a11ed02b29b5">CounterMap</a>& throughput_counters = <a class="code" href="namespaceimpala.html#a32793557c7c9e175d833303a8f3df878">coordinator_counters_</a>.<a class="code" href="structimpala_1_1FragmentInstanceCounters.html#a1b44d89f4c7de168ec44110894cfa0f1">throughput_counters</a>;</div> |
| <div class="line"><a name="l01042"></a><span class="lineno"> 1042</span>  CounterMap::iterator it = throughput_counters.find(node_id);</div> |
| <div class="line"><a name="l01043"></a><span class="lineno"> 1043</span>  <span class="keywordflow">if</span> (it != throughput_counters.end()) {</div> |
| <div class="line"><a name="l01044"></a><span class="lineno"> 1044</span>  value += it->second->value();</div> |
| <div class="line"><a name="l01045"></a><span class="lineno"> 1045</span>  }</div> |
| <div class="line"><a name="l01046"></a><span class="lineno"> 1046</span>  <span class="keywordflow">return</span> value;</div> |
| <div class="line"><a name="l01047"></a><span class="lineno"> 1047</span> }</div> |
| <div class="line"><a name="l01048"></a><span class="lineno"> 1048</span> </div> |
| <div class="line"><a name="l01049"></a><span class="lineno"> 1049</span> int64_t <a class="code" href="namespaceimpala.html#a342149bf019a87aa2c45ec6b8a902865">Coordinator::ComputeTotalScanRangesComplete</a>(<span class="keywordtype">int</span> node_id) {</div> |
| <div class="line"><a name="l01050"></a><span class="lineno"> 1050</span>  int64_t value = 0;</div> |
| <div class="line"><a name="l01051"></a><span class="lineno"> 1051</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01052"></a><span class="lineno"> 1052</span>  BackendExecState* exec_state = <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i];</div> |
| <div class="line"><a name="l01053"></a><span class="lineno"> 1053</span>  value += exec_state->GetNumScanRangesCompleted(node_id);</div> |
| <div class="line"><a name="l01054"></a><span class="lineno"> 1054</span>  }</div> |
| <div class="line"><a name="l01055"></a><span class="lineno"> 1055</span>  <span class="comment">// Add up the local fragment throughput counter</span></div> |
| <div class="line"><a name="l01056"></a><span class="lineno"> 1056</span>  <a class="code" href="namespaceimpala.html#a47d8255dea5b52dc4c93a11ed02b29b5">CounterMap</a>& scan_ranges_complete = <a class="code" href="namespaceimpala.html#a32793557c7c9e175d833303a8f3df878">coordinator_counters_</a>.<a class="code" href="structimpala_1_1FragmentInstanceCounters.html#ae39f9f7b6955c58f73d95a30d2f4ee1a">scan_ranges_complete_counters</a>;</div> |
| <div class="line"><a name="l01057"></a><span class="lineno"> 1057</span>  CounterMap::iterator it = scan_ranges_complete.find(node_id);</div> |
| <div class="line"><a name="l01058"></a><span class="lineno"> 1058</span>  <span class="keywordflow">if</span> (it != scan_ranges_complete.end()) {</div> |
| <div class="line"><a name="l01059"></a><span class="lineno"> 1059</span>  value += it->second->value();</div> |
| <div class="line"><a name="l01060"></a><span class="lineno"> 1060</span>  }</div> |
| <div class="line"><a name="l01061"></a><span class="lineno"> 1061</span>  <span class="keywordflow">return</span> value;</div> |
| <div class="line"><a name="l01062"></a><span class="lineno"> 1062</span> }</div> |
| <div class="line"><a name="l01063"></a><span class="lineno"> 1063</span> </div> |
| <div class="line"><a name="l01064"></a><span class="lineno"> 1064</span> Status <a class="code" href="namespaceimpala.html#a4abf2193b29fb5607bb924bd14dfa347">Coordinator::ExecRemoteFragment</a>(<span class="keywordtype">void</span>* exec_state_arg) {</div> |
| <div class="line"><a name="l01065"></a><span class="lineno"> 1065</span>  BackendExecState* exec_state = <span class="keyword">reinterpret_cast<</span>BackendExecState*<span class="keyword">></span>(exec_state_arg);</div> |
| <div class="line"><a name="l01066"></a><span class="lineno"> 1066</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"making rpc: ExecPlanFragment query_id="</span> << query_id_</div> |
| <div class="line"><a name="l01067"></a><span class="lineno"> 1067</span>  << <span class="stringliteral">" instance_id="</span> << exec_state->fragment_instance_id</div> |
| <div class="line"><a name="l01068"></a><span class="lineno"> 1068</span>  << <span class="stringliteral">" host="</span> << exec_state->backend_address;</div> |
| <div class="line"><a name="l01069"></a><span class="lineno"> 1069</span>  lock_guard<mutex> l(exec_state->lock);</div> |
| <div class="line"><a name="l01070"></a><span class="lineno"> 1070</span> </div> |
| <div class="line"><a name="l01071"></a><span class="lineno"> 1071</span>  Status status;</div> |
| <div class="line"><a name="l01072"></a><span class="lineno"> 1072</span>  <a class="code" href="namespaceimpala.html#a5d79ffeb06ab02bf3083bfa1eb45d878">ImpalaInternalServiceConnection</a> backend_client(</div> |
| <div class="line"><a name="l01073"></a><span class="lineno"> 1073</span>  <a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a4e470b8606b776eb409005c95c575361">impalad_client_cache</a>(), exec_state->backend_address, &status);</div> |
| <div class="line"><a name="l01074"></a><span class="lineno"> 1074</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(status);</div> |
| <div class="line"><a name="l01075"></a><span class="lineno"> 1075</span> </div> |
| <div class="line"><a name="l01076"></a><span class="lineno"> 1076</span>  TExecPlanFragmentResult thrift_result;</div> |
| <div class="line"><a name="l01077"></a><span class="lineno"> 1077</span>  Status rpc_status = backend_client.DoRpc(&ImpalaInternalServiceClient::ExecPlanFragment,</div> |
| <div class="line"><a name="l01078"></a><span class="lineno"> 1078</span>  exec_state->rpc_params, &thrift_result);</div> |
| <div class="line"><a name="l01079"></a><span class="lineno"> 1079</span>  <span class="keywordflow">if</span> (!rpc_status.ok()) {</div> |
| <div class="line"><a name="l01080"></a><span class="lineno"> 1080</span>  stringstream msg;</div> |
| <div class="line"><a name="l01081"></a><span class="lineno"> 1081</span>  msg << <span class="stringliteral">"ExecPlanRequest rpc query_id="</span> << query_id_</div> |
| <div class="line"><a name="l01082"></a><span class="lineno"> 1082</span>  << <span class="stringliteral">" instance_id="</span> << exec_state->fragment_instance_id</div> |
| <div class="line"><a name="l01083"></a><span class="lineno"> 1083</span>  << <span class="stringliteral">" failed: "</span> << rpc_status.msg().msg();</div> |
| <div class="line"><a name="l01084"></a><span class="lineno"> 1084</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << msg.str();</div> |
| <div class="line"><a name="l01085"></a><span class="lineno"> 1085</span>  exec_state->status = Status(msg.str());</div> |
| <div class="line"><a name="l01086"></a><span class="lineno"> 1086</span>  <span class="keywordflow">return</span> status;</div> |
| <div class="line"><a name="l01087"></a><span class="lineno"> 1087</span>  }</div> |
| <div class="line"><a name="l01088"></a><span class="lineno"> 1088</span> </div> |
| <div class="line"><a name="l01089"></a><span class="lineno"> 1089</span>  exec_state->status = thrift_result.status;</div> |
| <div class="line"><a name="l01090"></a><span class="lineno"> 1090</span>  <span class="keywordflow">if</span> (exec_state->status.ok()) {</div> |
| <div class="line"><a name="l01091"></a><span class="lineno"> 1091</span>  exec_state->initiated = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l01092"></a><span class="lineno"> 1092</span>  exec_state->stopwatch.Start();</div> |
| <div class="line"><a name="l01093"></a><span class="lineno"> 1093</span>  }</div> |
| <div class="line"><a name="l01094"></a><span class="lineno"> 1094</span>  <span class="keywordflow">return</span> exec_state->status;</div> |
| <div class="line"><a name="l01095"></a><span class="lineno"> 1095</span> }</div> |
| <div class="line"><a name="l01096"></a><span class="lineno"> 1096</span> </div> |
| <div class="line"><a name="l01097"></a><span class="lineno"> 1097</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a508481848105bae62dd5722fd747572a">Coordinator::Cancel</a>(<span class="keyword">const</span> Status* cause) {</div> |
| <div class="line"><a name="l01098"></a><span class="lineno"> 1098</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l01099"></a><span class="lineno"> 1099</span>  <span class="comment">// if the query status indicates an error, cancellation has already been initiated</span></div> |
| <div class="line"><a name="l01100"></a><span class="lineno"> 1100</span>  <span class="keywordflow">if</span> (!<a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>()) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l01101"></a><span class="lineno"> 1101</span>  <span class="comment">// prevent others from cancelling a second time</span></div> |
| <div class="line"><a name="l01102"></a><span class="lineno"> 1102</span>  <a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a> = (cause != NULL && !cause->ok()) ? *cause : <a class="code" href="classimpala_1_1Status.html#a318b69bf42d4d23f7ae71a8006efb85b">Status::CANCELLED</a>;</div> |
| <div class="line"><a name="l01103"></a><span class="lineno"> 1103</span>  <a class="code" href="namespaceimpala.html#a721f135364b8f28d9cf09cd45e95745d">CancelInternal</a>();</div> |
| <div class="line"><a name="l01104"></a><span class="lineno"> 1104</span> }</div> |
| <div class="line"><a name="l01105"></a><span class="lineno"> 1105</span> </div> |
| <div class="line"><a name="l01106"></a><span class="lineno"> 1106</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a721f135364b8f28d9cf09cd45e95745d">Coordinator::CancelInternal</a>() {</div> |
| <div class="line"><a name="l01107"></a><span class="lineno"> 1107</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Cancel() query_id="</span> << <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>;</div> |
| <div class="line"><a name="l01108"></a><span class="lineno"> 1108</span>  DCHECK(!<a class="code" href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">query_status_</a>.<a class="code" href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">ok</a>());</div> |
| <div class="line"><a name="l01109"></a><span class="lineno"> 1109</span> </div> |
| <div class="line"><a name="l01110"></a><span class="lineno"> 1110</span>  <span class="comment">// cancel local fragment</span></div> |
| <div class="line"><a name="l01111"></a><span class="lineno"> 1111</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL) <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->Cancel();</div> |
| <div class="line"><a name="l01112"></a><span class="lineno"> 1112</span> </div> |
| <div class="line"><a name="l01113"></a><span class="lineno"> 1113</span>  <a class="code" href="namespaceimpala.html#a693c7d4098922a8f6d32dcc0c03cc9b7">CancelRemoteFragments</a>();</div> |
| <div class="line"><a name="l01114"></a><span class="lineno"> 1114</span> </div> |
| <div class="line"><a name="l01115"></a><span class="lineno"> 1115</span>  <span class="comment">// Report the summary with whatever progress the query made before being cancelled.</span></div> |
| <div class="line"><a name="l01116"></a><span class="lineno"> 1116</span>  <a class="code" href="namespaceimpala.html#a0e6d46f5800bcf4765efd3c21ce535bb">ReportQuerySummary</a>();</div> |
| <div class="line"><a name="l01117"></a><span class="lineno"> 1117</span> }</div> |
| <div class="line"><a name="l01118"></a><span class="lineno"> 1118</span> </div> |
| <div class="line"><a name="l01119"></a><span class="lineno"> 1119</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a693c7d4098922a8f6d32dcc0c03cc9b7">Coordinator::CancelRemoteFragments</a>() {</div> |
| <div class="line"><a name="l01120"></a><span class="lineno"> 1120</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01121"></a><span class="lineno"> 1121</span>  BackendExecState* exec_state = <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i];</div> |
| <div class="line"><a name="l01122"></a><span class="lineno"> 1122</span> </div> |
| <div class="line"><a name="l01123"></a><span class="lineno"> 1123</span>  <span class="comment">// If a fragment failed before we finished issuing all remote fragments,</span></div> |
| <div class="line"><a name="l01124"></a><span class="lineno"> 1124</span>  <span class="comment">// this function will have been called before we finished populating</span></div> |
| <div class="line"><a name="l01125"></a><span class="lineno"> 1125</span>  <span class="comment">// backend_exec_states_. Skip any such uninitialized exec states.</span></div> |
| <div class="line"><a name="l01126"></a><span class="lineno"> 1126</span>  <span class="keywordflow">if</span> (exec_state == NULL) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01127"></a><span class="lineno"> 1127</span> </div> |
| <div class="line"><a name="l01128"></a><span class="lineno"> 1128</span>  <span class="comment">// lock each exec_state individually to synchronize correctly with</span></div> |
| <div class="line"><a name="l01129"></a><span class="lineno"> 1129</span>  <span class="comment">// UpdateFragmentExecStatus() (which doesn't get the global lock_</span></div> |
| <div class="line"><a name="l01130"></a><span class="lineno"> 1130</span>  <span class="comment">// to set its status)</span></div> |
| <div class="line"><a name="l01131"></a><span class="lineno"> 1131</span>  lock_guard<mutex> l(exec_state->lock);</div> |
| <div class="line"><a name="l01132"></a><span class="lineno"> 1132</span> </div> |
| <div class="line"><a name="l01133"></a><span class="lineno"> 1133</span>  <span class="comment">// no need to cancel if we already know it terminated w/ an error status</span></div> |
| <div class="line"><a name="l01134"></a><span class="lineno"> 1134</span>  <span class="keywordflow">if</span> (!exec_state->status.ok()) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01135"></a><span class="lineno"> 1135</span> </div> |
| <div class="line"><a name="l01136"></a><span class="lineno"> 1136</span>  <span class="comment">// Nothing to cancel if the exec rpc was not sent</span></div> |
| <div class="line"><a name="l01137"></a><span class="lineno"> 1137</span>  <span class="keywordflow">if</span> (!exec_state->initiated) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01138"></a><span class="lineno"> 1138</span> </div> |
| <div class="line"><a name="l01139"></a><span class="lineno"> 1139</span>  <span class="comment">// don't cancel if it already finished</span></div> |
| <div class="line"><a name="l01140"></a><span class="lineno"> 1140</span>  <span class="keywordflow">if</span> (exec_state->done) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01141"></a><span class="lineno"> 1141</span> </div> |
| <div class="line"><a name="l01142"></a><span class="lineno"> 1142</span>  <span class="comment">// set an error status to make sure we only cancel this once</span></div> |
| <div class="line"><a name="l01143"></a><span class="lineno"> 1143</span>  exec_state->status = <a class="code" href="classimpala_1_1Status.html#a318b69bf42d4d23f7ae71a8006efb85b">Status::CANCELLED</a>;</div> |
| <div class="line"><a name="l01144"></a><span class="lineno"> 1144</span> </div> |
| <div class="line"><a name="l01145"></a><span class="lineno"> 1145</span>  <span class="comment">// if we get an error while trying to get a connection to the backend,</span></div> |
| <div class="line"><a name="l01146"></a><span class="lineno"> 1146</span>  <span class="comment">// keep going</span></div> |
| <div class="line"><a name="l01147"></a><span class="lineno"> 1147</span>  Status status;</div> |
| <div class="line"><a name="l01148"></a><span class="lineno"> 1148</span>  <a class="code" href="namespaceimpala.html#a5d79ffeb06ab02bf3083bfa1eb45d878">ImpalaInternalServiceConnection</a> backend_client(</div> |
| <div class="line"><a name="l01149"></a><span class="lineno"> 1149</span>  <a class="code" href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">exec_env_</a>-><a class="code" href="classimpala_1_1ExecEnv.html#a4e470b8606b776eb409005c95c575361">impalad_client_cache</a>(), exec_state->backend_address, &status);</div> |
| <div class="line"><a name="l01150"></a><span class="lineno"> 1150</span>  <span class="keywordflow">if</span> (!status.ok()) {</div> |
| <div class="line"><a name="l01151"></a><span class="lineno"> 1151</span>  <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01152"></a><span class="lineno"> 1152</span>  }</div> |
| <div class="line"><a name="l01153"></a><span class="lineno"> 1153</span> </div> |
| <div class="line"><a name="l01154"></a><span class="lineno"> 1154</span>  TCancelPlanFragmentParams params;</div> |
| <div class="line"><a name="l01155"></a><span class="lineno"> 1155</span>  params.protocol_version = ImpalaInternalServiceVersion::V1;</div> |
| <div class="line"><a name="l01156"></a><span class="lineno"> 1156</span>  params.__set_fragment_instance_id(exec_state->fragment_instance_id);</div> |
| <div class="line"><a name="l01157"></a><span class="lineno"> 1157</span>  TCancelPlanFragmentResult res;</div> |
| <div class="line"><a name="l01158"></a><span class="lineno"> 1158</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"sending CancelPlanFragment rpc for instance_id="</span></div> |
| <div class="line"><a name="l01159"></a><span class="lineno"> 1159</span>  << exec_state->fragment_instance_id << <span class="stringliteral">" backend="</span></div> |
| <div class="line"><a name="l01160"></a><span class="lineno"> 1160</span>  << exec_state->backend_address;</div> |
| <div class="line"><a name="l01161"></a><span class="lineno"> 1161</span>  Status rpc_status = backend_client.DoRpc(</div> |
| <div class="line"><a name="l01162"></a><span class="lineno"> 1162</span>  &ImpalaInternalServiceClient::CancelPlanFragment, params, &res);</div> |
| <div class="line"><a name="l01163"></a><span class="lineno"> 1163</span>  <span class="keywordflow">if</span> (!rpc_status.ok()) {</div> |
| <div class="line"><a name="l01164"></a><span class="lineno"> 1164</span>  exec_state->status.MergeStatus(rpc_status);</div> |
| <div class="line"><a name="l01165"></a><span class="lineno"> 1165</span>  stringstream msg;</div> |
| <div class="line"><a name="l01166"></a><span class="lineno"> 1166</span>  msg << <span class="stringliteral">"CancelPlanFragment rpc query_id="</span> << query_id_</div> |
| <div class="line"><a name="l01167"></a><span class="lineno"> 1167</span>  << <span class="stringliteral">" instance_id="</span> << exec_state->fragment_instance_id</div> |
| <div class="line"><a name="l01168"></a><span class="lineno"> 1168</span>  << <span class="stringliteral">" failed: "</span> << rpc_status.msg().msg();</div> |
| <div class="line"><a name="l01169"></a><span class="lineno"> 1169</span>  <span class="comment">// make a note of the error status, but keep on cancelling the other fragments</span></div> |
| <div class="line"><a name="l01170"></a><span class="lineno"> 1170</span>  exec_state->status.AddDetail(msg.str());</div> |
| <div class="line"><a name="l01171"></a><span class="lineno"> 1171</span>  <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01172"></a><span class="lineno"> 1172</span>  }</div> |
| <div class="line"><a name="l01173"></a><span class="lineno"> 1173</span>  <span class="keywordflow">if</span> (res.status.status_code != <a class="code" href="namespaceimpala.html#ac55a10ca0171687156609e8d6ba28127a45b8e4259988c3518a05b9202efb0898">TErrorCode::OK</a>) {</div> |
| <div class="line"><a name="l01174"></a><span class="lineno"> 1174</span>  exec_state->status.AddDetail(join(res.status.error_msgs, <span class="stringliteral">"; "</span>));</div> |
| <div class="line"><a name="l01175"></a><span class="lineno"> 1175</span>  }</div> |
| <div class="line"><a name="l01176"></a><span class="lineno"> 1176</span>  }</div> |
| <div class="line"><a name="l01177"></a><span class="lineno"> 1177</span> </div> |
| <div class="line"><a name="l01178"></a><span class="lineno"> 1178</span>  <span class="comment">// notify that we completed with an error</span></div> |
| <div class="line"><a name="l01179"></a><span class="lineno"> 1179</span>  <a class="code" href="namespaceimpala.html#a6a1bc3d15242d20a00589ed2fd3c6066">backend_completion_cv_</a>.notify_all();</div> |
| <div class="line"><a name="l01180"></a><span class="lineno"> 1180</span> }</div> |
| <div class="line"><a name="l01181"></a><span class="lineno"> 1181</span> </div> |
| <div class="line"><a name="l01182"></a><span class="lineno"> 1182</span> Status <a class="code" href="namespaceimpala.html#a169a05f020e28fcf0fc6dedcbd06361a">Coordinator::UpdateFragmentExecStatus</a>(<span class="keyword">const</span> TReportExecStatusParams& params) {</div> |
| <div class="line"><a name="l01183"></a><span class="lineno"> 1183</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"UpdateFragmentExecStatus() query_id="</span> << query_id_</div> |
| <div class="line"><a name="l01184"></a><span class="lineno"> 1184</span>  << <span class="stringliteral">" status="</span> << params.status.status_code</div> |
| <div class="line"><a name="l01185"></a><span class="lineno"> 1185</span>  << <span class="stringliteral">" done="</span> << (params.done ? <span class="stringliteral">"true"</span> : <span class="stringliteral">"false"</span>);</div> |
| <div class="line"><a name="l01186"></a><span class="lineno"> 1186</span>  <span class="keywordflow">if</span> (params.backend_num >= <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size()) {</div> |
| <div class="line"><a name="l01187"></a><span class="lineno"> 1187</span>  <span class="keywordflow">return</span> Status(TErrorCode::INTERNAL_ERROR, <span class="stringliteral">"unknown backend number"</span>);</div> |
| <div class="line"><a name="l01188"></a><span class="lineno"> 1188</span>  }</div> |
| <div class="line"><a name="l01189"></a><span class="lineno"> 1189</span>  BackendExecState* exec_state = <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[params.backend_num];</div> |
| <div class="line"><a name="l01190"></a><span class="lineno"> 1190</span> </div> |
| <div class="line"><a name="l01191"></a><span class="lineno"> 1191</span>  <span class="keyword">const</span> TRuntimeProfileTree& cumulative_profile = params.profile;</div> |
| <div class="line"><a name="l01192"></a><span class="lineno"> 1192</span>  Status status(params.status);</div> |
| <div class="line"><a name="l01193"></a><span class="lineno"> 1193</span>  {</div> |
| <div class="line"><a name="l01194"></a><span class="lineno"> 1194</span>  lock_guard<mutex> l(exec_state->lock);</div> |
| <div class="line"><a name="l01195"></a><span class="lineno"> 1195</span>  <span class="keywordflow">if</span> (!status.ok()) {</div> |
| <div class="line"><a name="l01196"></a><span class="lineno"> 1196</span>  <span class="comment">// During query cancellation, exec_state is set to CANCELLED. However, we might</span></div> |
| <div class="line"><a name="l01197"></a><span class="lineno"> 1197</span>  <span class="comment">// process a non-error message from a fragment executor that is sent</span></div> |
| <div class="line"><a name="l01198"></a><span class="lineno"> 1198</span>  <span class="comment">// before query cancellation is invoked. Make sure we don't go from error status to</span></div> |
| <div class="line"><a name="l01199"></a><span class="lineno"> 1199</span>  <span class="comment">// OK.</span></div> |
| <div class="line"><a name="l01200"></a><span class="lineno"> 1200</span>  exec_state->status = status;</div> |
| <div class="line"><a name="l01201"></a><span class="lineno"> 1201</span>  }</div> |
| <div class="line"><a name="l01202"></a><span class="lineno"> 1202</span>  exec_state->done = params.done;</div> |
| <div class="line"><a name="l01203"></a><span class="lineno"> 1203</span>  <span class="keywordflow">if</span> (exec_state->status.ok()) {</div> |
| <div class="line"><a name="l01204"></a><span class="lineno"> 1204</span>  <span class="comment">// We can't update this backend's profile if ReportQuerySummary() is running,</span></div> |
| <div class="line"><a name="l01205"></a><span class="lineno"> 1205</span>  <span class="comment">// because it depends on all profiles not changing during its execution (when it</span></div> |
| <div class="line"><a name="l01206"></a><span class="lineno"> 1206</span>  <span class="comment">// calls SortChildren()). ReportQuerySummary() only gets called after</span></div> |
| <div class="line"><a name="l01207"></a><span class="lineno"> 1207</span>  <span class="comment">// WaitForAllBackends() returns or at the end of CancelRemoteFragments().</span></div> |
| <div class="line"><a name="l01208"></a><span class="lineno"> 1208</span>  <span class="comment">// WaitForAllBackends() only returns after all backends have completed (in which</span></div> |
| <div class="line"><a name="l01209"></a><span class="lineno"> 1209</span>  <span class="comment">// case we wouldn't be in this function), or when there's an error, in which case</span></div> |
| <div class="line"><a name="l01210"></a><span class="lineno"> 1210</span>  <span class="comment">// CancelRemoteFragments() is called. CancelRemoteFragments sets all exec_state's</span></div> |
| <div class="line"><a name="l01211"></a><span class="lineno"> 1211</span>  <span class="comment">// statuses to cancelled.</span></div> |
| <div class="line"><a name="l01212"></a><span class="lineno"> 1212</span>  <span class="comment">// TODO: We're losing this profile information. Call ReportQuerySummary only after</span></div> |
| <div class="line"><a name="l01213"></a><span class="lineno"> 1213</span>  <span class="comment">// all backends have completed.</span></div> |
| <div class="line"><a name="l01214"></a><span class="lineno"> 1214</span>  exec_state->profile->Update(cumulative_profile);</div> |
| <div class="line"><a name="l01215"></a><span class="lineno"> 1215</span> </div> |
| <div class="line"><a name="l01216"></a><span class="lineno"> 1216</span>  <span class="comment">// Update the average profile for the fragment corresponding to this instance.</span></div> |
| <div class="line"><a name="l01217"></a><span class="lineno"> 1217</span>  exec_state->profile->ComputeTimeInProfile();</div> |
| <div class="line"><a name="l01218"></a><span class="lineno"> 1218</span>  <a class="code" href="namespaceimpala.html#af8a4757a740abbb7e521cf91b2d6cc75">UpdateAverageProfile</a>(exec_state);</div> |
| <div class="line"><a name="l01219"></a><span class="lineno"> 1219</span>  <a class="code" href="namespaceimpala.html#ae85655273cec6d52b5d875516bc212f8">UpdateExecSummary</a>(exec_state->fragment_idx, exec_state->instance_idx,</div> |
| <div class="line"><a name="l01220"></a><span class="lineno"> 1220</span>  exec_state->profile);</div> |
| <div class="line"><a name="l01221"></a><span class="lineno"> 1221</span>  }</div> |
| <div class="line"><a name="l01222"></a><span class="lineno"> 1222</span>  <span class="keywordflow">if</span> (!exec_state->profile_created) {</div> |
| <div class="line"><a name="l01223"></a><span class="lineno"> 1223</span>  <a class="code" href="namespaceimpala.html#a20d70f7b44aeb530802e3ccc198f19fa">CollectScanNodeCounters</a>(exec_state->profile, &exec_state->aggregate_counters);</div> |
| <div class="line"><a name="l01224"></a><span class="lineno"> 1224</span>  }</div> |
| <div class="line"><a name="l01225"></a><span class="lineno"> 1225</span>  exec_state->profile_created = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l01226"></a><span class="lineno"> 1226</span> </div> |
| <div class="line"><a name="l01227"></a><span class="lineno"> 1227</span>  <span class="comment">// Log messages aggregated by type</span></div> |
| <div class="line"><a name="l01228"></a><span class="lineno"> 1228</span>  <span class="keywordflow">if</span> (params.__isset.error_log && params.error_log.size() > 0) {</div> |
| <div class="line"><a name="l01229"></a><span class="lineno"> 1229</span>  <span class="comment">// Append the log messages from each update with the global state of the query</span></div> |
| <div class="line"><a name="l01230"></a><span class="lineno"> 1230</span>  <span class="comment">// execution</span></div> |
| <div class="line"><a name="l01231"></a><span class="lineno"> 1231</span>  <a class="code" href="namespaceimpala.html#acc0060a61afd94d6b1cf84ae7b1b6280">MergeErrorMaps</a>(&exec_state->error_log, params.error_log);</div> |
| <div class="line"><a name="l01232"></a><span class="lineno"> 1232</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"instance_id="</span> << exec_state->fragment_instance_id</div> |
| <div class="line"><a name="l01233"></a><span class="lineno"> 1233</span>  << <span class="stringliteral">" error log: "</span> << <a class="code" href="namespaceimpala.html#acb1aecd358146a66f5d9d02835822cd1">PrintErrorMapToString</a>(exec_state->error_log);</div> |
| <div class="line"><a name="l01234"></a><span class="lineno"> 1234</span>  }</div> |
| <div class="line"><a name="l01235"></a><span class="lineno"> 1235</span>  <a class="code" href="namespaceimpala.html#ab3b57d53bc4b6adfed05b760c72e0faf">progress_</a>.<a class="code" href="classimpala_1_1ProgressUpdater.html#a3e05b2a32fd1bdf32d2a275c10fc431f">Update</a>(exec_state->UpdateNumScanRangesCompleted());</div> |
| <div class="line"><a name="l01236"></a><span class="lineno"> 1236</span>  }</div> |
| <div class="line"><a name="l01237"></a><span class="lineno"> 1237</span> </div> |
| <div class="line"><a name="l01238"></a><span class="lineno"> 1238</span>  <span class="keywordflow">if</span> (params.done && params.__isset.insert_exec_status) {</div> |
| <div class="line"><a name="l01239"></a><span class="lineno"> 1239</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l01240"></a><span class="lineno"> 1240</span>  <span class="comment">// Merge in table update data (partitions written to, files to be moved as part of</span></div> |
| <div class="line"><a name="l01241"></a><span class="lineno"> 1241</span>  <span class="comment">// finalization)</span></div> |
| <div class="line"><a name="l01242"></a><span class="lineno"> 1242</span>  BOOST_FOREACH(<span class="keyword">const</span> PartitionStatusMap::value_type& partition,</div> |
| <div class="line"><a name="l01243"></a><span class="lineno"> 1243</span>  params.insert_exec_status.per_partition_status) {</div> |
| <div class="line"><a name="l01244"></a><span class="lineno"> 1244</span>  TInsertPartitionStatus* status = &(<a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a>[partition.first]);</div> |
| <div class="line"><a name="l01245"></a><span class="lineno"> 1245</span>  status->num_appended_rows += partition.second.num_appended_rows;</div> |
| <div class="line"><a name="l01246"></a><span class="lineno"> 1246</span>  status->id = partition.second.id;</div> |
| <div class="line"><a name="l01247"></a><span class="lineno"> 1247</span>  <span class="keywordflow">if</span> (!status->__isset.stats) status->__set_stats(TInsertStats());</div> |
| <div class="line"><a name="l01248"></a><span class="lineno"> 1248</span>  <a class="code" href="classimpala_1_1DataSink.html#ab95c1680a2c851bed71977e502139b32">DataSink::MergeInsertStats</a>(partition.second.stats, &status->stats);</div> |
| <div class="line"><a name="l01249"></a><span class="lineno"> 1249</span>  }</div> |
| <div class="line"><a name="l01250"></a><span class="lineno"> 1250</span>  files_to_move_.insert(</div> |
| <div class="line"><a name="l01251"></a><span class="lineno"> 1251</span>  params.insert_exec_status.files_to_move.begin(),</div> |
| <div class="line"><a name="l01252"></a><span class="lineno"> 1252</span>  params.insert_exec_status.files_to_move.end());</div> |
| <div class="line"><a name="l01253"></a><span class="lineno"> 1253</span>  }</div> |
| <div class="line"><a name="l01254"></a><span class="lineno"> 1254</span> </div> |
| <div class="line"><a name="l01255"></a><span class="lineno"> 1255</span>  <span class="keywordflow">if</span> (<a class="code" href="logging_8h.html#a6f29cadf91ddc478eb9fa4e02cd8e0a0">VLOG_FILE_IS_ON</a>) {</div> |
| <div class="line"><a name="l01256"></a><span class="lineno"> 1256</span>  stringstream s;</div> |
| <div class="line"><a name="l01257"></a><span class="lineno"> 1257</span>  exec_state->profile->PrettyPrint(&s);</div> |
| <div class="line"><a name="l01258"></a><span class="lineno"> 1258</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"profile for query_id="</span> << query_id_</div> |
| <div class="line"><a name="l01259"></a><span class="lineno"> 1259</span>  << <span class="stringliteral">" instance_id="</span> << exec_state->fragment_instance_id</div> |
| <div class="line"><a name="l01260"></a><span class="lineno"> 1260</span>  << <span class="stringliteral">"\n"</span> << s.str();</div> |
| <div class="line"><a name="l01261"></a><span class="lineno"> 1261</span>  }</div> |
| <div class="line"><a name="l01262"></a><span class="lineno"> 1262</span>  <span class="comment">// also print the cumulative profile</span></div> |
| <div class="line"><a name="l01263"></a><span class="lineno"> 1263</span>  <span class="comment">// TODO: fix the coordinator/PlanFragmentExecutor, so this isn't needed</span></div> |
| <div class="line"><a name="l01264"></a><span class="lineno"> 1264</span>  <span class="keywordflow">if</span> (<a class="code" href="logging_8h.html#a6f29cadf91ddc478eb9fa4e02cd8e0a0">VLOG_FILE_IS_ON</a>) {</div> |
| <div class="line"><a name="l01265"></a><span class="lineno"> 1265</span>  stringstream s;</div> |
| <div class="line"><a name="l01266"></a><span class="lineno"> 1266</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->PrettyPrint(&s);</div> |
| <div class="line"><a name="l01267"></a><span class="lineno"> 1267</span>  <a class="code" href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a> << <span class="stringliteral">"cumulative profile for query_id="</span> << query_id_</div> |
| <div class="line"><a name="l01268"></a><span class="lineno"> 1268</span>  << <span class="stringliteral">"\n"</span> << s.str();</div> |
| <div class="line"><a name="l01269"></a><span class="lineno"> 1269</span>  }</div> |
| <div class="line"><a name="l01270"></a><span class="lineno"> 1270</span> </div> |
| <div class="line"><a name="l01271"></a><span class="lineno"> 1271</span>  <span class="comment">// for now, abort the query if we see any error except if the error is cancelled</span></div> |
| <div class="line"><a name="l01272"></a><span class="lineno"> 1272</span>  <span class="comment">// and returned_all_results_ is true.</span></div> |
| <div class="line"><a name="l01273"></a><span class="lineno"> 1273</span>  <span class="comment">// (UpdateStatus() initiates cancellation, if it hasn't already been initiated)</span></div> |
| <div class="line"><a name="l01274"></a><span class="lineno"> 1274</span>  <span class="keywordflow">if</span> (!(<a class="code" href="namespaceimpala.html#aea293e10bddb1360a019cd1e9abf45f2">returned_all_results_</a> && status.IsCancelled()) && !status.ok()) {</div> |
| <div class="line"><a name="l01275"></a><span class="lineno"> 1275</span>  <a class="code" href="namespaceimpala.html#aba23271e9c88b75efea05a2afb25856b">UpdateStatus</a>(status, &exec_state->fragment_instance_id);</div> |
| <div class="line"><a name="l01276"></a><span class="lineno"> 1276</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l01277"></a><span class="lineno"> 1277</span>  }</div> |
| <div class="line"><a name="l01278"></a><span class="lineno"> 1278</span> </div> |
| <div class="line"><a name="l01279"></a><span class="lineno"> 1279</span>  <span class="keywordflow">if</span> (params.done) {</div> |
| <div class="line"><a name="l01280"></a><span class="lineno"> 1280</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l01281"></a><span class="lineno"> 1281</span>  exec_state->stopwatch.Stop();</div> |
| <div class="line"><a name="l01282"></a><span class="lineno"> 1282</span>  DCHECK_GT(num_remaining_backends_, 0);</div> |
| <div class="line"><a name="l01283"></a><span class="lineno"> 1283</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"Backend "</span> << params.backend_num << <span class="stringliteral">" completed, "</span></div> |
| <div class="line"><a name="l01284"></a><span class="lineno"> 1284</span>  << num_remaining_backends_ - 1 << <span class="stringliteral">" remaining: query_id="</span> << <a class="code" href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">query_id_</a>;</div> |
| <div class="line"><a name="l01285"></a><span class="lineno"> 1285</span>  <span class="keywordflow">if</span> (<a class="code" href="logging_8h.html#ac976d11f99eb284f949261fcee44dffd">VLOG_QUERY_IS_ON</a> && num_remaining_backends_ > 1) {</div> |
| <div class="line"><a name="l01286"></a><span class="lineno"> 1286</span>  <span class="comment">// print host/port info for the first backend that's still in progress as a</span></div> |
| <div class="line"><a name="l01287"></a><span class="lineno"> 1287</span>  <span class="comment">// debugging aid for backend deadlocks</span></div> |
| <div class="line"><a name="l01288"></a><span class="lineno"> 1288</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01289"></a><span class="lineno"> 1289</span>  BackendExecState* exec_state = <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i];</div> |
| <div class="line"><a name="l01290"></a><span class="lineno"> 1290</span>  lock_guard<mutex> l2(exec_state->lock);</div> |
| <div class="line"><a name="l01291"></a><span class="lineno"> 1291</span>  <span class="keywordflow">if</span> (!exec_state->done) {</div> |
| <div class="line"><a name="l01292"></a><span class="lineno"> 1292</span>  <a class="code" href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a> << <span class="stringliteral">"query_id="</span> << query_id_ << <span class="stringliteral">": first in-progress backend: "</span></div> |
| <div class="line"><a name="l01293"></a><span class="lineno"> 1293</span>  << exec_state->backend_address;</div> |
| <div class="line"><a name="l01294"></a><span class="lineno"> 1294</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l01295"></a><span class="lineno"> 1295</span>  }</div> |
| <div class="line"><a name="l01296"></a><span class="lineno"> 1296</span>  }</div> |
| <div class="line"><a name="l01297"></a><span class="lineno"> 1297</span>  }</div> |
| <div class="line"><a name="l01298"></a><span class="lineno"> 1298</span>  <span class="keywordflow">if</span> (--num_remaining_backends_ == 0) {</div> |
| <div class="line"><a name="l01299"></a><span class="lineno"> 1299</span>  <a class="code" href="namespaceimpala.html#a6a1bc3d15242d20a00589ed2fd3c6066">backend_completion_cv_</a>.notify_all();</div> |
| <div class="line"><a name="l01300"></a><span class="lineno"> 1300</span>  }</div> |
| <div class="line"><a name="l01301"></a><span class="lineno"> 1301</span>  }</div> |
| <div class="line"><a name="l01302"></a><span class="lineno"> 1302</span> </div> |
| <div class="line"><a name="l01303"></a><span class="lineno"> 1303</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l01304"></a><span class="lineno"> 1304</span> }</div> |
| <div class="line"><a name="l01305"></a><span class="lineno"> 1305</span> </div> |
| <div class="line"><a name="l01306"></a><span class="lineno"> 1306</span> <span class="keyword">const</span> RowDescriptor& <a class="code" href="namespaceimpala.html#aa98447566dd6700a2faaaaf3059f4d95">Coordinator::row_desc</a>()<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l01307"></a><span class="lineno"> 1307</span>  DCHECK(<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL);</div> |
| <div class="line"><a name="l01308"></a><span class="lineno"> 1308</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->row_desc();</div> |
| <div class="line"><a name="l01309"></a><span class="lineno"> 1309</span> }</div> |
| <div class="line"><a name="l01310"></a><span class="lineno"> 1310</span> </div> |
| <div class="line"><a name="l01311"></a><span class="lineno"> 1311</span> RuntimeState* <a class="code" href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">Coordinator::runtime_state</a>() {</div> |
| <div class="line"><a name="l01312"></a><span class="lineno"> 1312</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() == NULL ? NULL : <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->runtime_state();</div> |
| <div class="line"><a name="l01313"></a><span class="lineno"> 1313</span> }</div> |
| <div class="line"><a name="l01314"></a><span class="lineno"> 1314</span> </div> |
| <div class="line"><a name="l01315"></a><span class="lineno"> 1315</span> MemTracker* <a class="code" href="namespaceimpala.html#a7ab8f672190232021bdfcf200502140c">Coordinator::query_mem_tracker</a>() {</div> |
| <div class="line"><a name="l01316"></a><span class="lineno"> 1316</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() == NULL ? query_mem_tracker_.get() :</div> |
| <div class="line"><a name="l01317"></a><span class="lineno"> 1317</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->runtime_state()->query_mem_tracker();</div> |
| <div class="line"><a name="l01318"></a><span class="lineno"> 1318</span> }</div> |
| <div class="line"><a name="l01319"></a><span class="lineno"> 1319</span> </div> |
| <div class="line"><a name="l01320"></a><span class="lineno"> 1320</span> <span class="keywordtype">bool</span> <a class="code" href="namespaceimpala.html#a4f03145f4a655319c059d8e57397d0cd">Coordinator::PrepareCatalogUpdate</a>(TUpdateCatalogRequest* catalog_update) {</div> |
| <div class="line"><a name="l01321"></a><span class="lineno"> 1321</span>  <span class="comment">// Assume we are called only after all fragments have completed</span></div> |
| <div class="line"><a name="l01322"></a><span class="lineno"> 1322</span>  DCHECK(<a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a>);</div> |
| <div class="line"><a name="l01323"></a><span class="lineno"> 1323</span> </div> |
| <div class="line"><a name="l01324"></a><span class="lineno"> 1324</span>  BOOST_FOREACH(<span class="keyword">const</span> PartitionStatusMap::value_type& partition, <a class="code" href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">per_partition_status_</a>) {</div> |
| <div class="line"><a name="l01325"></a><span class="lineno"> 1325</span>  catalog_update->created_partitions.insert(partition.first);</div> |
| <div class="line"><a name="l01326"></a><span class="lineno"> 1326</span>  }</div> |
| <div class="line"><a name="l01327"></a><span class="lineno"> 1327</span> </div> |
| <div class="line"><a name="l01328"></a><span class="lineno"> 1328</span>  <span class="keywordflow">return</span> catalog_update->created_partitions.size() != 0;</div> |
| <div class="line"><a name="l01329"></a><span class="lineno"> 1329</span> }</div> |
| <div class="line"><a name="l01330"></a><span class="lineno"> 1330</span> </div> |
| <div class="line"><a name="l01331"></a><span class="lineno"> 1331</span> <span class="comment">// Comparator to order fragments by descending total time</span></div> |
| <div class="line"><a name="l01332"></a><span class="lineno"><a class="line" href="structimpala_1_1InstanceComparator.html"> 1332</a></span> <span class="keyword">typedef</span> <span class="keyword">struct </span>{</div> |
| <div class="line"><a name="l01333"></a><span class="lineno"><a class="line" href="structimpala_1_1InstanceComparator.html#a97427bbefe6128d674ddfd45e15fbff7"> 1333</a></span>  <span class="keyword">typedef</span> pair<RuntimeProfile*, bool> <a class="code" href="structimpala_1_1InstanceComparator.html#a97427bbefe6128d674ddfd45e15fbff7">Profile</a>;</div> |
| <div class="line"><a name="l01334"></a><span class="lineno"><a class="line" href="structimpala_1_1InstanceComparator.html#a75d19a7a9a8c3b061b027a9d91427f57"> 1334</a></span>  <span class="keywordtype">bool</span> <a class="code" href="structimpala_1_1InstanceComparator.html#a75d19a7a9a8c3b061b027a9d91427f57">operator()</a>(<span class="keyword">const</span> Profile& a, <span class="keyword">const</span> Profile& b)<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l01335"></a><span class="lineno"> 1335</span>  <span class="comment">// Reverse ordering: we want the longest first</span></div> |
| <div class="line"><a name="l01336"></a><span class="lineno"> 1336</span>  <span class="keywordflow">return</span></div> |
| <div class="line"><a name="l01337"></a><span class="lineno"> 1337</span>  a.first->total_time_counter()->value() > b.first->total_time_counter()->value();</div> |
| <div class="line"><a name="l01338"></a><span class="lineno"> 1338</span>  }</div> |
| <div class="line"><a name="l01339"></a><span class="lineno"> 1339</span> } InstanceComparator;</div> |
| <div class="line"><a name="l01340"></a><span class="lineno"> 1340</span> </div> |
| <div class="line"><a name="l01341"></a><span class="lineno"> 1341</span> <span class="comment">// Update fragment average profile information from a backend execution state.</span></div> |
| <div class="line"><a name="l01342"></a><span class="lineno"> 1342</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#af8a4757a740abbb7e521cf91b2d6cc75">Coordinator::UpdateAverageProfile</a>(BackendExecState* backend_exec_state) {</div> |
| <div class="line"><a name="l01343"></a><span class="lineno"> 1343</span>  <span class="keywordtype">int</span> fragment_idx = backend_exec_state->fragment_idx;</div> |
| <div class="line"><a name="l01344"></a><span class="lineno"> 1344</span>  DCHECK_GE(fragment_idx, 0);</div> |
| <div class="line"><a name="l01345"></a><span class="lineno"> 1345</span>  DCHECK_LT(fragment_idx, <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>.size());</div> |
| <div class="line"><a name="l01346"></a><span class="lineno"> 1346</span>  PerFragmentProfileData& data = <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[fragment_idx];</div> |
| <div class="line"><a name="l01347"></a><span class="lineno"> 1347</span> </div> |
| <div class="line"><a name="l01348"></a><span class="lineno"> 1348</span>  <span class="comment">// No locks are taken since UpdateAverage() and AddChild() take their own locks</span></div> |
| <div class="line"><a name="l01349"></a><span class="lineno"> 1349</span>  data.averaged_profile->UpdateAverage(backend_exec_state->profile);</div> |
| <div class="line"><a name="l01350"></a><span class="lineno"> 1350</span>  data.root_profile->AddChild(backend_exec_state->profile);</div> |
| <div class="line"><a name="l01351"></a><span class="lineno"> 1351</span> }</div> |
| <div class="line"><a name="l01352"></a><span class="lineno"> 1352</span> </div> |
| <div class="line"><a name="l01353"></a><span class="lineno"> 1353</span> <span class="comment">// Compute fragment summary information from a backend execution state.</span></div> |
| <div class="line"><a name="l01354"></a><span class="lineno"> 1354</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#af1eec3a76356f0cf625d434c21d6dd0d">Coordinator::ComputeFragmentSummaryStats</a>(BackendExecState* backend_exec_state) {</div> |
| <div class="line"><a name="l01355"></a><span class="lineno"> 1355</span>  <span class="keywordtype">int</span> fragment_idx = backend_exec_state->fragment_idx;</div> |
| <div class="line"><a name="l01356"></a><span class="lineno"> 1356</span>  DCHECK_GE(fragment_idx, 0);</div> |
| <div class="line"><a name="l01357"></a><span class="lineno"> 1357</span>  DCHECK_LT(fragment_idx, <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>.size());</div> |
| <div class="line"><a name="l01358"></a><span class="lineno"> 1358</span>  PerFragmentProfileData& data = <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[fragment_idx];</div> |
| <div class="line"><a name="l01359"></a><span class="lineno"> 1359</span> </div> |
| <div class="line"><a name="l01360"></a><span class="lineno"> 1360</span>  int64_t completion_time = backend_exec_state->stopwatch.ElapsedTime();</div> |
| <div class="line"><a name="l01361"></a><span class="lineno"> 1361</span>  data.completion_times(completion_time);</div> |
| <div class="line"><a name="l01362"></a><span class="lineno"> 1362</span>  data.rates(backend_exec_state->total_split_size / (completion_time / 1000.0</div> |
| <div class="line"><a name="l01363"></a><span class="lineno"> 1363</span>  / 1000.0 / 1000.0));</div> |
| <div class="line"><a name="l01364"></a><span class="lineno"> 1364</span> </div> |
| <div class="line"><a name="l01365"></a><span class="lineno"> 1365</span>  <span class="comment">// Add the child in case it has not been added previously</span></div> |
| <div class="line"><a name="l01366"></a><span class="lineno"> 1366</span>  <span class="comment">// via UpdateAverageProfile(). AddChild() will do nothing if the child</span></div> |
| <div class="line"><a name="l01367"></a><span class="lineno"> 1367</span>  <span class="comment">// already exists.</span></div> |
| <div class="line"><a name="l01368"></a><span class="lineno"> 1368</span>  data.root_profile->AddChild(backend_exec_state->profile);</div> |
| <div class="line"><a name="l01369"></a><span class="lineno"> 1369</span> }</div> |
| <div class="line"><a name="l01370"></a><span class="lineno"> 1370</span> </div> |
| <div class="line"><a name="l01371"></a><span class="lineno"> 1371</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#ae85655273cec6d52b5d875516bc212f8">Coordinator::UpdateExecSummary</a>(<span class="keywordtype">int</span> fragment_idx, <span class="keywordtype">int</span> instance_idx,</div> |
| <div class="line"><a name="l01372"></a><span class="lineno"> 1372</span>  RuntimeProfile* profile) {</div> |
| <div class="line"><a name="l01373"></a><span class="lineno"> 1373</span>  vector<RuntimeProfile*> children;</div> |
| <div class="line"><a name="l01374"></a><span class="lineno"> 1374</span>  profile->GetAllChildren(&children);</div> |
| <div class="line"><a name="l01375"></a><span class="lineno"> 1375</span> </div> |
| <div class="line"><a name="l01376"></a><span class="lineno"> 1376</span>  lock_guard<SpinLock> l(<a class="code" href="namespaceimpala.html#a3589885d3d8ec7b6e4ee6e0a51a89491">exec_summary_lock_</a>);</div> |
| <div class="line"><a name="l01377"></a><span class="lineno"> 1377</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < children.size(); ++i) {</div> |
| <div class="line"><a name="l01378"></a><span class="lineno"> 1378</span>  <span class="keywordtype">int</span> <span class="keywordtype">id</span> = <a class="code" href="classimpala_1_1ExecNode.html#a3e2d144b60f904f6b1f3219b9471aad3">ExecNode::GetNodeIdFromProfile</a>(children[i]);</div> |
| <div class="line"><a name="l01379"></a><span class="lineno"> 1379</span>  <span class="keywordflow">if</span> (<span class="keywordtype">id</span> == -1) <span class="keywordflow">continue</span>;</div> |
| <div class="line"><a name="l01380"></a><span class="lineno"> 1380</span> </div> |
| <div class="line"><a name="l01381"></a><span class="lineno"> 1381</span>  TPlanNodeExecSummary& <a class="code" href="namespaceimpala.html#a87319b8665a64adda1e0908ebc6963e5">exec_summary</a> =</div> |
| <div class="line"><a name="l01382"></a><span class="lineno"> 1382</span>  <a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>.nodes[<a class="code" href="namespaceimpala.html#a574dcc25c5967c10d13e4bf3fa4c9bcf">plan_node_id_to_summary_map_</a>[id]];</div> |
| <div class="line"><a name="l01383"></a><span class="lineno"> 1383</span>  <span class="keywordflow">if</span> (exec_summary.exec_stats.empty()) {</div> |
| <div class="line"><a name="l01384"></a><span class="lineno"> 1384</span>  <span class="comment">// First time, make an exec_stats for each instance this plan node is running on.</span></div> |
| <div class="line"><a name="l01385"></a><span class="lineno"> 1385</span>  DCHECK_LT(fragment_idx, <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>.size());</div> |
| <div class="line"><a name="l01386"></a><span class="lineno"> 1386</span>  exec_summary.exec_stats.resize(<a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[fragment_idx].num_instances);</div> |
| <div class="line"><a name="l01387"></a><span class="lineno"> 1387</span>  }</div> |
| <div class="line"><a name="l01388"></a><span class="lineno"> 1388</span>  DCHECK_LT(instance_idx, exec_summary.exec_stats.size());</div> |
| <div class="line"><a name="l01389"></a><span class="lineno"> 1389</span>  TExecStats& stats = exec_summary.exec_stats[instance_idx];</div> |
| <div class="line"><a name="l01390"></a><span class="lineno"> 1390</span> </div> |
| <div class="line"><a name="l01391"></a><span class="lineno"> 1391</span>  RuntimeProfile::Counter* rows_counter = children[i]->GetCounter(<span class="stringliteral">"RowsReturned"</span>);</div> |
| <div class="line"><a name="l01392"></a><span class="lineno"> 1392</span>  RuntimeProfile::Counter* mem_counter = children[i]->GetCounter(<span class="stringliteral">"PeakMemoryUsage"</span>);</div> |
| <div class="line"><a name="l01393"></a><span class="lineno"> 1393</span>  <span class="keywordflow">if</span> (rows_counter != NULL) stats.__set_cardinality(rows_counter->value());</div> |
| <div class="line"><a name="l01394"></a><span class="lineno"> 1394</span>  <span class="keywordflow">if</span> (mem_counter != NULL) stats.__set_memory_used(mem_counter->value());</div> |
| <div class="line"><a name="l01395"></a><span class="lineno"> 1395</span>  stats.__set_latency_ns(children[i]->local_time());</div> |
| <div class="line"><a name="l01396"></a><span class="lineno"> 1396</span>  <span class="comment">// TODO: we don't track cpu time per node now. Do that.</span></div> |
| <div class="line"><a name="l01397"></a><span class="lineno"> 1397</span>  exec_summary.__isset.exec_stats = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l01398"></a><span class="lineno"> 1398</span>  }</div> |
| <div class="line"><a name="l01399"></a><span class="lineno"> 1399</span>  VLOG(2) << <a class="code" href="namespaceimpala.html#a90daa83f383514b15451c6e5031deef9">PrintExecSummary</a>(<a class="code" href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">exec_summary_</a>);</div> |
| <div class="line"><a name="l01400"></a><span class="lineno"> 1400</span> }</div> |
| <div class="line"><a name="l01401"></a><span class="lineno"> 1401</span> </div> |
| <div class="line"><a name="l01402"></a><span class="lineno"> 1402</span> <span class="comment">// This function appends summary information to the query_profile_ before</span></div> |
| <div class="line"><a name="l01403"></a><span class="lineno"> 1403</span> <span class="comment">// outputting it to VLOG. It adds:</span></div> |
| <div class="line"><a name="l01404"></a><span class="lineno"> 1404</span> <span class="comment">// 1. Averaged remote fragment profiles (TODO: add outliers)</span></div> |
| <div class="line"><a name="l01405"></a><span class="lineno"> 1405</span> <span class="comment">// 2. Summary of remote fragment durations (min, max, mean, stddev)</span></div> |
| <div class="line"><a name="l01406"></a><span class="lineno"> 1406</span> <span class="comment">// 3. Summary of remote fragment rates (min, max, mean, stddev)</span></div> |
| <div class="line"><a name="l01407"></a><span class="lineno"> 1407</span> <span class="comment">// TODO: add histogram/percentile</span></div> |
| <div class="line"><a name="l01408"></a><span class="lineno"> 1408</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#a0e6d46f5800bcf4765efd3c21ce535bb">Coordinator::ReportQuerySummary</a>() {</div> |
| <div class="line"><a name="l01409"></a><span class="lineno"> 1409</span>  <span class="comment">// In this case, the query did not even get to start on all the remote nodes,</span></div> |
| <div class="line"><a name="l01410"></a><span class="lineno"> 1410</span>  <span class="comment">// some of the state that is used below might be uninitialized. In this case,</span></div> |
| <div class="line"><a name="l01411"></a><span class="lineno"> 1411</span>  <span class="comment">// the query has made so little progress, reporting a summary is not very useful.</span></div> |
| <div class="line"><a name="l01412"></a><span class="lineno"> 1412</span>  <span class="keywordflow">if</span> (!<a class="code" href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">has_called_wait_</a>) <span class="keywordflow">return</span>;</div> |
| <div class="line"><a name="l01413"></a><span class="lineno"> 1413</span> </div> |
| <div class="line"><a name="l01414"></a><span class="lineno"> 1414</span>  <span class="comment">// The fragment has finished executing. Update the profile to compute the</span></div> |
| <div class="line"><a name="l01415"></a><span class="lineno"> 1415</span>  <span class="comment">// fraction of time spent in each node.</span></div> |
| <div class="line"><a name="l01416"></a><span class="lineno"> 1416</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL) {</div> |
| <div class="line"><a name="l01417"></a><span class="lineno"> 1417</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile()->ComputeTimeInProfile();</div> |
| <div class="line"><a name="l01418"></a><span class="lineno"> 1418</span>  <a class="code" href="namespaceimpala.html#ae85655273cec6d52b5d875516bc212f8">UpdateExecSummary</a>(0, 0, <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile());</div> |
| <div class="line"><a name="l01419"></a><span class="lineno"> 1419</span>  }</div> |
| <div class="line"><a name="l01420"></a><span class="lineno"> 1420</span> </div> |
| <div class="line"><a name="l01421"></a><span class="lineno"> 1421</span>  <span class="keywordflow">if</span> (!<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.empty()) {</div> |
| <div class="line"><a name="l01422"></a><span class="lineno"> 1422</span>  <span class="comment">// Average all remote fragments for each fragment.</span></div> |
| <div class="line"><a name="l01423"></a><span class="lineno"> 1423</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01424"></a><span class="lineno"> 1424</span>  <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->profile->ComputeTimeInProfile();</div> |
| <div class="line"><a name="l01425"></a><span class="lineno"> 1425</span>  <a class="code" href="namespaceimpala.html#af8a4757a740abbb7e521cf91b2d6cc75">UpdateAverageProfile</a>(<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]);</div> |
| <div class="line"><a name="l01426"></a><span class="lineno"> 1426</span>  <a class="code" href="namespaceimpala.html#af1eec3a76356f0cf625d434c21d6dd0d">ComputeFragmentSummaryStats</a>(<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]);</div> |
| <div class="line"><a name="l01427"></a><span class="lineno"> 1427</span>  <a class="code" href="namespaceimpala.html#ae85655273cec6d52b5d875516bc212f8">UpdateExecSummary</a>(<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->fragment_idx,</div> |
| <div class="line"><a name="l01428"></a><span class="lineno"> 1428</span>  <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->instance_idx, <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->profile);</div> |
| <div class="line"><a name="l01429"></a><span class="lineno"> 1429</span>  }</div> |
| <div class="line"><a name="l01430"></a><span class="lineno"> 1430</span> </div> |
| <div class="line"><a name="l01431"></a><span class="lineno"> 1431</span>  InstanceComparator comparator;</div> |
| <div class="line"><a name="l01432"></a><span class="lineno"> 1432</span>  <span class="comment">// Per fragment instances have been collected, output summaries</span></div> |
| <div class="line"><a name="l01433"></a><span class="lineno"> 1433</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL ? 1 : 0); i < <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01434"></a><span class="lineno"> 1434</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].root_profile->SortChildren(comparator);</div> |
| <div class="line"><a name="l01435"></a><span class="lineno"> 1435</span>  <a class="code" href="namespaceimpala.html#ad4f56de7b31b5daea6ea0e854715925e">SummaryStats</a>& completion_times = <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].completion_times;</div> |
| <div class="line"><a name="l01436"></a><span class="lineno"> 1436</span>  <a class="code" href="namespaceimpala.html#ad4f56de7b31b5daea6ea0e854715925e">SummaryStats</a>& rates = <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].rates;</div> |
| <div class="line"><a name="l01437"></a><span class="lineno"> 1437</span> </div> |
| <div class="line"><a name="l01438"></a><span class="lineno"> 1438</span>  stringstream times_label;</div> |
| <div class="line"><a name="l01439"></a><span class="lineno"> 1439</span>  times_label</div> |
| <div class="line"><a name="l01440"></a><span class="lineno"> 1440</span>  << <span class="stringliteral">"min:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01441"></a><span class="lineno"> 1441</span>  accumulators::min(completion_times), TUnit::TIME_NS)</div> |
| <div class="line"><a name="l01442"></a><span class="lineno"> 1442</span>  << <span class="stringliteral">" max:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01443"></a><span class="lineno"> 1443</span>  accumulators::max(completion_times), TUnit::TIME_NS)</div> |
| <div class="line"><a name="l01444"></a><span class="lineno"> 1444</span>  << <span class="stringliteral">" mean: "</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01445"></a><span class="lineno"> 1445</span>  accumulators::mean(completion_times), TUnit::TIME_NS)</div> |
| <div class="line"><a name="l01446"></a><span class="lineno"> 1446</span>  << <span class="stringliteral">" stddev:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01447"></a><span class="lineno"> 1447</span>  sqrt(accumulators::variance(completion_times)), TUnit::TIME_NS);</div> |
| <div class="line"><a name="l01448"></a><span class="lineno"> 1448</span> </div> |
| <div class="line"><a name="l01449"></a><span class="lineno"> 1449</span>  stringstream rates_label;</div> |
| <div class="line"><a name="l01450"></a><span class="lineno"> 1450</span>  rates_label</div> |
| <div class="line"><a name="l01451"></a><span class="lineno"> 1451</span>  << <span class="stringliteral">"min:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01452"></a><span class="lineno"> 1452</span>  accumulators::min(rates), TUnit::BYTES_PER_SECOND)</div> |
| <div class="line"><a name="l01453"></a><span class="lineno"> 1453</span>  << <span class="stringliteral">" max:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01454"></a><span class="lineno"> 1454</span>  accumulators::max(rates), TUnit::BYTES_PER_SECOND)</div> |
| <div class="line"><a name="l01455"></a><span class="lineno"> 1455</span>  << <span class="stringliteral">" mean:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01456"></a><span class="lineno"> 1456</span>  accumulators::mean(rates), TUnit::BYTES_PER_SECOND)</div> |
| <div class="line"><a name="l01457"></a><span class="lineno"> 1457</span>  << <span class="stringliteral">" stddev:"</span> << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(</div> |
| <div class="line"><a name="l01458"></a><span class="lineno"> 1458</span>  sqrt(accumulators::variance(rates)), TUnit::BYTES_PER_SECOND);</div> |
| <div class="line"><a name="l01459"></a><span class="lineno"> 1459</span> </div> |
| <div class="line"><a name="l01460"></a><span class="lineno"> 1460</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile->AddInfoString(</div> |
| <div class="line"><a name="l01461"></a><span class="lineno"> 1461</span>  <span class="stringliteral">"completion times"</span>, times_label.str());</div> |
| <div class="line"><a name="l01462"></a><span class="lineno"> 1462</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile->AddInfoString(</div> |
| <div class="line"><a name="l01463"></a><span class="lineno"> 1463</span>  <span class="stringliteral">"execution rates"</span>, rates_label.str());</div> |
| <div class="line"><a name="l01464"></a><span class="lineno"> 1464</span>  <a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].averaged_profile->AddInfoString(</div> |
| <div class="line"><a name="l01465"></a><span class="lineno"> 1465</span>  <span class="stringliteral">"num instances"</span>, lexical_cast<string>(<a class="code" href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">fragment_profiles_</a>[i].num_instances));</div> |
| <div class="line"><a name="l01466"></a><span class="lineno"> 1466</span>  }</div> |
| <div class="line"><a name="l01467"></a><span class="lineno"> 1467</span> </div> |
| <div class="line"><a name="l01468"></a><span class="lineno"> 1468</span>  <span class="comment">// Add per node peak memory usage as InfoString</span></div> |
| <div class="line"><a name="l01469"></a><span class="lineno"> 1469</span>  <span class="comment">// Map from Impalad address to peak memory usage of this query</span></div> |
| <div class="line"><a name="l01470"></a><span class="lineno"> 1470</span>  <span class="keyword">typedef</span> boost::unordered_map<TNetworkAddress, int64_t> PerNodePeakMemoryUsage;</div> |
| <div class="line"><a name="l01471"></a><span class="lineno"> 1471</span>  PerNodePeakMemoryUsage per_node_peak_mem_usage;</div> |
| <div class="line"><a name="l01472"></a><span class="lineno"> 1472</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL) {</div> |
| <div class="line"><a name="l01473"></a><span class="lineno"> 1473</span>  <span class="comment">// Coordinator fragment is not included in backend_exec_states_.</span></div> |
| <div class="line"><a name="l01474"></a><span class="lineno"> 1474</span>  RuntimeProfile::Counter* mem_usage_counter =</div> |
| <div class="line"><a name="l01475"></a><span class="lineno"> 1475</span>  <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->profile()->GetCounter(</div> |
| <div class="line"><a name="l01476"></a><span class="lineno"> 1476</span>  <a class="code" href="classimpala_1_1PlanFragmentExecutor.html#ac12ad0f6d8bb9ddbb62d58e04a412c93">PlanFragmentExecutor::PER_HOST_PEAK_MEM_COUNTER</a>);</div> |
| <div class="line"><a name="l01477"></a><span class="lineno"> 1477</span>  <span class="keywordflow">if</span> (mem_usage_counter != NULL) {</div> |
| <div class="line"><a name="l01478"></a><span class="lineno"> 1478</span>  TNetworkAddress coord = <a class="code" href="namespaceimpala.html#a309108c8b3eaf4e5b154bc4eb4624880">MakeNetworkAddress</a>(FLAGS_hostname, FLAGS_be_port);</div> |
| <div class="line"><a name="l01479"></a><span class="lineno"> 1479</span>  per_node_peak_mem_usage[coord] = mem_usage_counter->value();</div> |
| <div class="line"><a name="l01480"></a><span class="lineno"> 1480</span>  }</div> |
| <div class="line"><a name="l01481"></a><span class="lineno"> 1481</span>  }</div> |
| <div class="line"><a name="l01482"></a><span class="lineno"> 1482</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01483"></a><span class="lineno"> 1483</span>  int64_t initial_usage = 0;</div> |
| <div class="line"><a name="l01484"></a><span class="lineno"> 1484</span>  int64_t* mem_usage = <a class="code" href="namespaceimpala.html#ac79f8d29c9aaf2dddb617ae5e36ca23f">FindOrInsert</a>(&per_node_peak_mem_usage,</div> |
| <div class="line"><a name="l01485"></a><span class="lineno"> 1485</span>  <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->backend_address, initial_usage);</div> |
| <div class="line"><a name="l01486"></a><span class="lineno"> 1486</span>  RuntimeProfile::Counter* mem_usage_counter =</div> |
| <div class="line"><a name="l01487"></a><span class="lineno"> 1487</span>  <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->profile->GetCounter(</div> |
| <div class="line"><a name="l01488"></a><span class="lineno"> 1488</span>  <a class="code" href="classimpala_1_1PlanFragmentExecutor.html#ac12ad0f6d8bb9ddbb62d58e04a412c93">PlanFragmentExecutor::PER_HOST_PEAK_MEM_COUNTER</a>);</div> |
| <div class="line"><a name="l01489"></a><span class="lineno"> 1489</span>  <span class="keywordflow">if</span> (mem_usage_counter != NULL && mem_usage_counter->value() > *mem_usage) {</div> |
| <div class="line"><a name="l01490"></a><span class="lineno"> 1490</span>  per_node_peak_mem_usage[<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->backend_address] =</div> |
| <div class="line"><a name="l01491"></a><span class="lineno"> 1491</span>  mem_usage_counter->value();</div> |
| <div class="line"><a name="l01492"></a><span class="lineno"> 1492</span>  }</div> |
| <div class="line"><a name="l01493"></a><span class="lineno"> 1493</span>  }</div> |
| <div class="line"><a name="l01494"></a><span class="lineno"> 1494</span>  stringstream info;</div> |
| <div class="line"><a name="l01495"></a><span class="lineno"> 1495</span>  BOOST_FOREACH(PerNodePeakMemoryUsage::value_type entry, per_node_peak_mem_usage) {</div> |
| <div class="line"><a name="l01496"></a><span class="lineno"> 1496</span>  info << entry.first << <span class="stringliteral">"("</span></div> |
| <div class="line"><a name="l01497"></a><span class="lineno"> 1497</span>  << <a class="code" href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">PrettyPrinter::Print</a>(entry.second, TUnit::BYTES) << <span class="stringliteral">") "</span>;</div> |
| <div class="line"><a name="l01498"></a><span class="lineno"> 1498</span>  }</div> |
| <div class="line"><a name="l01499"></a><span class="lineno"> 1499</span>  <a class="code" href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">query_profile_</a>->AddInfoString(<span class="stringliteral">"Per Node Peak Memory Usage"</span>, info.str());</div> |
| <div class="line"><a name="l01500"></a><span class="lineno"> 1500</span>  }</div> |
| <div class="line"><a name="l01501"></a><span class="lineno"> 1501</span> }</div> |
| <div class="line"><a name="l01502"></a><span class="lineno"> 1502</span> </div> |
| <div class="line"><a name="l01503"></a><span class="lineno"> 1503</span> <span class="keywordtype">string</span> <a class="code" href="namespaceimpala.html#afdc2218a15eac469107b45c5db2dd289">Coordinator::GetErrorLog</a>() {</div> |
| <div class="line"><a name="l01504"></a><span class="lineno"> 1504</span>  <a class="code" href="namespaceimpala.html#a940125179d49c976f146709006813049">ErrorLogMap</a> merged;</div> |
| <div class="line"><a name="l01505"></a><span class="lineno"> 1505</span>  {</div> |
| <div class="line"><a name="l01506"></a><span class="lineno"> 1506</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">lock_</a>);</div> |
| <div class="line"><a name="l01507"></a><span class="lineno"> 1507</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>.get() != NULL && <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->runtime_state() != NULL &&</div> |
| <div class="line"><a name="l01508"></a><span class="lineno"> 1508</span>  !<a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->runtime_state()->ErrorLogIsEmpty()) {</div> |
| <div class="line"><a name="l01509"></a><span class="lineno"> 1509</span>  <a class="code" href="namespaceimpala.html#acc0060a61afd94d6b1cf84ae7b1b6280">MergeErrorMaps</a>(&merged, <a class="code" href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">executor_</a>->runtime_state()->error_log());</div> |
| <div class="line"><a name="l01510"></a><span class="lineno"> 1510</span>  }</div> |
| <div class="line"><a name="l01511"></a><span class="lineno"> 1511</span>  }</div> |
| <div class="line"><a name="l01512"></a><span class="lineno"> 1512</span>  <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i < <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>.size(); ++i) {</div> |
| <div class="line"><a name="l01513"></a><span class="lineno"> 1513</span>  lock_guard<mutex> l(<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->lock);</div> |
| <div class="line"><a name="l01514"></a><span class="lineno"> 1514</span>  <span class="keywordflow">if</span> (<a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->error_log.size() > 0) {</div> |
| <div class="line"><a name="l01515"></a><span class="lineno"> 1515</span>  <a class="code" href="namespaceimpala.html#acc0060a61afd94d6b1cf84ae7b1b6280">MergeErrorMaps</a>(&merged, <a class="code" href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">backend_exec_states_</a>[i]->error_log);</div> |
| <div class="line"><a name="l01516"></a><span class="lineno"> 1516</span>  }</div> |
| <div class="line"><a name="l01517"></a><span class="lineno"> 1517</span>  }</div> |
| <div class="line"><a name="l01518"></a><span class="lineno"> 1518</span>  <span class="keywordflow">return</span> <a class="code" href="namespaceimpala.html#acb1aecd358146a66f5d9d02835822cd1">PrintErrorMapToString</a>(merged);</div> |
| <div class="line"><a name="l01519"></a><span class="lineno"> 1519</span> }</div> |
| <div class="line"><a name="l01520"></a><span class="lineno"> 1520</span> </div> |
| <div class="line"><a name="l01521"></a><span class="lineno"> 1521</span> <span class="keywordtype">void</span> <a class="code" href="namespaceimpala.html#ad9d7363b9825635e741e2d45e1d944fa">Coordinator::SetExecPlanFragmentParams</a>(</div> |
| <div class="line"><a name="l01522"></a><span class="lineno"> 1522</span>  QuerySchedule& schedule, <span class="keywordtype">int</span> backend_num, <span class="keyword">const</span> TPlanFragment& fragment,</div> |
| <div class="line"><a name="l01523"></a><span class="lineno"> 1523</span>  <span class="keywordtype">int</span> fragment_idx, <span class="keyword">const</span> FragmentExecParams& params, <span class="keywordtype">int</span> instance_idx,</div> |
| <div class="line"><a name="l01524"></a><span class="lineno"> 1524</span>  <span class="keyword">const</span> TNetworkAddress& coord, TExecPlanFragmentParams* rpc_params) {</div> |
| <div class="line"><a name="l01525"></a><span class="lineno"> 1525</span>  rpc_params->__set_protocol_version(ImpalaInternalServiceVersion::V1);</div> |
| <div class="line"><a name="l01526"></a><span class="lineno"> 1526</span>  rpc_params->__set_fragment(fragment);</div> |
| <div class="line"><a name="l01527"></a><span class="lineno"> 1527</span>  rpc_params->__set_desc_tbl(<a class="code" href="namespaceimpala.html#a0a06145acd27005a1c1f482dc2375742">desc_tbl_</a>);</div> |
| <div class="line"><a name="l01528"></a><span class="lineno"> 1528</span>  TNetworkAddress exec_host = params.hosts[instance_idx];</div> |
| <div class="line"><a name="l01529"></a><span class="lineno"> 1529</span>  <span class="keywordflow">if</span> (schedule.HasReservation()) {</div> |
| <div class="line"><a name="l01530"></a><span class="lineno"> 1530</span>  <span class="comment">// The reservation has already have been validated at this point.</span></div> |
| <div class="line"><a name="l01531"></a><span class="lineno"> 1531</span>  TNetworkAddress resource_hostport;</div> |
| <div class="line"><a name="l01532"></a><span class="lineno"> 1532</span>  schedule.GetResourceHostport(exec_host, &resource_hostport);</div> |
| <div class="line"><a name="l01533"></a><span class="lineno"> 1533</span>  map<TNetworkAddress, llama::TAllocatedResource>::const_iterator it =</div> |
| <div class="line"><a name="l01534"></a><span class="lineno"> 1534</span>  schedule.reservation()->allocated_resources.find(resource_hostport);</div> |
| <div class="line"><a name="l01535"></a><span class="lineno"> 1535</span>  <span class="comment">// Only set reserved resource if we actually have one for this plan</span></div> |
| <div class="line"><a name="l01536"></a><span class="lineno"> 1536</span>  <span class="comment">// fragment. Otherwise, don't set it (usually this the coordinator fragment), and it</span></div> |
| <div class="line"><a name="l01537"></a><span class="lineno"> 1537</span>  <span class="comment">// won't participate in dynamic RM controls.</span></div> |
| <div class="line"><a name="l01538"></a><span class="lineno"> 1538</span>  <span class="keywordflow">if</span> (it != schedule.reservation()->allocated_resources.end()) {</div> |
| <div class="line"><a name="l01539"></a><span class="lineno"> 1539</span>  rpc_params->__set_reserved_resource(it->second);</div> |
| <div class="line"><a name="l01540"></a><span class="lineno"> 1540</span>  rpc_params->__set_local_resource_address(resource_hostport);</div> |
| <div class="line"><a name="l01541"></a><span class="lineno"> 1541</span>  }</div> |
| <div class="line"><a name="l01542"></a><span class="lineno"> 1542</span>  }</div> |
| <div class="line"><a name="l01543"></a><span class="lineno"> 1543</span>  rpc_params->params.__set_request_pool(schedule.request_pool());</div> |
| <div class="line"><a name="l01544"></a><span class="lineno"> 1544</span>  FragmentScanRangeAssignment::const_iterator it =</div> |
| <div class="line"><a name="l01545"></a><span class="lineno"> 1545</span>  params.scan_range_assignment.find(exec_host);</div> |
| <div class="line"><a name="l01546"></a><span class="lineno"> 1546</span>  <span class="comment">// Scan ranges may not always be set, so use an empty structure if so.</span></div> |
| <div class="line"><a name="l01547"></a><span class="lineno"> 1547</span>  <span class="keyword">const</span> <a class="code" href="namespaceimpala.html#a48e0e1c5d4b43432450c92750fb35cb3">PerNodeScanRanges</a>& scan_ranges =</div> |
| <div class="line"><a name="l01548"></a><span class="lineno"> 1548</span>  (it != params.scan_range_assignment.end()) ? it->second : <a class="code" href="namespaceimpala.html#a48e0e1c5d4b43432450c92750fb35cb3">PerNodeScanRanges</a>();</div> |
| <div class="line"><a name="l01549"></a><span class="lineno"> 1549</span> </div> |
| <div class="line"><a name="l01550"></a><span class="lineno"> 1550</span>  rpc_params->params.__set_per_node_scan_ranges(scan_ranges);</div> |
| <div class="line"><a name="l01551"></a><span class="lineno"> 1551</span>  rpc_params->params.__set_per_exch_num_senders(params.per_exch_num_senders);</div> |
| <div class="line"><a name="l01552"></a><span class="lineno"> 1552</span>  rpc_params->params.__set_destinations(params.destinations);</div> |
| <div class="line"><a name="l01553"></a><span class="lineno"> 1553</span>  rpc_params->params.__set_sender_id(params.sender_id_base + instance_idx);</div> |
| <div class="line"><a name="l01554"></a><span class="lineno"> 1554</span>  rpc_params->__isset.params = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l01555"></a><span class="lineno"> 1555</span>  rpc_params->fragment_instance_ctx.__set_query_ctx(<a class="code" href="namespaceimpala.html#ab296eb278342117dde26e2e28bbe8190">query_ctx_</a>);</div> |
| <div class="line"><a name="l01556"></a><span class="lineno"> 1556</span>  rpc_params->fragment_instance_ctx.fragment_instance_id =</div> |
| <div class="line"><a name="l01557"></a><span class="lineno"> 1557</span>  params.instance_ids[instance_idx];</div> |
| <div class="line"><a name="l01558"></a><span class="lineno"> 1558</span>  rpc_params->fragment_instance_ctx.fragment_instance_idx = instance_idx;</div> |
| <div class="line"><a name="l01559"></a><span class="lineno"> 1559</span>  rpc_params->fragment_instance_ctx.num_fragment_instances = params.instance_ids.size();</div> |
| <div class="line"><a name="l01560"></a><span class="lineno"> 1560</span>  rpc_params->fragment_instance_ctx.backend_num = backend_num;</div> |
| <div class="line"><a name="l01561"></a><span class="lineno"> 1561</span>  rpc_params->__isset.fragment_instance_ctx = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l01562"></a><span class="lineno"> 1562</span> }</div> |
| <div class="line"><a name="l01563"></a><span class="lineno"> 1563</span> </div> |
| <div class="line"><a name="l01564"></a><span class="lineno"> 1564</span> }</div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_1_1Counter_html_a6db04a7861baf700e025bea45de5a2e1"><div class="ttname"><a href="classimpala_1_1RuntimeProfile_1_1Counter.html#a6db04a7861baf700e025bea45de5a2e1">impala::RuntimeProfile::Counter::value</a></div><div class="ttdeci">virtual int64_t value() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00108">runtime-profile.h:108</a></div></div> |
| <div class="ttc" id="classimpala_1_1MemTracker_html_a5b39ede97dce30ef1c7c00b117887a56"><div class="ttname"><a href="classimpala_1_1MemTracker.html#a5b39ede97dce30ef1c7c00b117887a56">impala::MemTracker::GetQueryMemTracker</a></div><div class="ttdeci">static boost::shared_ptr< MemTracker > GetQueryMemTracker(const TUniqueId &id, int64_t byte_limit, int64_t rm_reserved_limit, MemTracker *parent, QueryResourceMgr *res_mgr)</div><div class="ttdef"><b>Definition:</b> <a href="mem-tracker_8cc_source.html#l00156">mem-tracker.cc:156</a></div></div> |
| <div class="ttc" id="row-batch_8h_html"><div class="ttname"><a href="row-batch_8h.html">row-batch.h</a></div></div> |
| <div class="ttc" id="structimpala_1_1DebugOptions_html_afde33d0a06fa1cd97549f04c5c797c73"><div class="ttname"><a href="structimpala_1_1DebugOptions.html#afde33d0a06fa1cd97549f04c5c797c73">impala::DebugOptions::phase</a></div><div class="ttdeci">TExecNodePhase::type phase</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00093">coordinator.cc:93</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a169a05f020e28fcf0fc6dedcbd06361a"><div class="ttname"><a href="namespaceimpala.html#a169a05f020e28fcf0fc6dedcbd06361a">impala::UpdateFragmentExecStatus</a></div><div class="ttdeci">Status UpdateFragmentExecStatus(const TReportExecStatusParams &params)</div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_aed65f272a2672410ec13d5fa02d2b7a3"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#aed65f272a2672410ec13d5fa02d2b7a3">impala::Coordinator::BackendExecState::initiated</a></div><div class="ttdeci">bool initiated</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00130">coordinator.cc:130</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html">impala::Coordinator::BackendExecState</a></div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00104">coordinator.cc:104</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_acb1aecd358146a66f5d9d02835822cd1"><div class="ttname"><a href="namespaceimpala.html#acb1aecd358146a66f5d9d02835822cd1">impala::PrintErrorMapToString</a></div><div class="ttdeci">string PrintErrorMapToString(const ErrorLogMap &errors)</div><div class="ttdef"><b>Definition:</b> <a href="error-util_8cc_source.html#l00153">error-util.cc:153</a></div></div> |
| <div class="ttc" id="sasl-test_8cc_html_a2db66f7b463c0e190f8933e51f43d91e"><div class="ttname"><a href="sasl-test_8cc.html#a2db66f7b463c0e190f8933e51f43d91e">path</a></div><div class="ttdeci">string path("/usr/lib/sasl2:/usr/lib64/sasl2:/usr/local/lib/sasl2:/usr/lib/x86_64-linux-gnu/sasl2")</div></div> |
| <div class="ttc" id="namespaceimpala_html_aca80061c98b44477ea84e4332993b7e7"><div class="ttname"><a href="namespaceimpala.html#aca80061c98b44477ea84e4332993b7e7">impala::PlanNodeId</a></div><div class="ttdeci">int PlanNodeId</div><div class="ttdef"><b>Definition:</b> <a href="global-types_8h_source.html#l00026">global-types.h:26</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a33916d0527ad4e5ed5fb5b8089e90888"><div class="ttname"><a href="namespaceimpala.html#a33916d0527ad4e5ed5fb5b8089e90888">impala::~Coordinator</a></div><div class="ttdeci">~Coordinator()</div></div> |
| <div class="ttc" id="namespaceimpala_html_a98367e13193bbe0af1fc61a4422b7e6fa8dc9bcd2611acd50486660a42f7feec1"><div class="ttname"><a href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa8dc9bcd2611acd50486660a42f7feec1">impala::DELETE_THEN_CREATE</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-bulk-ops_8h_source.html#l00032">hdfs-bulk-ops.h:32</a></div></div> |
| <div class="ttc" id="coordinator_8h_html"><div class="ttname"><a href="coordinator_8h.html">coordinator.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1MonotonicStopWatch_html"><div class="ttname"><a href="classimpala_1_1MonotonicStopWatch.html">impala::MonotonicStopWatch</a></div><div class="ttdef"><b>Definition:</b> <a href="stopwatch_8h_source.html#l00074">stopwatch.h:74</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_af13668dfa75d4affc4f3823c30395aad"><div class="ttname"><a href="namespaceimpala.html#af13668dfa75d4affc4f3823c30395aad">impala::runtime_state</a></div><div class="ttdeci">RuntimeState * runtime_state()</div><div class="ttdoc">only valid after calling Exec(), and may return NULL if there is no executor </div></div> |
| <div class="ttc" id="namespaceimpala_html_a4abf2193b29fb5607bb924bd14dfa347"><div class="ttname"><a href="namespaceimpala.html#a4abf2193b29fb5607bb924bd14dfa347">impala::ExecRemoteFragment</a></div><div class="ttdeci">Status ExecRemoteFragment(void *exec_state)</div></div> |
| <div class="ttc" id="namespaceimpala_html_a5368f0a1443b0bd404d98b27107678a4"><div class="ttname"><a href="namespaceimpala.html#a5368f0a1443b0bd404d98b27107678a4">impala::events</a></div><div class="ttdeci">client RuntimeProfile::EventSequence * events</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00064">coordinator.h:64</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_af279911558f793413306452d5b16e6d6"><div class="ttname"><a href="namespaceimpala.html#af279911558f793413306452d5b16e6d6">impala::GetNext</a></div><div class="ttdeci">Status GetNext(RowBatch **batch, RuntimeState *state)</div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a947d4f7961a00d4b92cb3061d4b51f67"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a947d4f7961a00d4b92cb3061d4b51f67">impala::Coordinator::BackendExecState::rpc_params</a></div><div class="ttdeci">TExecPlanFragmentParams rpc_params</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00112">coordinator.cc:112</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a2a3a06b5063753bc8d23f612d4d88bba"><div class="ttname"><a href="namespaceimpala.html#a2a3a06b5063753bc8d23f612d4d88bba">impala::finalize_params_</a></div><div class="ttdeci">TFinalizeParams finalize_params_</div><div class="ttdoc">Only valid if needs_finalization is true. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00222">coordinator.h:222</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScanNode_html_ab56d0e228f9b992bf4eb15a24808be43"><div class="ttname"><a href="classimpala_1_1ScanNode.html#ab56d0e228f9b992bf4eb15a24808be43">impala::ScanNode::TOTAL_THROUGHPUT_COUNTER</a></div><div class="ttdeci">static const std::string TOTAL_THROUGHPUT_COUNTER</div><div class="ttdef"><b>Definition:</b> <a href="scan-node_8h_source.html#l00125">scan-node.h:125</a></div></div> |
| <div class="ttc" id="structimpala_1_1FragmentInstanceCounters_html"><div class="ttname"><a href="structimpala_1_1FragmentInstanceCounters.html">impala::FragmentInstanceCounters</a></div><div class="ttdoc">Struct for per fragment instance counters that will be aggregated by the coordinator. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00207">coordinator.h:207</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ab3b57d53bc4b6adfed05b760c72e0faf"><div class="ttname"><a href="namespaceimpala.html#ab3b57d53bc4b6adfed05b760c72e0faf">impala::progress_</a></div><div class="ttdeci">ProgressUpdater progress_</div><div class="ttdoc">Keeps track of number of completed ranges and total scan ranges. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00230">coordinator.h:230</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExecNode_html_a3e2d144b60f904f6b1f3219b9471aad3"><div class="ttname"><a href="classimpala_1_1ExecNode.html#a3e2d144b60f904f6b1f3219b9471aad3">impala::ExecNode::GetNodeIdFromProfile</a></div><div class="ttdeci">static int GetNodeIdFromProfile(RuntimeProfile *p)</div><div class="ttdoc">Extract node id from p->name(). </div><div class="ttdef"><b>Definition:</b> <a href="exec-node_8cc_source.html#l00062">exec-node.cc:62</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_acf144f0a59543984e935c4afb3525cbc"><div class="ttname"><a href="namespaceimpala.html#acf144f0a59543984e935c4afb3525cbc">impala::query_id_</a></div><div class="ttdeci">TUniqueId query_id_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00194">coordinator.h:194</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a94595ccb59d6b9ee3bf56ebb49097c4d"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a94595ccb59d6b9ee3bf56ebb49097c4d">impala::Coordinator::BackendExecState::lock</a></div><div class="ttdeci">boost::mutex lock</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00123">coordinator.cc:123</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a1c183dc242f42291fbf439e78bebc8fa"><div class="ttname"><a href="namespaceimpala.html#a1c183dc242f42291fbf439e78bebc8fa">impala::obj_pool_</a></div><div class="ttdeci">boost::scoped_ptr< ObjectPool > obj_pool_</div><div class="ttdoc">Object pool owned by the coordinator. Any executor will have its own pool. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00296">coordinator.h:296</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aba8224df5d033a6fc4c3cd908df76886"><div class="ttname"><a href="namespaceimpala.html#aba8224df5d033a6fc4c3cd908df76886">impala::exec_summary_</a></div><div class="ttdeci">TExecSummary exec_summary_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00300">coordinator.h:300</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aadc20c1e4da4986703816cb21b3ecf44"><div class="ttname"><a href="namespaceimpala.html#aadc20c1e4da4986703816cb21b3ecf44">impala::CreateAggregateCounters</a></div><div class="ttdeci">void CreateAggregateCounters(const std::vector< TPlanFragment > &fragments)</div><div class="ttdoc">Create aggregate counters for all scan nodes in any of the fragments. </div></div> |
| <div class="ttc" id="status_8h_html_a85f7d0e774e15eb35b74f53264305e16"><div class="ttname"><a href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a></div><div class="ttdeci">#define RETURN_IF_ERROR(stmt)</div><div class="ttdoc">some generally useful macros </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00242">status.h:242</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa62513ecce3dc2a2c7fc202d31a9a1f2"><div class="ttname"><a href="namespaceimpala.html#aa62513ecce3dc2a2c7fc202d31a9a1f2">impala::PopulatePathPermissionCache</a></div><div class="ttdeci">void PopulatePathPermissionCache(hdfsFS fs, const std::string &path_str, PermissionCache *permissions_cache)</div></div> |
| <div class="ttc" id="hdfs-util_8h_html"><div class="ttname"><a href="hdfs-util_8h.html">hdfs-util.h</a></div></div> |
| <div class="ttc" id="error-util_8h_html"><div class="ttname"><a href="error-util_8h.html">error-util.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ad9d7363b9825635e741e2d45e1d944fa"><div class="ttname"><a href="namespaceimpala.html#ad9d7363b9825635e741e2d45e1d944fa">impala::SetExecPlanFragmentParams</a></div><div class="ttdeci">void SetExecPlanFragmentParams(QuerySchedule &schedule, int backend_num, const TPlanFragment &fragment, int fragment_idx, const FragmentExecParams &params, int instance_idx, const TNetworkAddress &coord, TExecPlanFragmentParams *rpc_params)</div><div class="ttdoc">Fill in rpc_params based on parameters. </div></div> |
| <div class="ttc" id="namespaceimpala_html_a420d4837428729719338ca91ed375f3c"><div class="ttname"><a href="namespaceimpala.html#a420d4837428729719338ca91ed375f3c">impala::lock_</a></div><div class="ttdeci">boost::mutex lock_</div><div class="ttdoc">protects all fields below </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00233">coordinator.h:233</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_a8841ca205b2a05d608e82d443bad3a77"><div class="ttname"><a href="runtime-profile_8h.html#a8841ca205b2a05d608e82d443bad3a77">ADD_TIMER</a></div><div class="ttdeci">#define ADD_TIMER(profile, name)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00050">runtime-profile.h:50</a></div></div> |
| <div class="ttc" id="data-stream-mgr_8h_html"><div class="ttname"><a href="data-stream-mgr_8h.html">data-stream-mgr.h</a></div></div> |
| <div class="ttc" id="plan-fragment-executor_8h_html"><div class="ttname"><a href="plan-fragment-executor_8h.html">plan-fragment-executor.h</a></div></div> |
| <div class="ttc" id="coordinator_8cc_html_a0bc8067794763b22cd794e6cac87ad6b"><div class="ttname"><a href="coordinator_8cc.html#a0bc8067794763b22cd794e6cac87ad6b">DECLARE_int32</a></div><div class="ttdeci">DECLARE_int32(be_port)</div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a759b34e1712af6039b9c6e81ed83a798"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a759b34e1712af6039b9c6e81ed83a798">impala::Coordinator::BackendExecState::profile</a></div><div class="ttdeci">RuntimeProfile * profile</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00133">coordinator.cc:133</a></div></div> |
| <div class="ttc" id="structimpala_1_1InstanceComparator_html_a97427bbefe6128d674ddfd45e15fbff7"><div class="ttname"><a href="structimpala_1_1InstanceComparator.html#a97427bbefe6128d674ddfd45e15fbff7">impala::InstanceComparator::Profile</a></div><div class="ttdeci">pair< RuntimeProfile *, bool > Profile</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l01333">coordinator.cc:1333</a></div></div> |
| <div class="ttc" id="structimpala_1_1FragmentInstanceCounters_html_a1b44d89f4c7de168ec44110894cfa0f1"><div class="ttname"><a href="structimpala_1_1FragmentInstanceCounters.html#a1b44d89f4c7de168ec44110894cfa0f1">impala::FragmentInstanceCounters::throughput_counters</a></div><div class="ttdeci">CounterMap throughput_counters</div><div class="ttdoc">Throughput counters per node. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00209">coordinator.h:209</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a0a06145acd27005a1c1f482dc2375742"><div class="ttname"><a href="namespaceimpala.html#a0a06145acd27005a1c1f482dc2375742">impala::desc_tbl_</a></div><div class="ttdeci">TDescriptorTable desc_tbl_</div><div class="ttdoc">copied from TQueryExecRequest; constant across all fragments </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00197">coordinator.h:197</a></div></div> |
| <div class="ttc" id="coordinator_8cc_html_a8586080ec51402cfb11689fea6660a39"><div class="ttname"><a href="coordinator_8cc.html#a8586080ec51402cfb11689fea6660a39">DEFINE_bool</a></div><div class="ttdeci">DEFINE_bool(insert_inherit_permissions, false,"If true, new directories created by ""INSERTs will inherit the permissions of their parent directories")</div></div> |
| <div class="ttc" id="namespaceimpala_html_ab1a9829c4d817be471e86d26f1f316b8"><div class="ttname"><a href="namespaceimpala.html#ab1a9829c4d817be471e86d26f1f316b8">impala::files_to_move_</a></div><div class="ttdeci">FileMoveMap files_to_move_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00293">coordinator.h:293</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_af1fe611bdd6d25b0e8ba4938b8efb4e2"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#af1fe611bdd6d25b0e8ba4938b8efb4e2">impala::Coordinator::BackendExecState::fragment_idx</a></div><div class="ttdeci">int fragment_idx</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00115">coordinator.cc:115</a></div></div> |
| <div class="ttc" id="client-cache_8h_html"><div class="ttname"><a href="client-cache_8h.html">client-cache.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ObjectPool_html"><div class="ttname"><a href="classimpala_1_1ObjectPool.html">impala::ObjectPool</a></div><div class="ttdef"><b>Definition:</b> <a href="object-pool_8h_source.html#l00030">object-pool.h:30</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ae85655273cec6d52b5d875516bc212f8"><div class="ttname"><a href="namespaceimpala.html#ae85655273cec6d52b5d875516bc212f8">impala::UpdateExecSummary</a></div><div class="ttdeci">void UpdateExecSummary(int fragment_idx, int instance_idx, RuntimeProfile *profile)</div></div> |
| <div class="ttc" id="namespaceimpala_html_a7ce4ff5974d4d8b4a5c20b36f061af7a"><div class="ttname"><a href="namespaceimpala.html#a7ce4ff5974d4d8b4a5c20b36f061af7a">impala::ProcessQueryOptions</a></div><div class="ttdeci">static void ProcessQueryOptions(const TQueryOptions &query_options, DebugOptions *debug_options)</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00278">coordinator.cc:278</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a309108c8b3eaf4e5b154bc4eb4624880"><div class="ttname"><a href="namespaceimpala.html#a309108c8b3eaf4e5b154bc4eb4624880">impala::MakeNetworkAddress</a></div><div class="ttdeci">TNetworkAddress MakeNetworkAddress(const string &hostname, int port)</div><div class="ttdef"><b>Definition:</b> <a href="network-util_8cc_source.html#l00096">network-util.cc:96</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_acc0060a61afd94d6b1cf84ae7b1b6280"><div class="ttname"><a href="namespaceimpala.html#acc0060a61afd94d6b1cf84ae7b1b6280">impala::MergeErrorMaps</a></div><div class="ttdeci">void MergeErrorMaps(ErrorLogMap *left, const ErrorLogMap &right)</div><div class="ttdef"><b>Definition:</b> <a href="error-util_8cc_source.html#l00159">error-util.cc:159</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_ac75dbc9b6de0c2c1616eb0d56a9602db"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#ac75dbc9b6de0c2c1616eb0d56a9602db">impala::Coordinator::BackendExecState::backend_address</a></div><div class="ttdeci">const TNetworkAddress backend_address</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00108">coordinator.cc:108</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a19e01fc90bc956ac7e1a735d2bea5a5e"><div class="ttname"><a href="namespaceimpala.html#a19e01fc90bc956ac7e1a735d2bea5a5e">impala::PrintId</a></div><div class="ttdeci">string PrintId(const TUniqueId &id, const string &separator)</div><div class="ttdef"><b>Definition:</b> <a href="debug-util_8cc_source.html#l00097">debug-util.cc:97</a></div></div> |
| <div class="ttc" id="logging_8h_html"><div class="ttname"><a href="logging_8h.html">logging.h</a></div></div> |
| <div class="ttc" id="summary-util_8h_html"><div class="ttname"><a href="summary-util_8h.html">summary-util.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a508481848105bae62dd5722fd747572a"><div class="ttname"><a href="namespaceimpala.html#a508481848105bae62dd5722fd747572a">impala::Cancel</a></div><div class="ttdeci">void Cancel(const Status *cause=NULL)</div></div> |
| <div class="ttc" id="namespaceimpala_html_a056594cbd3eecaf53fffce06852ce561"><div class="ttname"><a href="namespaceimpala.html#a056594cbd3eecaf53fffce06852ce561">impala::backend_exec_states_</a></div><div class="ttdeci">std::vector< BackendExecState * > backend_exec_states_</div><div class="ttdoc">BackendExecStates owned by obj_pool() </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00216">coordinator.h:216</a></div></div> |
| <div class="ttc" id="coordinator_8cc_html_a3bcc9693782d97a40316b227514bcd06"><div class="ttname"><a href="coordinator_8cc.html#a3bcc9693782d97a40316b227514bcd06">DECLARE_string</a></div><div class="ttdeci">DECLARE_string(hostname)</div></div> |
| <div class="ttc" id="structimpala_1_1DebugOptions_html_aa704942f14835910654194513bdfbd85"><div class="ttname"><a href="structimpala_1_1DebugOptions.html#aa704942f14835910654194513bdfbd85">impala::DebugOptions::DebugOptions</a></div><div class="ttdeci">DebugOptions()</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00095">coordinator.cc:95</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_a8d817beaf6c704ff590230236d7591b1"><div class="ttname"><a href="runtime-profile_8h.html#a8d817beaf6c704ff590230236d7591b1">ADD_CHILD_TIMER</a></div><div class="ttdeci">#define ADD_CHILD_TIMER(profile, name, parent)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00051">runtime-profile.h:51</a></div></div> |
| <div class="ttc" id="scheduler_8h_html"><div class="ttname"><a href="scheduler_8h.html">scheduler.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_1_1Counter_html"><div class="ttname"><a href="classimpala_1_1RuntimeProfile_1_1Counter.html">impala::RuntimeProfile::Counter</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00085">runtime-profile.h:85</a></div></div> |
| <div class="ttc" id="runtime-profile_8h_html_aaa9a2971c6368e3ddd3f5140a0295eb7"><div class="ttname"><a href="runtime-profile_8h.html#aaa9a2971c6368e3ddd3f5140a0295eb7">SCOPED_TIMER</a></div><div class="ttdeci">#define SCOPED_TIMER(c)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00053">runtime-profile.h:53</a></div></div> |
| <div class="ttc" id="data-sink_8h_html"><div class="ttname"><a href="data-sink_8h.html">data-sink.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a9baeffdce3e7221d0c3e644adcb910bd"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a9baeffdce3e7221d0c3e644adcb910bd">impala::Coordinator::BackendExecState::done</a></div><div class="ttdeci">bool done</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00131">coordinator.cc:131</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a342149bf019a87aa2c45ec6b8a902865"><div class="ttname"><a href="namespaceimpala.html#a342149bf019a87aa2c45ec6b8a902865">impala::ComputeTotalScanRangesComplete</a></div><div class="ttdeci">int64_t ComputeTotalScanRangesComplete(int node_id)</div></div> |
| <div class="ttc" id="classimpala_1_1PrettyPrinter_html_a6e5f06a1ce274f747cc4f9928b797c5a"><div class="ttname"><a href="classimpala_1_1PrettyPrinter.html#a6e5f06a1ce274f747cc4f9928b797c5a">impala::PrettyPrinter::Print</a></div><div class="ttdeci">static std::string Print(bool value, TUnit::type ignored, bool verbose=false)</div><div class="ttdef"><b>Definition:</b> <a href="pretty-printer_8h_source.html#l00033">pretty-printer.h:33</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a8173907db7b6f1785b33db9d4e253be7"><div class="ttname"><a href="namespaceimpala.html#a8173907db7b6f1785b33db9d4e253be7">impala::Exec</a></div><div class="ttdeci">Status Exec(QuerySchedule &schedule, std::vector< ExprContext * > *output_expr_ctxs)</div></div> |
| <div class="ttc" id="namespaceimpala_html_a1114a1388a929cb5fcb5648310fffcd4"><div class="ttname"><a href="namespaceimpala.html#a1114a1388a929cb5fcb5648310fffcd4">impala::FinalizeQuery</a></div><div class="ttdeci">Status FinalizeQuery()</div></div> |
| <div class="ttc" id="pretty-printer_8h_html"><div class="ttname"><a href="pretty-printer_8h.html">pretty-printer.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a32fe01519869388a7c3d63c8ec085e93"><div class="ttname"><a href="namespaceimpala.html#a32fe01519869388a7c3d63c8ec085e93">impala::has_called_wait_</a></div><div class="ttdeci">bool has_called_wait_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00227">coordinator.h:227</a></div></div> |
| <div class="ttc" id="hdfs-fs-cache_8h_html"><div class="ttname"><a href="hdfs-fs-cache_8h.html">hdfs-fs-cache.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a693c7d4098922a8f6d32dcc0c03cc9b7"><div class="ttname"><a href="namespaceimpala.html#a693c7d4098922a8f6d32dcc0c03cc9b7">impala::CancelRemoteFragments</a></div><div class="ttdeci">void CancelRemoteFragments()</div></div> |
| <div class="ttc" id="namespaceimpala_html_a4f03145f4a655319c059d8e57397d0cd"><div class="ttname"><a href="namespaceimpala.html#a4f03145f4a655319c059d8e57397d0cd">impala::PrepareCatalogUpdate</a></div><div class="ttdeci">bool PrepareCatalogUpdate(TUpdateCatalogRequest *catalog_update)</div></div> |
| <div class="ttc" id="namespaceimpala_html_aae7232bb5a43094cb03cc189745e66b8"><div class="ttname"><a href="namespaceimpala.html#aae7232bb5a43094cb03cc189745e66b8">impala::Wait</a></div><div class="ttdeci">Status Wait()</div></div> |
| <div class="ttc" id="namespaceimpala_html_a6bf80cb1d633d79570ccf617684439b1"><div class="ttname"><a href="namespaceimpala.html#a6bf80cb1d633d79570ccf617684439b1">impala::obj_pool</a></div><div class="ttdeci">ObjectPool * obj_pool()</div><div class="ttdoc">Returns a local object pool. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00263">coordinator.h:263</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a95a1797a63017a4504f27118b984adf2"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a95a1797a63017a4504f27118b984adf2">impala::Coordinator::BackendExecState::aggregate_counters</a></div><div class="ttdeci">FragmentInstanceCounters aggregate_counters</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00139">coordinator.cc:139</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a3589885d3d8ec7b6e4ee6e0a51a89491"><div class="ttname"><a href="namespaceimpala.html#a3589885d3d8ec7b6e4ee6e0a51a89491">impala::exec_summary_lock_</a></div><div class="ttdeci">SpinLock exec_summary_lock_</div><div class="ttdoc">Execution summary for this query. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00299">coordinator.h:299</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsFsCache_html_af7b18daf8c4da6bfce2c7612a515bf64"><div class="ttname"><a href="classimpala_1_1HdfsFsCache.html#af7b18daf8c4da6bfce2c7612a515bf64">impala::HdfsFsCache::instance</a></div><div class="ttdeci">static HdfsFsCache * instance()</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-fs-cache_8h_source.html#l00043">hdfs-fs-cache.h:43</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ac79f8d29c9aaf2dddb617ae5e36ca23f"><div class="ttname"><a href="namespaceimpala.html#ac79f8d29c9aaf2dddb617ae5e36ca23f">impala::FindOrInsert</a></div><div class="ttdeci">V * FindOrInsert(std::map< K, V > *m, const K &key, const V &default_val)</div><div class="ttdef"><b>Definition:</b> <a href="container-util_8h_source.html#l00051">container-util.h:51</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExecEnv_html_a609be125f8de92848139a7cae6e975bc"><div class="ttname"><a href="classimpala_1_1ExecEnv.html#a609be125f8de92848139a7cae6e975bc">impala::ExecEnv::process_mem_tracker</a></div><div class="ttdeci">MemTracker * process_mem_tracker()</div><div class="ttdef"><b>Definition:</b> <a href="exec-env_8h_source.html#l00086">exec-env.h:86</a></div></div> |
| <div class="ttc" id="logging_8h_html_a6f29cadf91ddc478eb9fa4e02cd8e0a0"><div class="ttname"><a href="logging_8h.html#a6f29cadf91ddc478eb9fa4e02cd8e0a0">VLOG_FILE_IS_ON</a></div><div class="ttdeci">#define VLOG_FILE_IS_ON</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00065">logging.h:65</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html"><div class="ttname"><a href="classimpala_1_1Status.html">impala::Status</a></div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00081">status.h:81</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aea293e10bddb1360a019cd1e9abf45f2"><div class="ttname"><a href="namespaceimpala.html#aea293e10bddb1360a019cd1e9abf45f2">impala::returned_all_results_</a></div><div class="ttdeci">bool returned_all_results_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00243">coordinator.h:243</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_1_1EventSequence_html_a7bd1a59efa58c0fbf69f41180f6dbfa1"><div class="ttname"><a href="classimpala_1_1RuntimeProfile_1_1EventSequence.html#a7bd1a59efa58c0fbf69f41180f6dbfa1">impala::RuntimeProfile::EventSequence::MarkEvent</a></div><div class="ttdeci">void MarkEvent(const std::string &label)</div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00286">runtime-profile.h:286</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ad4f56de7b31b5daea6ea0e854715925e"><div class="ttname"><a href="namespaceimpala.html#ad4f56de7b31b5daea6ea0e854715925e">impala::SummaryStats</a></div><div class="ttdeci">boost::accumulators::accumulator_set< int64_t, boost::accumulators::features< boost::accumulators::tag::min, boost::accumulators::tag::max, boost::accumulators::tag::mean, boost::accumulators::tag::variance > > SummaryStats</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00181">coordinator.h:181</a></div></div> |
| <div class="ttc" id="classimpala_1_1ScanNode_html_a86d533cdcfa34c18ae02db91536e0d4c"><div class="ttname"><a href="classimpala_1_1ScanNode.html#a86d533cdcfa34c18ae02db91536e0d4c">impala::ScanNode::SCAN_RANGES_COMPLETE_COUNTER</a></div><div class="ttdeci">static const std::string SCAN_RANGES_COMPLETE_COUNTER</div><div class="ttdef"><b>Definition:</b> <a href="scan-node_8h_source.html#l00129">scan-node.h:129</a></div></div> |
| <div class="ttc" id="logging_8h_html_a916e38366cee7c32fd5fae7e2ee5e4a9"><div class="ttname"><a href="logging_8h.html#a916e38366cee7c32fd5fae7e2ee5e4a9">VLOG_QUERY</a></div><div class="ttdeci">#define VLOG_QUERY</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00057">logging.h:57</a></div></div> |
| <div class="ttc" id="scan-node_8h_html"><div class="ttname"><a href="scan-node_8h.html">scan-node.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aafcf2d4284686d708e6849c58d1ad902"><div class="ttname"><a href="namespaceimpala.html#aafcf2d4284686d708e6849c58d1ad902">impala::query_status_</a></div><div class="ttdeci">Status query_status_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00237">coordinator.h:237</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a48e0e1c5d4b43432450c92750fb35cb3"><div class="ttname"><a href="namespaceimpala.html#a48e0e1c5d4b43432450c92750fb35cb3">impala::PerNodeScanRanges</a></div><div class="ttdeci">std::map< TPlanNodeId, std::vector< TScanRangeParams > > PerNodeScanRanges</div><div class="ttdoc">map from scan node id to a list of scan ranges </div><div class="ttdef"><b>Definition:</b> <a href="query-schedule_8h_source.html#l00035">query-schedule.h:35</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a47d8255dea5b52dc4c93a11ed02b29b5"><div class="ttname"><a href="namespaceimpala.html#a47d8255dea5b52dc4c93a11ed02b29b5">impala::CounterMap</a></div><div class="ttdeci">std::map< PlanNodeId, RuntimeProfile::Counter * > CounterMap</div><div class="ttdoc">map from id of a scan node to a specific counter in the node's profile </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00204">coordinator.h:204</a></div></div> |
| <div class="ttc" id="structimpala_1_1DebugOptions_html_ab0848258c00b410e8178975ece709db8"><div class="ttname"><a href="structimpala_1_1DebugOptions.html#ab0848258c00b410e8178975ece709db8">impala::DebugOptions::backend_num</a></div><div class="ttdeci">int backend_num</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00090">coordinator.cc:90</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a427ff48858b169ea3a12329b72d022b5"><div class="ttname"><a href="namespaceimpala.html#a427ff48858b169ea3a12329b72d022b5">impala::GetExecNodePhase</a></div><div class="ttdeci">TExecNodePhase::type GetExecNodePhase(const string &key)</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00255">coordinator.cc:255</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a5d05598507a4b64b2288e3522cf38c58"><div class="ttname"><a href="namespaceimpala.html#a5d05598507a4b64b2288e3522cf38c58">impala::GetStatus</a></div><div class="ttdeci">Status GetStatus()</div><div class="ttdoc">Returns query_status_. </div></div> |
| <div class="ttc" id="namespaceimpala_html_a98367e13193bbe0af1fc61a4422b7e6fa5fb0cb0f52f73f240132738b9019cd5d"><div class="ttname"><a href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa5fb0cb0f52f73f240132738b9019cd5d">impala::CREATE_DIR</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-bulk-ops_8h_source.html#l00030">hdfs-bulk-ops.h:30</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a98367e13193bbe0af1fc61a4422b7e6fad29ef8d6894805fad47239ec0e5bec7b"><div class="ttname"><a href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fad29ef8d6894805fad47239ec0e5bec7b">impala::RENAME</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-bulk-ops_8h_source.html#l00031">hdfs-bulk-ops.h:31</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a55cca46ac54085054a0c08d31e05d4fe"><div class="ttname"><a href="namespaceimpala.html#a55cca46ac54085054a0c08d31e05d4fe">impala::ComputeTotalThroughput</a></div><div class="ttdeci">int64_t ComputeTotalThroughput(int node_id)</div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeState_html_a4c01f227dc12d859d5824e7c91d6d1aa"><div class="ttname"><a href="classimpala_1_1RuntimeState.html#a4c01f227dc12d859d5824e7c91d6d1aa">impala::RuntimeState::obj_pool</a></div><div class="ttdeci">ObjectPool * obj_pool() const </div><div class="ttdef"><b>Definition:</b> <a href="runtime-state_8h_source.html#l00092">runtime-state.h:92</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a20991e967dbb80151f6500eb14145943"><div class="ttname"><a href="namespaceimpala.html#a20991e967dbb80151f6500eb14145943">impala::num_remaining_backends_</a></div><div class="ttdeci">int num_remaining_backends_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00278">coordinator.h:278</a></div></div> |
| <div class="ttc" id="debug-util_8h_html"><div class="ttname"><a href="debug-util_8h.html">debug-util.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1QuerySchedule_html"><div class="ttname"><a href="classimpala_1_1QuerySchedule.html">impala::QuerySchedule</a></div><div class="ttdef"><b>Definition:</b> <a href="query-schedule_8h_source.html#l00067">query-schedule.h:67</a></div></div> |
| <div class="ttc" id="structimpala_1_1DebugOptions_html_a585e0f226f2c13e409d89203f50f43ae"><div class="ttname"><a href="structimpala_1_1DebugOptions.html#a585e0f226f2c13e409d89203f50f43ae">impala::DebugOptions::node_id</a></div><div class="ttdeci">int node_id</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00091">coordinator.cc:91</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aae8267be6175d1f4208f54913a68ea7c"><div class="ttname"><a href="namespaceimpala.html#aae8267be6175d1f4208f54913a68ea7c">impala::needs_finalization_</a></div><div class="ttdeci">bool needs_finalization_</div><div class="ttdoc">True if the query needs a post-execution step to tidy up. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00219">coordinator.h:219</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExecEnv_html_a1aa344745ea152e6df02c4c4f7693aab"><div class="ttname"><a href="classimpala_1_1ExecEnv.html#a1aa344745ea152e6df02c4c4f7693aab">impala::ExecEnv::hdfs_op_thread_pool</a></div><div class="ttdeci">HdfsOpThreadPool * hdfs_op_thread_pool()</div><div class="ttdef"><b>Definition:</b> <a href="exec-env_8h_source.html#l00089">exec-env.h:89</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a574dcc25c5967c10d13e4bf3fa4c9bcf"><div class="ttname"><a href="namespaceimpala.html#a574dcc25c5967c10d13e4bf3fa4c9bcf">impala::plan_node_id_to_summary_map_</a></div><div class="ttdeci">boost::unordered_map< TPlanNodeId, int > plan_node_id_to_summary_map_</div><div class="ttdoc">A mapping of plan node ids to index into exec_summary_.nodes. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00303">coordinator.h:303</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a32793557c7c9e175d833303a8f3df878"><div class="ttname"><a href="namespaceimpala.html#a32793557c7c9e175d833303a8f3df878">impala::coordinator_counters_</a></div><div class="ttdeci">FragmentInstanceCounters coordinator_counters_</div><div class="ttdoc">Throughput counters for the coordinator fragment. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00342">coordinator.h:342</a></div></div> |
| <div class="ttc" id="classimpala_1_1DataSink_html_acb54f550a23683ce30805c392d24ebc7"><div class="ttname"><a href="classimpala_1_1DataSink.html#acb54f550a23683ce30805c392d24ebc7">impala::DataSink::OutputInsertStats</a></div><div class="ttdeci">static std::string OutputInsertStats(const PartitionStatusMap &stats, const std::string &prefix="")</div><div class="ttdoc">Outputs the insert stats contained in the map of insert partition updates to a string. </div><div class="ttdef"><b>Definition:</b> <a href="data-sink_8cc_source.html#l00103">data-sink.cc:103</a></div></div> |
| <div class="ttc" id="logging_8h_html_a6ccc2106c47622db4e52a401a415fc58"><div class="ttname"><a href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a></div><div class="ttdeci">#define VLOG_ROW</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00059">logging.h:59</a></div></div> |
| <div class="ttc" id="classimpala_1_1DataSink_html_ab95c1680a2c851bed71977e502139b32"><div class="ttname"><a href="classimpala_1_1DataSink.html#ab95c1680a2c851bed71977e502139b32">impala::DataSink::MergeInsertStats</a></div><div class="ttdeci">static void MergeInsertStats(const TInsertStats &src_stats, TInsertStats *dst_stats)</div><div class="ttdef"><b>Definition:</b> <a href="data-sink_8cc_source.html#l00090">data-sink.cc:90</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a7d75818564bd9d86830ca57bbe71116d"><div class="ttname"><a href="namespaceimpala.html#a7d75818564bd9d86830ca57bbe71116d">impala::finalization_timer_</a></div><div class="ttdeci">RuntimeProfile::Counter * finalization_timer_</div><div class="ttdoc">Total time spent in finalization (typically 0 except for INSERT into hdfs tables) ...</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00348">coordinator.h:348</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_1_1EventSequence_html"><div class="ttname"><a href="classimpala_1_1RuntimeProfile_1_1EventSequence.html">impala::RuntimeProfile::EventSequence</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00267">runtime-profile.h:267</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aab92a7cc1123d2a61596f47ec2fea1fa"><div class="ttname"><a href="namespaceimpala.html#aab92a7cc1123d2a61596f47ec2fea1fa">impala::wait_lock_</a></div><div class="ttdeci">boost::mutex wait_lock_</div><div class="ttdoc">ensures single-threaded execution of Wait(); must not hold lock_ when acquiring this ...</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00225">coordinator.h:225</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aa98447566dd6700a2faaaaf3059f4d95"><div class="ttname"><a href="namespaceimpala.html#aa98447566dd6700a2faaaaf3059f4d95">impala::row_desc</a></div><div class="ttdeci">const RowDescriptor & row_desc() const </div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a318b69bf42d4d23f7ae71a8006efb85b"><div class="ttname"><a href="classimpala_1_1Status.html#a318b69bf42d4d23f7ae71a8006efb85b">impala::Status::CANCELLED</a></div><div class="ttdeci">static const Status CANCELLED</div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00088">status.h:88</a></div></div> |
| <div class="ttc" id="structimpala_1_1FragmentInstanceCounters_html_ae39f9f7b6955c58f73d95a30d2f4ee1a"><div class="ttname"><a href="structimpala_1_1FragmentInstanceCounters.html#ae39f9f7b6955c58f73d95a30d2f4ee1a">impala::FragmentInstanceCounters::scan_ranges_complete_counters</a></div><div class="ttdeci">CounterMap scan_ranges_complete_counters</div><div class="ttdoc">Total finished scan ranges per node. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00212">coordinator.h:212</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a72ec1818c46664b276a7a29ed73d4fdb"><div class="ttname"><a href="namespaceimpala.html#a72ec1818c46664b276a7a29ed73d4fdb">impala::PermissionCache</a></div><div class="ttdeci">boost::unordered_map< std::string, std::pair< bool, short > > PermissionCache</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00450">coordinator.h:450</a></div></div> |
| <div class="ttc" id="parallel-executor_8h_html"><div class="ttname"><a href="parallel-executor_8h.html">parallel-executor.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a90daa83f383514b15451c6e5031deef9"><div class="ttname"><a href="namespaceimpala.html#a90daa83f383514b15451c6e5031deef9">impala::PrintExecSummary</a></div><div class="ttdeci">std::string PrintExecSummary(const TExecSummary &exec_summary)</div><div class="ttdoc">Print the exec summary as a formatted table. </div><div class="ttdef"><b>Definition:</b> <a href="summary-util_8cc_source.html#l00108">summary-util.cc:108</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a3f3e0b2aa35c67a0659f77a9fe752bd8"><div class="ttname"><a href="namespaceimpala.html#a3f3e0b2aa35c67a0659f77a9fe752bd8">impala::per_partition_status_</a></div><div class="ttdeci">PartitionStatusMap per_partition_status_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00289">coordinator.h:289</a></div></div> |
| <div class="ttc" id="hdfs-bulk-ops_8h_html"><div class="ttname"><a href="hdfs-bulk-ops_8h.html">hdfs-bulk-ops.h</a></div></div> |
| <div class="ttc" id="structimpala_1_1DebugOptions_html"><div class="ttname"><a href="structimpala_1_1DebugOptions.html">impala::DebugOptions</a></div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00089">coordinator.cc:89</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a90b9637d6b3e8b12fdefc25fde7786a8"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a90b9637d6b3e8b12fdefc25fde7786a8">impala::Coordinator::BackendExecState::BackendExecState</a></div><div class="ttdeci">BackendExecState(QuerySchedule &schedule, Coordinator *coord, const TNetworkAddress &coord_address, int backend_num, const TPlanFragment &fragment, int fragment_idx, const FragmentExecParams &params, int instance_idx, DebugOptions *debug_options, ObjectPool *obj_pool)</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00141">coordinator.cc:141</a></div></div> |
| <div class="ttc" id="network-util_8h_html"><div class="ttname"><a href="network-util_8h.html">network-util.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aecf5ef62c6e2482353b0f41f80528245"><div class="ttname"><a href="namespaceimpala.html#aecf5ef62c6e2482353b0f41f80528245">impala::InitExecProfile</a></div><div class="ttdeci">void InitExecProfile(const TQueryExecRequest &request)</div></div> |
| <div class="ttc" id="classimpala_1_1MemTracker_html_a22fbc3febb516f0f5edd1d40699e6991"><div class="ttname"><a href="classimpala_1_1MemTracker.html#a22fbc3febb516f0f5edd1d40699e6991">impala::MemTracker::GetRequestPoolMemTracker</a></div><div class="ttdeci">static MemTracker * GetRequestPoolMemTracker(const std::string &pool_name, MemTracker *parent)</div><div class="ttdef"><b>Definition:</b> <a href="mem-tracker_8cc_source.html#l00134">mem-tracker.cc:134</a></div></div> |
| <div class="ttc" id="classimpala_1_1RuntimeProfile_html"><div class="ttname"><a href="classimpala_1_1RuntimeProfile.html">impala::RuntimeProfile</a></div><div class="ttdef"><b>Definition:</b> <a href="runtime-profile_8h_source.html#l00083">runtime-profile.h:83</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ac55a10ca0171687156609e8d6ba28127a45b8e4259988c3518a05b9202efb0898"><div class="ttname"><a href="namespaceimpala.html#ac55a10ca0171687156609e8d6ba28127a45b8e4259988c3518a05b9202efb0898">impala::OK</a></div><div class="ttdef"><b>Definition:</b> <a href="webserver_8cc_source.html#l00115">webserver.cc:115</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a5219328e34f90ec0679655b8e66bd31f"><div class="ttname"><a href="namespaceimpala.html#a5219328e34f90ec0679655b8e66bd31f">impala::query_mem_tracker_</a></div><div class="ttdeci">boost::shared_ptr< MemTracker > query_mem_tracker_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00252">coordinator.h:252</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ab71d76e113dc636fe9a9ef47c61f678a"><div class="ttname"><a href="namespaceimpala.html#ab71d76e113dc636fe9a9ef47c61f678a">impala::WaitForAllBackends</a></div><div class="ttdeci">Status WaitForAllBackends()</div></div> |
| <div class="ttc" id="namespaceimpala_html_a6a1bc3d15242d20a00589ed2fd3c6066"><div class="ttname"><a href="namespaceimpala.html#a6a1bc3d15242d20a00589ed2fd3c6066">impala::backend_completion_cv_</a></div><div class="ttdeci">boost::condition_variable backend_completion_cv_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00274">coordinator.h:274</a></div></div> |
| <div class="ttc" id="structimpala_1_1InstanceComparator_html_a75d19a7a9a8c3b061b027a9d91427f57"><div class="ttname"><a href="structimpala_1_1InstanceComparator.html#a75d19a7a9a8c3b061b027a9d91427f57">impala::InstanceComparator::operator()</a></div><div class="ttdeci">bool operator()(const Profile &a, const Profile &b) const </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l01334">coordinator.cc:1334</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ae9b624f85ef8731ba4cf2a0bc8f622d0"><div class="ttname"><a href="namespaceimpala.html#ae9b624f85ef8731ba4cf2a0bc8f622d0">impala::fragment_profiles_</a></div><div class="ttdeci">std::vector< PerFragmentProfileData > fragment_profiles_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00339">coordinator.h:339</a></div></div> |
| <div class="ttc" id="container-util_8h_html"><div class="ttname"><a href="container-util_8h.html">container-util.h</a></div></div> |
| <div class="ttc" id="llama-util_8h_html"><div class="ttname"><a href="llama-util_8h.html">llama-util.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a98367e13193bbe0af1fc61a4422b7e6fa7e830d2a8a807bbb551965b429eba876"><div class="ttname"><a href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa7e830d2a8a807bbb551965b429eba876">impala::CHMOD</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-bulk-ops_8h_source.html#l00033">hdfs-bulk-ops.h:33</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_af8a4757a740abbb7e521cf91b2d6cc75"><div class="ttname"><a href="namespaceimpala.html#af8a4757a740abbb7e521cf91b2d6cc75">impala::UpdateAverageProfile</a></div><div class="ttdeci">void UpdateAverageProfile(BackendExecState *backend_exec_state)</div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a0de3913d5153b5256a357db2328e650d"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a0de3913d5153b5256a357db2328e650d">impala::Coordinator::BackendExecState::fragment_instance_id</a></div><div class="ttdeci">TUniqueId fragment_instance_id</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00106">coordinator.cc:106</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_ab9ac162db9bba61a67ef0dc1db24ff91"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#ab9ac162db9bba61a67ef0dc1db24ff91">impala::Coordinator::BackendExecState::total_split_size</a></div><div class="ttdeci">int64_t total_split_size</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00109">coordinator.cc:109</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ab2bf118ff6d9897b828f5fb8cce70ad3"><div class="ttname"><a href="namespaceimpala.html#ab2bf118ff6d9897b828f5fb8cce70ad3">impala::PrintPlanNodeType</a></div><div class="ttdeci">string PrintPlanNodeType(const TPlanNodeType::type &type)</div><div class="ttdef"><b>Definition:</b> <a href="debug-util_8cc_source.html#l00157">debug-util.cc:157</a></div></div> |
| <div class="ttc" id="data-stream-sender_8h_html"><div class="ttname"><a href="data-stream-sender_8h.html">data-stream-sender.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a98367e13193bbe0af1fc61a4422b7e6fa0331700e2301dedaf6a902d078218682"><div class="ttname"><a href="namespaceimpala.html#a98367e13193bbe0af1fc61a4422b7e6fa0331700e2301dedaf6a902d078218682">impala::DELETE</a></div><div class="ttdef"><b>Definition:</b> <a href="hdfs-bulk-ops_8h_source.html#l00029">hdfs-bulk-ops.h:29</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a21ef571aa3ec060718ac944c5ef47001"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a21ef571aa3ec060718ac944c5ef47001">impala::Coordinator::BackendExecState::status</a></div><div class="ttdeci">Status status</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00128">coordinator.cc:128</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a580565665ea944eb64f3f495b1bee1e0"><div class="ttname"><a href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">impala::Status::OK</a></div><div class="ttdeci">static const Status OK</div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00087">status.h:87</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a87319b8665a64adda1e0908ebc6963e5"><div class="ttname"><a href="namespaceimpala.html#a87319b8665a64adda1e0908ebc6963e5">impala::exec_summary</a></div><div class="ttdeci">const TExecSummary & exec_summary() const </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00173">coordinator.h:173</a></div></div> |
| <div class="ttc" id="classimpala_1_1PlanFragmentExecutor_html_a155af468fa604258d08f8dab01b309e7"><div class="ttname"><a href="classimpala_1_1PlanFragmentExecutor.html#a155af468fa604258d08f8dab01b309e7">impala::PlanFragmentExecutor::ReportStatusCallback</a></div><div class="ttdeci">boost::function< void(const Status &status, RuntimeProfile *profile, bool done)> ReportStatusCallback</div><div class="ttdef"><b>Definition:</b> <a href="plan-fragment-executor_8h_source.html#l00074">plan-fragment-executor.h:74</a></div></div> |
| <div class="ttc" id="expr_8h_html"><div class="ttname"><a href="expr_8h.html">expr.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_ae11a4460de7695cbe1bcd94d8286df5c"><div class="ttname"><a href="namespaceimpala.html#ae11a4460de7695cbe1bcd94d8286df5c">impala::IsHiddenFile</a></div><div class="ttdeci">bool IsHiddenFile(const string &filename)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-util_8cc_source.html#l00050">hdfs-util.cc:50</a></div></div> |
| <div class="ttc" id="classimpala_1_1DescriptorTbl_html_a7eec9a1b9f48220a92e3bdd4fdba44ba"><div class="ttname"><a href="classimpala_1_1DescriptorTbl.html#a7eec9a1b9f48220a92e3bdd4fdba44ba">impala::DescriptorTbl::Create</a></div><div class="ttdeci">static Status Create(ObjectPool *pool, const TDescriptorTable &thrift_tbl, DescriptorTbl **tbl)</div><div class="ttdef"><b>Definition:</b> <a href="descriptors_8cc_source.html#l00378">descriptors.cc:378</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_af1eec3a76356f0cf625d434c21d6dd0d"><div class="ttname"><a href="namespaceimpala.html#af1eec3a76356f0cf625d434c21d6dd0d">impala::ComputeFragmentSummaryStats</a></div><div class="ttdeci">void ComputeFragmentSummaryStats(BackendExecState *backend_exec_state)</div></div> |
| <div class="ttc" id="namespaceimpala_html_a57eaecb97c9c87e4c57c8a8a81211422"><div class="ttname"><a href="namespaceimpala.html#a57eaecb97c9c87e4c57c8a8a81211422">impala::exec_env_</a></div><div class="ttdeci">ExecEnv * exec_env_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00193">coordinator.h:193</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a0da064b03c3877a662a1830117222c1a"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a0da064b03c3877a662a1830117222c1a">impala::Coordinator::BackendExecState::profile_created</a></div><div class="ttdeci">bool profile_created</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00132">coordinator.cc:132</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a316415821307d27237bed61e0be1ef0e"><div class="ttname"><a href="namespaceimpala.html#a316415821307d27237bed61e0be1ef0e">impala::query_events_</a></div><div class="ttdeci">RuntimeProfile::EventSequence * query_events_</div><div class="ttdoc">Event timeline for this query. Unowned. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00309">coordinator.h:309</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_a21a85a27441062e63b8858c1edde1886"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#a21a85a27441062e63b8858c1edde1886">impala::Coordinator::BackendExecState::stopwatch</a></div><div class="ttdeci">MonotonicStopWatch stopwatch</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00107">coordinator.cc:107</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_afdc2218a15eac469107b45c5db2dd289"><div class="ttname"><a href="namespaceimpala.html#afdc2218a15eac469107b45c5db2dd289">impala::GetErrorLog</a></div><div class="ttdeci">std::string GetErrorLog()</div></div> |
| <div class="ttc" id="names_8h_html"><div class="ttname"><a href="names_8h.html">names.h</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aba23271e9c88b75efea05a2afb25856b"><div class="ttname"><a href="namespaceimpala.html#aba23271e9c88b75efea05a2afb25856b">impala::UpdateStatus</a></div><div class="ttdeci">Status UpdateStatus(const Status &status, const TUniqueId *failed_fragment)</div></div> |
| <div class="ttc" id="classimpala_1_1ObjectPool_html_ab191078c99825682a0a9915ca1e0c05c"><div class="ttname"><a href="classimpala_1_1ObjectPool.html#ab191078c99825682a0a9915ca1e0c05c">impala::ObjectPool::Add</a></div><div class="ttdeci">T * Add(T *t)</div><div class="ttdef"><b>Definition:</b> <a href="object-pool_8h_source.html#l00042">object-pool.h:42</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_a6b9c01a432ae49b57482c9fc42a73681"><div class="ttname"><a href="classimpala_1_1Expr.html#a6b9c01a432ae49b57482c9fc42a73681">impala::Expr::CreateExprTrees</a></div><div class="ttdeci">static Status CreateExprTrees(ObjectPool *pool, const std::vector< TExpr > &texprs, std::vector< ExprContext * > *ctxs)</div><div class="ttdef"><b>Definition:</b> <a href="expr_8cc_source.html#l00149">expr.cc:149</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_abb951b2220d3da8ec3b3d819dfbe2e92"><div class="ttname"><a href="namespaceimpala.html#abb951b2220d3da8ec3b3d819dfbe2e92">impala::executor_</a></div><div class="ttdeci">ImpaladQueryExecutor * executor_</div><div class="ttdoc">execution state of coordinator fragment </div><div class="ttdef"><b>Definition:</b> <a href="expr-test_8cc_source.html#l00071">expr-test.cc:71</a></div></div> |
| <div class="ttc" id="classimpala_1_1Expr_html_a3ae02e50debba50ac5c7b6dd9f8016e7"><div class="ttname"><a href="classimpala_1_1Expr.html#a3ae02e50debba50ac5c7b6dd9f8016e7">impala::Expr::Prepare</a></div><div class="ttdeci">static Status Prepare(const std::vector< ExprContext * > &ctxs, RuntimeState *state, const RowDescriptor &row_desc, MemTracker *tracker)</div></div> |
| <div class="ttc" id="logging_8h_html_a0bdc3628c740982e847cd24114c07994"><div class="ttname"><a href="logging_8h.html#a0bdc3628c740982e847cd24114c07994">VLOG_FILE</a></div><div class="ttdeci">#define VLOG_FILE</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00058">logging.h:58</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a5d79ffeb06ab02bf3083bfa1eb45d878"><div class="ttname"><a href="namespaceimpala.html#a5d79ffeb06ab02bf3083bfa1eb45d878">impala::ImpalaInternalServiceConnection</a></div><div class="ttdeci">ClientConnection< ImpalaInternalServiceClient > ImpalaInternalServiceConnection</div><div class="ttdef"><b>Definition:</b> <a href="client-cache_8h_source.html#l00346">client-cache.h:346</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExecEnv_html"><div class="ttname"><a href="classimpala_1_1ExecEnv.html">impala::ExecEnv</a></div><div class="ttdef"><b>Definition:</b> <a href="exec-env_8h_source.html#l00053">exec-env.h:53</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a721f135364b8f28d9cf09cd45e95745d"><div class="ttname"><a href="namespaceimpala.html#a721f135364b8f28d9cf09cd45e95745d">impala::CancelInternal</a></div><div class="ttdeci">void CancelInternal()</div><div class="ttdoc">Runs cancel logic. Assumes that lock_ is held. </div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_af31d42eeb3f5ce24b0e4548b7593a99a"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#af31d42eeb3f5ce24b0e4548b7593a99a">impala::Coordinator::BackendExecState::total_ranges_complete</a></div><div class="ttdeci">int64_t total_ranges_complete</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00137">coordinator.cc:137</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a95ba859e42fe93445b340533220836ac"><div class="ttname"><a href="classimpala_1_1Status.html#a95ba859e42fe93445b340533220836ac">impala::Status::ok</a></div><div class="ttdeci">bool ok() const </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00172">status.h:172</a></div></div> |
| <div class="ttc" id="classimpala_1_1PlanFragmentExecutor_html_ac12ad0f6d8bb9ddbb62d58e04a412c93"><div class="ttname"><a href="classimpala_1_1PlanFragmentExecutor.html#ac12ad0f6d8bb9ddbb62d58e04a412c93">impala::PlanFragmentExecutor::PER_HOST_PEAK_MEM_COUNTER</a></div><div class="ttdeci">static const std::string PER_HOST_PEAK_MEM_COUNTER</div><div class="ttdoc">Name of the counter that is tracking per query, per host peak mem usage. </div><div class="ttdef"><b>Definition:</b> <a href="plan-fragment-executor_8h_source.html#l00134">plan-fragment-executor.h:134</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aed2594aa39638a14b42554a4e3229dcf"><div class="ttname"><a href="namespaceimpala.html#aed2594aa39638a14b42554a4e3229dcf">impala::GetHdfsErrorMsg</a></div><div class="ttdeci">string GetHdfsErrorMsg(const string &prefix, const string &file)</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-util_8cc_source.html#l00026">hdfs-util.cc:26</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a0e6d46f5800bcf4765efd3c21ce535bb"><div class="ttname"><a href="namespaceimpala.html#a0e6d46f5800bcf4765efd3c21ce535bb">impala::ReportQuerySummary</a></div><div class="ttdeci">void ReportQuerySummary()</div></div> |
| <div class="ttc" id="namespaceimpala_html_a7ab8f672190232021bdfcf200502140c"><div class="ttname"><a href="namespaceimpala.html#a7ab8f672190232021bdfcf200502140c">impala::query_mem_tracker</a></div><div class="ttdeci">MemTracker * query_mem_tracker()</div></div> |
| <div class="ttc" id="structimpala_1_1DebugOptions_html_ac7cf72312d0af9c73b81ca3805be904b"><div class="ttname"><a href="structimpala_1_1DebugOptions.html#ac7cf72312d0af9c73b81ca3805be904b">impala::DebugOptions::action</a></div><div class="ttdeci">TDebugAction::type action</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00092">coordinator.cc:92</a></div></div> |
| <div class="ttc" id="classimpala_1_1HdfsOperationSet_html_ae5b0a59660945825eb1647236e76b537"><div class="ttname"><a href="classimpala_1_1HdfsOperationSet.html#ae5b0a59660945825eb1647236e76b537">impala::HdfsOperationSet::Error</a></div><div class="ttdeci">std::pair< const HdfsOp *, std::string > Error</div><div class="ttdef"><b>Definition:</b> <a href="hdfs-bulk-ops_8h_source.html#l00109">hdfs-bulk-ops.h:109</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_aaedda9a1b006ac22f15b591ce15f0281"><div class="ttname"><a href="namespaceimpala.html#aaedda9a1b006ac22f15b591ce15f0281">impala::FinalizeSuccessfulInsert</a></div><div class="ttdeci">Status FinalizeSuccessfulInsert()</div><div class="ttdoc">Moves all temporary staging files to their final destinations. </div></div> |
| <div class="ttc" id="namespaceimpala_html_ab296eb278342117dde26e2e28bbe8190"><div class="ttname"><a href="namespaceimpala.html#ab296eb278342117dde26e2e28bbe8190">impala::query_ctx_</a></div><div class="ttdeci">TQueryCtx query_ctx_</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00198">coordinator.h:198</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a20d70f7b44aeb530802e3ccc198f19fa"><div class="ttname"><a href="namespaceimpala.html#a20d70f7b44aeb530802e3ccc198f19fa">impala::CollectScanNodeCounters</a></div><div class="ttdeci">void CollectScanNodeCounters(RuntimeProfile *, FragmentInstanceCounters *result)</div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_af612858a21f2acad7ff0ee8f58dadf94"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#af612858a21f2acad7ff0ee8f58dadf94">impala::Coordinator::BackendExecState::instance_idx</a></div><div class="ttdeci">int instance_idx</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00118">coordinator.cc:118</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a940125179d49c976f146709006813049"><div class="ttname"><a href="namespaceimpala.html#a940125179d49c976f146709006813049">impala::ErrorLogMap</a></div><div class="ttdeci">std::map< TErrorCode::type, TErrorLogEntry > ErrorLogMap</div><div class="ttdoc">Tracks log messages per error code. </div><div class="ttdef"><b>Definition:</b> <a href="error-util_8h_source.html#l00144">error-util.h:144</a></div></div> |
| <div class="ttc" id="classimpala_1_1ParallelExecutor_html_abad564e6b67909483e30f0ac1c0f30cc"><div class="ttname"><a href="classimpala_1_1ParallelExecutor.html#abad564e6b67909483e30f0ac1c0f30cc">impala::ParallelExecutor::Exec</a></div><div class="ttdeci">static Status Exec(Function function, void **args, int num_args, StatsMetric< double > *latencies=NULL)</div><div class="ttdoc">Callers may pass a StatsMetric to gather the latency distribution of task execution. </div><div class="ttdef"><b>Definition:</b> <a href="parallel-executor_8cc_source.html#l00026">parallel-executor.cc:26</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a187e8324ceb67822e78984b068033d12"><div class="ttname"><a href="namespaceimpala.html#a187e8324ceb67822e78984b068033d12">impala::query_profile_</a></div><div class="ttdeci">boost::scoped_ptr< RuntimeProfile > query_profile_</div><div class="ttdoc">Aggregate counters for the entire query. </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00306">coordinator.h:306</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a4f604de2fe46ab0a3969d06f4126bc77"><div class="ttname"><a href="namespaceimpala.html#a4f604de2fe46ab0a3969d06f4126bc77">impala::stmt_type_</a></div><div class="ttdeci">TStmtType::type stmt_type_</div><div class="ttdoc">copied from TQueryExecRequest, governs when to call ReportQuerySummary </div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8h_source.html#l00201">coordinator.h:201</a></div></div> |
| <div class="ttc" id="classimpala_1_1Coordinator_1_1BackendExecState_html_ae011202240d67890415790622b7ea5c2"><div class="ttname"><a href="classimpala_1_1Coordinator_1_1BackendExecState.html#ae011202240d67890415790622b7ea5c2">impala::Coordinator::BackendExecState::error_log</a></div><div class="ttdeci">ErrorLogMap error_log</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00134">coordinator.cc:134</a></div></div> |
| <div class="ttc" id="logging_8h_html_ac976d11f99eb284f949261fcee44dffd"><div class="ttname"><a href="logging_8h.html#ac976d11f99eb284f949261fcee44dffd">VLOG_QUERY_IS_ON</a></div><div class="ttdeci">#define VLOG_QUERY_IS_ON</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00064">logging.h:64</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a87523263d912f04770414ae7dbe55d01"><div class="ttname"><a href="namespaceimpala.html#a87523263d912f04770414ae7dbe55d01">impala::GetDebugAction</a></div><div class="ttdeci">TDebugAction::type GetDebugAction(const string &key)</div><div class="ttdef"><b>Definition:</b> <a href="coordinator_8cc_source.html#l00267">coordinator.cc:267</a></div></div> |
| <div class="ttc" id="structimpala_1_1FragmentExecParams_html"><div class="ttname"><a href="structimpala_1_1FragmentExecParams.html">impala::FragmentExecParams</a></div><div class="ttdef"><b>Definition:</b> <a href="query-schedule_8h_source.html#l00047">query-schedule.h:47</a></div></div> |
| <div class="ttc" id="namespaceimpala_html_a7dff4fbd53ca26eb055bc173471c7be3"><div class="ttname"><a href="namespaceimpala.html#a7dff4fbd53ca26eb055bc173471c7be3">impala::PrintBackendInfo</a></div><div class="ttdeci">void PrintBackendInfo()</div></div> |
| <div class="ttc" id="classimpala_1_1ProgressUpdater_html_a3e05b2a32fd1bdf32d2a275c10fc431f"><div class="ttname"><a href="classimpala_1_1ProgressUpdater.html#a3e05b2a32fd1bdf32d2a275c10fc431f">impala::ProgressUpdater::Update</a></div><div class="ttdeci">void Update(int64_t delta)</div><div class="ttdef"><b>Definition:</b> <a href="progress-updater_8cc_source.html#l00034">progress-updater.cc:34</a></div></div> |
| <div class="ttc" id="exec-env_8h_html"><div class="ttname"><a href="exec-env_8h.html">exec-env.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1ExecEnv_html_a4e470b8606b776eb409005c95c575361"><div class="ttname"><a href="classimpala_1_1ExecEnv.html#a4e470b8606b776eb409005c95c575361">impala::ExecEnv::impalad_client_cache</a></div><div class="ttdeci">ImpalaInternalServiceClientCache * impalad_client_cache()</div><div class="ttdef"><b>Definition:</b> <a href="exec-env_8h_source.html#l00076">exec-env.h:76</a></div></div> |
| </div><!-- fragment --></div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_e5d120be6b5e8a44336cbfd013b25604.html">be</a></li><li class="navelem"><a class="el" href="dir_68cf7cafb51a962d5bc4848b83cab0de.html">src</a></li><li class="navelem"><a class="el" href="dir_1ef28ab5153adb64262182d038c65543.html">runtime</a></li><li class="navelem"><a class="el" href="coordinator_8cc.html">coordinator.cc</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:36 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
