| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <meta http-equiv="X-UA-Compatible" content="IE=9"/> |
| <meta name="generator" content="Doxygen 1.8.6"/> |
| <title>Impala: be/src/util/decompress.cc Source File</title> |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| $(window).load(resizeHeight); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div, it is closed by doxygen! --> |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">Impala |
| </div> |
| <div id="projectbrief">Impalaistheopensource,nativeanalyticdatabaseforApacheHadoop.</div> |
| </td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <!-- end header part --> |
| <!-- Generated by Doxygen 1.8.6 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="namespaces.html"><span>Namespaces</span></a></li> |
| <li><a href="annotated.html"><span>Classes</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div><!-- top --> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| <div id="nav-sync" class="sync"></div> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| $(document).ready(function(){initNavTree('decompress_8cc_source.html','');}); |
| </script> |
| <div id="doc-content"> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark"> </span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark"> </span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark"> </span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark"> </span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark"> </span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark"> </span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark"> </span>Macros</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">decompress.cc</div> </div> |
| </div><!--header--> |
| <div class="contents"> |
| <a href="decompress_8cc.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">// Copyright 2012 Cloudera Inc.</span></div> |
| <div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment">// Licensed under the Apache License, Version 2.0 (the "License");</span></div> |
| <div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment">// you may not use this file except in compliance with the License.</span></div> |
| <div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment">// You may obtain a copy of the License at</span></div> |
| <div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment">// http://www.apache.org/licenses/LICENSE-2.0</span></div> |
| <div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment">//</span></div> |
| <div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment">// Unless required by applicable law or agreed to in writing, software</span></div> |
| <div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment">// distributed under the License is distributed on an "AS IS" BASIS,</span></div> |
| <div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment">// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div> |
| <div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment">// See the License for the specific language governing permissions and</span></div> |
| <div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment">// limitations under the License.</span></div> |
| <div class="line"><a name="l00014"></a><span class="lineno"> 14</span> </div> |
| <div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="preprocessor">#include <boost/assign/list_of.hpp></span></div> |
| <div class="line"><a name="l00016"></a><span class="lineno"> 16</span> <span class="preprocessor">#include "<a class="code" href="decompress_8h.html">util/decompress.h</a>"</span></div> |
| <div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="preprocessor">#include "<a class="code" href="read-write-util_8h.html">exec/read-write-util.h</a>"</span></div> |
| <div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="preprocessor">#include "<a class="code" href="runtime-state_8h.html">runtime/runtime-state.h</a>"</span></div> |
| <div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="preprocessor">#include "<a class="code" href="logging_8h.html">common/logging.h</a>"</span></div> |
| <div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="preprocessor">#include "gen-cpp/Descriptors_types.h"</span></div> |
| <div class="line"><a name="l00021"></a><span class="lineno"> 21</span> </div> |
| <div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="comment">// Codec libraries</span></div> |
| <div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="preprocessor">#include <zlib.h></span></div> |
| <div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="preprocessor">#include <bzlib.h></span></div> |
| <div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="preprocessor">#include <snappy.h></span></div> |
| <div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="preprocessor">#include <lz4.h></span></div> |
| <div class="line"><a name="l00027"></a><span class="lineno"> 27</span> </div> |
| <div class="line"><a name="l00028"></a><span class="lineno"> 28</span> <span class="preprocessor">#include "<a class="code" href="names_8h.html">common/names.h</a>"</span></div> |
| <div class="line"><a name="l00029"></a><span class="lineno"> 29</span> </div> |
| <div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="keyword">using namespace </span>impala;</div> |
| <div class="line"><a name="l00031"></a><span class="lineno"> 31</span> </div> |
| <div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="comment">// Output buffer size for streaming gzip</span></div> |
| <div class="line"><a name="l00033"></a><span class="lineno"><a class="line" href="decompress_8cc.html#a9448edeed1e4c2c2e69391d4cbbb256e"> 33</a></span> <span class="keyword">const</span> int64_t <a class="code" href="decompress_8cc.html#a9448edeed1e4c2c2e69391d4cbbb256e">STREAM_GZIP_OUT_BUF_SIZE</a> = 16 * 1024 * 1024;</div> |
| <div class="line"><a name="l00034"></a><span class="lineno"> 34</span> </div> |
| <div class="line"><a name="l00035"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#aca8878ef51a6e6bd670d3bdde5bc05ba"> 35</a></span> <a class="code" href="classimpala_1_1GzipDecompressor.html#aca8878ef51a6e6bd670d3bdde5bc05ba">GzipDecompressor::GzipDecompressor</a>(<a class="code" href="classimpala_1_1MemPool.html">MemPool</a>* mem_pool, <span class="keywordtype">bool</span> reuse_buffer, <span class="keywordtype">bool</span> is_deflate)</div> |
| <div class="line"><a name="l00036"></a><span class="lineno"> 36</span>  : <a class="code" href="classimpala_1_1Codec.html">Codec</a>(mem_pool, reuse_buffer),</div> |
| <div class="line"><a name="l00037"></a><span class="lineno"> 37</span>  is_deflate_(is_deflate) {</div> |
| <div class="line"><a name="l00038"></a><span class="lineno"> 38</span>  bzero(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>, <span class="keyword">sizeof</span>(<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>));</div> |
| <div class="line"><a name="l00039"></a><span class="lineno"> 39</span> }</div> |
| <div class="line"><a name="l00040"></a><span class="lineno"> 40</span> </div> |
| <div class="line"><a name="l00041"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#a11083f42122c1ae176019d5b392a3b69"> 41</a></span> <a class="code" href="classimpala_1_1GzipDecompressor.html#a11083f42122c1ae176019d5b392a3b69">GzipDecompressor::~GzipDecompressor</a>() {</div> |
| <div class="line"><a name="l00042"></a><span class="lineno"> 42</span>  (void)inflateEnd(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>);</div> |
| <div class="line"><a name="l00043"></a><span class="lineno"> 43</span> }</div> |
| <div class="line"><a name="l00044"></a><span class="lineno"> 44</span> </div> |
| <div class="line"><a name="l00045"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#a2e27e1fc7d43e02e2b48b9bb78db5931"> 45</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1GzipDecompressor.html#a2e27e1fc7d43e02e2b48b9bb78db5931">GzipDecompressor::Init</a>() {</div> |
| <div class="line"><a name="l00046"></a><span class="lineno"> 46</span>  <span class="keywordtype">int</span> ret;</div> |
| <div class="line"><a name="l00047"></a><span class="lineno"> 47</span>  <span class="comment">// Initialize to run either deflate or zlib/gzip format</span></div> |
| <div class="line"><a name="l00048"></a><span class="lineno"> 48</span>  <span class="keywordtype">int</span> window_bits = <a class="code" href="classimpala_1_1GzipDecompressor.html#ac6919e8e4c45879b01e56489b861fb8f">is_deflate_</a> ? -<a class="code" href="classimpala_1_1GzipDecompressor.html#a962735dde29923ed2049f927cecce94a">WINDOW_BITS</a> : <a class="code" href="classimpala_1_1GzipDecompressor.html#a962735dde29923ed2049f927cecce94a">WINDOW_BITS</a> | <a class="code" href="classimpala_1_1GzipDecompressor.html#a1e72ca38a6edbf62588957e6460e5743">DETECT_CODEC</a>;</div> |
| <div class="line"><a name="l00049"></a><span class="lineno"> 49</span>  <span class="keywordflow">if</span> ((ret = inflateInit2(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>, window_bits)) != Z_OK) {</div> |
| <div class="line"><a name="l00050"></a><span class="lineno"> 50</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"zlib inflateInit failed: "</span> + <span class="keywordtype">string</span>(<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg));</div> |
| <div class="line"><a name="l00051"></a><span class="lineno"> 51</span>  }</div> |
| <div class="line"><a name="l00052"></a><span class="lineno"> 52</span> </div> |
| <div class="line"><a name="l00053"></a><span class="lineno"> 53</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00054"></a><span class="lineno"> 54</span> }</div> |
| <div class="line"><a name="l00055"></a><span class="lineno"> 55</span> </div> |
| <div class="line"><a name="l00056"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#a6433c26c71c0fdb82f9688324313f54c"> 56</a></span> int64_t <a class="code" href="classimpala_1_1GzipDecompressor.html#a6433c26c71c0fdb82f9688324313f54c">GzipDecompressor::MaxOutputLen</a>(int64_t input_len, <span class="keyword">const</span> uint8_t* input) {</div> |
| <div class="line"><a name="l00057"></a><span class="lineno"> 57</span>  <span class="keywordflow">return</span> -1;</div> |
| <div class="line"><a name="l00058"></a><span class="lineno"> 58</span> }</div> |
| <div class="line"><a name="l00059"></a><span class="lineno"> 59</span> </div> |
| <div class="line"><a name="l00060"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#a976226c3d06c869a2e1c282ec11a4b3d"> 60</a></span> <span class="keywordtype">string</span> <a class="code" href="classimpala_1_1GzipDecompressor.html#a976226c3d06c869a2e1c282ec11a4b3d">GzipDecompressor::DebugStreamState</a>()<span class="keyword"> const </span>{</div> |
| <div class="line"><a name="l00061"></a><span class="lineno"> 61</span>  stringstream ss;</div> |
| <div class="line"><a name="l00062"></a><span class="lineno"> 62</span>  ss << <span class="stringliteral">"next_in="</span> << (<span class="keywordtype">void</span>*)<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.next_in;</div> |
| <div class="line"><a name="l00063"></a><span class="lineno"> 63</span>  ss << <span class="stringliteral">" avail_in="</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_in;</div> |
| <div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  ss << <span class="stringliteral">" total_in="</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.total_in;</div> |
| <div class="line"><a name="l00065"></a><span class="lineno"> 65</span>  ss << <span class="stringliteral">" next_out="</span> << (<span class="keywordtype">void</span>*)<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.next_out;</div> |
| <div class="line"><a name="l00066"></a><span class="lineno"> 66</span>  ss << <span class="stringliteral">" avail_out="</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_out;</div> |
| <div class="line"><a name="l00067"></a><span class="lineno"> 67</span>  ss << <span class="stringliteral">" total_out="</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.total_out;</div> |
| <div class="line"><a name="l00068"></a><span class="lineno"> 68</span>  <span class="keywordflow">return</span> ss.str();</div> |
| <div class="line"><a name="l00069"></a><span class="lineno"> 69</span> }</div> |
| <div class="line"><a name="l00070"></a><span class="lineno"> 70</span> </div> |
| <div class="line"><a name="l00071"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#a365af17fa61441862d200bc295ffc734"> 71</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1GzipDecompressor.html#a365af17fa61441862d200bc295ffc734">GzipDecompressor::ProcessBlockStreaming</a>(int64_t input_length, <span class="keyword">const</span> uint8_t* input,</div> |
| <div class="line"><a name="l00072"></a><span class="lineno"> 72</span>  int64_t* input_bytes_read, int64_t* output_length, uint8_t** output, <span class="keywordtype">bool</span>* eos) {</div> |
| <div class="line"><a name="l00073"></a><span class="lineno"> 73</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a> || <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> == NULL) {</div> |
| <div class="line"><a name="l00074"></a><span class="lineno"> 74</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = <a class="code" href="decompress_8cc.html#a9448edeed1e4c2c2e69391d4cbbb256e">STREAM_GZIP_OUT_BUF_SIZE</a>;</div> |
| <div class="line"><a name="l00075"></a><span class="lineno"> 75</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a375ea0ba90166ebb055a3d0c9a8936c9">memory_pool_</a>-><a class="code" href="classimpala_1_1MemPool.html#a190eb28e39279c901eda3d831a9fe726">Allocate</a>(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00076"></a><span class="lineno"> 76</span>  }</div> |
| <div class="line"><a name="l00077"></a><span class="lineno"> 77</span>  *output = <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>;</div> |
| <div class="line"><a name="l00078"></a><span class="lineno"> 78</span>  *output_length = <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>;</div> |
| <div class="line"><a name="l00079"></a><span class="lineno"> 79</span>  *input_bytes_read = 0;</div> |
| <div class="line"><a name="l00080"></a><span class="lineno"> 80</span>  *eos = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00081"></a><span class="lineno"> 81</span> </div> |
| <div class="line"><a name="l00082"></a><span class="lineno"> 82</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.next_in = <span class="keyword">const_cast<</span>Bytef*<span class="keyword">></span>(<span class="keyword">reinterpret_cast<</span><span class="keyword">const </span>Bytef*<span class="keyword">></span>(input));</div> |
| <div class="line"><a name="l00083"></a><span class="lineno"> 83</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_in = input_length;</div> |
| <div class="line"><a name="l00084"></a><span class="lineno"> 84</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.next_out = <span class="keyword">reinterpret_cast<</span>Bytef*<span class="keyword">></span>(*output);</div> |
| <div class="line"><a name="l00085"></a><span class="lineno"> 85</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_out = *output_length;</div> |
| <div class="line"><a name="l00086"></a><span class="lineno"> 86</span>  <a class="code" href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a> << <span class="stringliteral">"ProcessBlockStreaming() stream: "</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#a976226c3d06c869a2e1c282ec11a4b3d">DebugStreamState</a>();</div> |
| <div class="line"><a name="l00087"></a><span class="lineno"> 87</span> </div> |
| <div class="line"><a name="l00088"></a><span class="lineno"> 88</span>  <span class="keywordtype">int</span> ret = inflate(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>, Z_SYNC_FLUSH);</div> |
| <div class="line"><a name="l00089"></a><span class="lineno"> 89</span>  <span class="keywordflow">if</span> (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {</div> |
| <div class="line"><a name="l00090"></a><span class="lineno"> 90</span>  stringstream ss;</div> |
| <div class="line"><a name="l00091"></a><span class="lineno"> 91</span>  ss << <span class="stringliteral">"GzipDecompressor failed, ret="</span> << ret;</div> |
| <div class="line"><a name="l00092"></a><span class="lineno"> 92</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg != NULL) ss << <span class="stringliteral">" msg="</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg;</div> |
| <div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  }</div> |
| <div class="line"><a name="l00095"></a><span class="lineno"> 95</span> </div> |
| <div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  <span class="comment">// stream_.avail_out is the number of bytes *left* in the out buffer, but</span></div> |
| <div class="line"><a name="l00097"></a><span class="lineno"> 97</span>  <span class="comment">// we're interested in the number of bytes used.</span></div> |
| <div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  *output_length = *output_length - <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_out;</div> |
| <div class="line"><a name="l00099"></a><span class="lineno"> 99</span>  *input_bytes_read = input_length - <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_in;</div> |
| <div class="line"><a name="l00100"></a><span class="lineno"> 100</span>  <a class="code" href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a> << <span class="stringliteral">"inflate() ret="</span> << ret << <span class="stringliteral">" consumed="</span> << *input_bytes_read</div> |
| <div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  << <span class="stringliteral">" produced="</span> << *output_length << <span class="stringliteral">" stream: "</span> << <a class="code" href="classimpala_1_1GzipDecompressor.html#a976226c3d06c869a2e1c282ec11a4b3d">DebugStreamState</a>();</div> |
| <div class="line"><a name="l00102"></a><span class="lineno"> 102</span> </div> |
| <div class="line"><a name="l00103"></a><span class="lineno"> 103</span>  <span class="keywordflow">if</span> (ret == Z_BUF_ERROR) {</div> |
| <div class="line"><a name="l00104"></a><span class="lineno"> 104</span>  <span class="comment">// Z_BUF_ERROR is returned if no progress was made. This should be very unlikely.</span></div> |
| <div class="line"><a name="l00105"></a><span class="lineno"> 105</span>  <span class="comment">// The caller should check for this case (where 0 bytes were consumed, 0 bytes</span></div> |
| <div class="line"><a name="l00106"></a><span class="lineno"> 106</span>  <span class="comment">// produced) and try again with more input.</span></div> |
| <div class="line"><a name="l00107"></a><span class="lineno"> 107</span>  DCHECK_EQ(0, *output_length);</div> |
| <div class="line"><a name="l00108"></a><span class="lineno"> 108</span>  DCHECK_EQ(0, *input_bytes_read);</div> |
| <div class="line"><a name="l00109"></a><span class="lineno"> 109</span>  } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (ret == Z_STREAM_END) {</div> |
| <div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  *eos = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <span class="keywordflow">if</span> (inflateReset(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>) != Z_OK) {</div> |
| <div class="line"><a name="l00112"></a><span class="lineno"> 112</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"zlib inflateReset failed: "</span> + <span class="keywordtype">string</span>(<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg));</div> |
| <div class="line"><a name="l00113"></a><span class="lineno"> 113</span>  }</div> |
| <div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  }</div> |
| <div class="line"><a name="l00115"></a><span class="lineno"> 115</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00116"></a><span class="lineno"> 116</span> }</div> |
| <div class="line"><a name="l00117"></a><span class="lineno"> 117</span> </div> |
| <div class="line"><a name="l00118"></a><span class="lineno"><a class="line" href="classimpala_1_1GzipDecompressor.html#aef1273f520daba6e1552371d9adf2596"> 118</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1GzipDecompressor.html#aef1273f520daba6e1552371d9adf2596">GzipDecompressor::ProcessBlock</a>(<span class="keywordtype">bool</span> output_preallocated, int64_t input_length,</div> |
| <div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  <span class="keyword">const</span> uint8_t* input, int64_t* output_length, uint8_t** output) {</div> |
| <div class="line"><a name="l00120"></a><span class="lineno"> 120</span>  <span class="keywordflow">if</span> (output_preallocated && *output_length == 0) {</div> |
| <div class="line"><a name="l00121"></a><span class="lineno"> 121</span>  <span class="comment">// The zlib library does not allow *output to be NULL, even when output_length is 0</span></div> |
| <div class="line"><a name="l00122"></a><span class="lineno"> 122</span>  <span class="comment">// (inflate() will return Z_STREAM_ERROR). We don't consider this an error, so bail</span></div> |
| <div class="line"><a name="l00123"></a><span class="lineno"> 123</span>  <span class="comment">// early if no output is expected. Note that we don't signal an error if the input</span></div> |
| <div class="line"><a name="l00124"></a><span class="lineno"> 124</span>  <span class="comment">// actually contains compressed data.</span></div> |
| <div class="line"><a name="l00125"></a><span class="lineno"> 125</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00126"></a><span class="lineno"> 126</span>  }</div> |
| <div class="line"><a name="l00127"></a><span class="lineno"> 127</span> </div> |
| <div class="line"><a name="l00128"></a><span class="lineno"> 128</span>  <span class="keywordtype">bool</span> use_temp = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00129"></a><span class="lineno"> 129</span>  <span class="keywordflow">if</span> (!output_preallocated) {</div> |
| <div class="line"><a name="l00130"></a><span class="lineno"> 130</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a> || <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> == NULL) {</div> |
| <div class="line"><a name="l00131"></a><span class="lineno"> 131</span>  <span class="comment">// guess that we will need 2x the input length.</span></div> |
| <div class="line"><a name="l00132"></a><span class="lineno"> 132</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = input_length * 2;</div> |
| <div class="line"><a name="l00133"></a><span class="lineno"> 133</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00134"></a><span class="lineno"> 134</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Decompressor: block size is too big"</span>);</div> |
| <div class="line"><a name="l00135"></a><span class="lineno"> 135</span>  }</div> |
| <div class="line"><a name="l00136"></a><span class="lineno"> 136</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>->Allocate(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00137"></a><span class="lineno"> 137</span>  }</div> |
| <div class="line"><a name="l00138"></a><span class="lineno"> 138</span>  use_temp = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00139"></a><span class="lineno"> 139</span>  *output = <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>;</div> |
| <div class="line"><a name="l00140"></a><span class="lineno"> 140</span>  *output_length = <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>;</div> |
| <div class="line"><a name="l00141"></a><span class="lineno"> 141</span>  }</div> |
| <div class="line"><a name="l00142"></a><span class="lineno"> 142</span> </div> |
| <div class="line"><a name="l00143"></a><span class="lineno"> 143</span>  <span class="comment">// Reset the stream for this block</span></div> |
| <div class="line"><a name="l00144"></a><span class="lineno"> 144</span>  <span class="keywordflow">if</span> (inflateReset(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>) != Z_OK) {</div> |
| <div class="line"><a name="l00145"></a><span class="lineno"> 145</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"zlib inflateReset failed: "</span> + <span class="keywordtype">string</span>(<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg));</div> |
| <div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  }</div> |
| <div class="line"><a name="l00147"></a><span class="lineno"> 147</span> </div> |
| <div class="line"><a name="l00148"></a><span class="lineno"> 148</span>  <span class="keywordtype">int</span> ret = 0;</div> |
| <div class="line"><a name="l00149"></a><span class="lineno"> 149</span>  <span class="comment">// gzip can run in streaming mode or non-streaming mode. We only</span></div> |
| <div class="line"><a name="l00150"></a><span class="lineno"> 150</span>  <span class="comment">// support the non-streaming use case where we present it the entire</span></div> |
| <div class="line"><a name="l00151"></a><span class="lineno"> 151</span>  <span class="comment">// compressed input and a buffer big enough to contain the entire</span></div> |
| <div class="line"><a name="l00152"></a><span class="lineno"> 152</span>  <span class="comment">// compressed output. In the case where we don't know the output,</span></div> |
| <div class="line"><a name="l00153"></a><span class="lineno"> 153</span>  <span class="comment">// we just make a bigger buffer and try the non-streaming mode</span></div> |
| <div class="line"><a name="l00154"></a><span class="lineno"> 154</span>  <span class="comment">// from the beginning again.</span></div> |
| <div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  <span class="comment">// TODO: support streaming, especially for compressed text.</span></div> |
| <div class="line"><a name="l00156"></a><span class="lineno"> 156</span>  <span class="keywordflow">while</span> (ret != Z_STREAM_END) {</div> |
| <div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.next_in = <span class="keyword">const_cast<</span>Bytef*<span class="keyword">></span>(<span class="keyword">reinterpret_cast<</span><span class="keyword">const </span>Bytef*<span class="keyword">></span>(input));</div> |
| <div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_in = input_length;</div> |
| <div class="line"><a name="l00159"></a><span class="lineno"> 159</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.next_out = <span class="keyword">reinterpret_cast<</span>Bytef*<span class="keyword">></span>(*output);</div> |
| <div class="line"><a name="l00160"></a><span class="lineno"> 160</span>  <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_out = *output_length;</div> |
| <div class="line"><a name="l00161"></a><span class="lineno"> 161</span> </div> |
| <div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  <span class="keywordflow">if</span> (use_temp) {</div> |
| <div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  <span class="comment">// We don't know the output size, so this might fail.</span></div> |
| <div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  ret = inflate(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>, Z_PARTIAL_FLUSH);</div> |
| <div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  } <span class="keywordflow">else</span> {</div> |
| <div class="line"><a name="l00166"></a><span class="lineno"> 166</span>  <span class="comment">// We know the output size. In this case, we can use Z_FINISH</span></div> |
| <div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  <span class="comment">// which is more efficient.</span></div> |
| <div class="line"><a name="l00168"></a><span class="lineno"> 168</span>  ret = inflate(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>, Z_FINISH);</div> |
| <div class="line"><a name="l00169"></a><span class="lineno"> 169</span>  }</div> |
| <div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  <span class="keywordflow">if</span> (ret == Z_STREAM_END || ret != Z_OK) <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00171"></a><span class="lineno"> 171</span> </div> |
| <div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  <span class="comment">// Not enough output space.</span></div> |
| <div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  <span class="keywordflow">if</span> (!use_temp) {</div> |
| <div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  stringstream ss;</div> |
| <div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  ss << <span class="stringliteral">"Too small a buffer passed to GzipDecompressor. InputLength="</span></div> |
| <div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  << input_length << <span class="stringliteral">" OutputLength="</span> << *output_length;</div> |
| <div class="line"><a name="l00177"></a><span class="lineno"> 177</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00178"></a><span class="lineno"> 178</span>  }</div> |
| <div class="line"><a name="l00179"></a><span class="lineno"> 179</span> </div> |
| <div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  <span class="comment">// User didn't supply the buffer, double the buffer and try again.</span></div> |
| <div class="line"><a name="l00181"></a><span class="lineno"> 181</span>  <a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>->Clear();</div> |
| <div class="line"><a name="l00182"></a><span class="lineno"> 182</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> *= 2;</div> |
| <div class="line"><a name="l00183"></a><span class="lineno"> 183</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  stringstream ss;</div> |
| <div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  ss << <span class="stringliteral">"GzipDecompressor: block size is too big: "</span> << <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>;</div> |
| <div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00187"></a><span class="lineno"> 187</span>  }</div> |
| <div class="line"><a name="l00188"></a><span class="lineno"> 188</span> </div> |
| <div class="line"><a name="l00189"></a><span class="lineno"> 189</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>->Allocate(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00190"></a><span class="lineno"> 190</span>  *output = <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>;</div> |
| <div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  *output_length = <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>;</div> |
| <div class="line"><a name="l00192"></a><span class="lineno"> 192</span>  ret = inflateReset(&<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>);</div> |
| <div class="line"><a name="l00193"></a><span class="lineno"> 193</span>  }</div> |
| <div class="line"><a name="l00194"></a><span class="lineno"> 194</span> </div> |
| <div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  <span class="keywordflow">if</span> (ret != Z_STREAM_END) {</div> |
| <div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  stringstream ss;</div> |
| <div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  ss << <span class="stringliteral">"GzipDecompressor failed: "</span>;</div> |
| <div class="line"><a name="l00198"></a><span class="lineno"> 198</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg != NULL) ss << <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.msg;</div> |
| <div class="line"><a name="l00199"></a><span class="lineno"> 199</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00200"></a><span class="lineno"> 200</span>  }</div> |
| <div class="line"><a name="l00201"></a><span class="lineno"> 201</span> </div> |
| <div class="line"><a name="l00202"></a><span class="lineno"> 202</span>  <span class="comment">// stream_.avail_out is the number of bytes *left* in the out buffer, but</span></div> |
| <div class="line"><a name="l00203"></a><span class="lineno"> 203</span>  <span class="comment">// we're interested in the number of bytes used.</span></div> |
| <div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  *output_length = *output_length - <a class="code" href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">stream_</a>.avail_out;</div> |
| <div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  <span class="keywordflow">if</span> (use_temp) <a class="code" href="classimpala_1_1Codec.html#a375ea0ba90166ebb055a3d0c9a8936c9">memory_pool_</a>-><a class="code" href="classimpala_1_1MemPool.html#ab6774a8c0694da25e200601506d26237">AcquireData</a>(<a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>.get(), <a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a>);</div> |
| <div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00207"></a><span class="lineno"> 207</span> }</div> |
| <div class="line"><a name="l00208"></a><span class="lineno"> 208</span> </div> |
| <div class="line"><a name="l00209"></a><span class="lineno"><a class="line" href="classimpala_1_1BzipDecompressor.html#ad52d8b778cb9e1696e72d407f726bb59"> 209</a></span> <a class="code" href="classimpala_1_1BzipDecompressor.html#ad52d8b778cb9e1696e72d407f726bb59">BzipDecompressor::BzipDecompressor</a>(<a class="code" href="classimpala_1_1MemPool.html">MemPool</a>* mem_pool, <span class="keywordtype">bool</span> reuse_buffer)</div> |
| <div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  : <a class="code" href="classimpala_1_1Codec.html">Codec</a>(mem_pool, reuse_buffer) {</div> |
| <div class="line"><a name="l00211"></a><span class="lineno"> 211</span> }</div> |
| <div class="line"><a name="l00212"></a><span class="lineno"> 212</span> </div> |
| <div class="line"><a name="l00213"></a><span class="lineno"><a class="line" href="classimpala_1_1BzipDecompressor.html#a7b3c707cf38b61683fb70267e76ffeaf"> 213</a></span> int64_t <a class="code" href="classimpala_1_1BzipDecompressor.html#a7b3c707cf38b61683fb70267e76ffeaf">BzipDecompressor::MaxOutputLen</a>(int64_t input_len, <span class="keyword">const</span> uint8_t* input) {</div> |
| <div class="line"><a name="l00214"></a><span class="lineno"> 214</span>  <span class="keywordflow">return</span> -1;</div> |
| <div class="line"><a name="l00215"></a><span class="lineno"> 215</span> }</div> |
| <div class="line"><a name="l00216"></a><span class="lineno"> 216</span> </div> |
| <div class="line"><a name="l00217"></a><span class="lineno"><a class="line" href="classimpala_1_1BzipDecompressor.html#a7b9057b22f6a91dc3abbe8bde73d3e31"> 217</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1BzipDecompressor.html#a7b9057b22f6a91dc3abbe8bde73d3e31">BzipDecompressor::ProcessBlock</a>(<span class="keywordtype">bool</span> output_preallocated, int64_t input_length,</div> |
| <div class="line"><a name="l00218"></a><span class="lineno"> 218</span>  <span class="keyword">const</span> uint8_t* input, int64_t* output_length, uint8_t** output) {</div> |
| <div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  <span class="keywordflow">if</span> (output_preallocated && *output_length == 0) {</div> |
| <div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="comment">// Same problem as zlib library, see comment in GzipDecompressor::ProcessBlock()</span></div> |
| <div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  }</div> |
| <div class="line"><a name="l00223"></a><span class="lineno"> 223</span> </div> |
| <div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  <span class="keywordtype">bool</span> use_temp = <span class="keyword">false</span>;</div> |
| <div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  <span class="keywordflow">if</span> (output_preallocated) {</div> |
| <div class="line"><a name="l00226"></a><span class="lineno"> 226</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = *output_length;</div> |
| <div class="line"><a name="l00227"></a><span class="lineno"> 227</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = *output;</div> |
| <div class="line"><a name="l00228"></a><span class="lineno"> 228</span>  } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a> || <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> == NULL) {</div> |
| <div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  <span class="comment">// guess that we will need 2x the input length.</span></div> |
| <div class="line"><a name="l00230"></a><span class="lineno"> 230</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = input_length * 2;</div> |
| <div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00232"></a><span class="lineno"> 232</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Decompressor: block size is too big"</span>);</div> |
| <div class="line"><a name="l00233"></a><span class="lineno"> 233</span>  }</div> |
| <div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>->Allocate(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00235"></a><span class="lineno"> 235</span>  use_temp = <span class="keyword">true</span>;</div> |
| <div class="line"><a name="l00236"></a><span class="lineno"> 236</span>  }</div> |
| <div class="line"><a name="l00237"></a><span class="lineno"> 237</span> </div> |
| <div class="line"><a name="l00238"></a><span class="lineno"> 238</span>  <span class="keywordtype">int</span> ret = BZ_OUTBUFF_FULL;</div> |
| <div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  <span class="keywordtype">unsigned</span> <span class="keywordtype">int</span> outlen;</div> |
| <div class="line"><a name="l00240"></a><span class="lineno"> 240</span>  <span class="keywordflow">while</span> (ret == BZ_OUTBUFF_FULL) {</div> |
| <div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> == NULL) {</div> |
| <div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  DCHECK(!output_preallocated);</div> |
| <div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  <a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>->Clear();</div> |
| <div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> * 2;</div> |
| <div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Decompressor: block size is too big"</span>);</div> |
| <div class="line"><a name="l00247"></a><span class="lineno"> 247</span>  }</div> |
| <div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>->Allocate(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  }</div> |
| <div class="line"><a name="l00250"></a><span class="lineno"> 250</span>  outlen = <span class="keyword">static_cast<</span><span class="keywordtype">unsigned</span> <span class="keywordtype">int</span><span class="keyword">></span>(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00251"></a><span class="lineno"> 251</span>  <span class="keywordflow">if</span> ((ret = BZ2_bzBuffToBuffDecompress(reinterpret_cast<char*>(<a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>), &outlen,</div> |
| <div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  const_cast<char*>(reinterpret_cast<const char*>(input)),</div> |
| <div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  static_cast<unsigned int>(input_length), 0, 0)) == BZ_OUTBUFF_FULL) {</div> |
| <div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  <span class="keywordflow">if</span> (output_preallocated) {</div> |
| <div class="line"><a name="l00255"></a><span class="lineno"> 255</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Too small a buffer passed to BzipDecompressor"</span>);</div> |
| <div class="line"><a name="l00256"></a><span class="lineno"> 256</span>  }</div> |
| <div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = NULL;</div> |
| <div class="line"><a name="l00258"></a><span class="lineno"> 258</span>  }</div> |
| <div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  }</div> |
| <div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  <span class="keywordflow">if</span> (ret != BZ_OK) {</div> |
| <div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  stringstream ss;</div> |
| <div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  ss << <span class="stringliteral">"bzlib BZ2_bzBuffToBuffDecompressor failed: "</span> << ret;</div> |
| <div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00264"></a><span class="lineno"> 264</span> </div> |
| <div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  }</div> |
| <div class="line"><a name="l00266"></a><span class="lineno"> 266</span> </div> |
| <div class="line"><a name="l00267"></a><span class="lineno"> 267</span>  *output = <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>;</div> |
| <div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  *output_length = outlen;</div> |
| <div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  <span class="keywordflow">if</span> (use_temp) <a class="code" href="classimpala_1_1Codec.html#a375ea0ba90166ebb055a3d0c9a8936c9">memory_pool_</a>-><a class="code" href="classimpala_1_1MemPool.html#ab6774a8c0694da25e200601506d26237">AcquireData</a>(<a class="code" href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">temp_memory_pool_</a>.get(), <a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a>);</div> |
| <div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00271"></a><span class="lineno"> 271</span> }</div> |
| <div class="line"><a name="l00272"></a><span class="lineno"> 272</span> </div> |
| <div class="line"><a name="l00273"></a><span class="lineno"> 273</span> </div> |
| <div class="line"><a name="l00274"></a><span class="lineno"><a class="line" href="classimpala_1_1SnappyBlockDecompressor.html#ab35ca003f800b7eaf215cf8be3064ffa"> 274</a></span> <a class="code" href="classimpala_1_1SnappyBlockDecompressor.html#ab35ca003f800b7eaf215cf8be3064ffa">SnappyBlockDecompressor::SnappyBlockDecompressor</a>(<a class="code" href="classimpala_1_1MemPool.html">MemPool</a>* mem_pool, <span class="keywordtype">bool</span> reuse_buffer)</div> |
| <div class="line"><a name="l00275"></a><span class="lineno"> 275</span>  : <a class="code" href="classimpala_1_1Codec.html">Codec</a>(mem_pool, reuse_buffer) {</div> |
| <div class="line"><a name="l00276"></a><span class="lineno"> 276</span> }</div> |
| <div class="line"><a name="l00277"></a><span class="lineno"> 277</span> </div> |
| <div class="line"><a name="l00278"></a><span class="lineno"><a class="line" href="classimpala_1_1SnappyBlockDecompressor.html#a0aa0fb233daafcc313f6e0be8dd0575f"> 278</a></span> int64_t <a class="code" href="classimpala_1_1SnappyBlockDecompressor.html#a0aa0fb233daafcc313f6e0be8dd0575f">SnappyBlockDecompressor::MaxOutputLen</a>(int64_t input_len, <span class="keyword">const</span> uint8_t* input) {</div> |
| <div class="line"><a name="l00279"></a><span class="lineno"> 279</span>  <span class="keywordflow">return</span> -1;</div> |
| <div class="line"><a name="l00280"></a><span class="lineno"> 280</span> }</div> |
| <div class="line"><a name="l00281"></a><span class="lineno"> 281</span> </div> |
| <div class="line"><a name="l00282"></a><span class="lineno"> 282</span> <span class="comment">// Hadoop uses a block compression scheme on top of snappy. As per the hadoop docs</span></div> |
| <div class="line"><a name="l00283"></a><span class="lineno"> 283</span> <span class="comment">// the input is split into blocks. Each block "contains the uncompressed length for</span></div> |
| <div class="line"><a name="l00284"></a><span class="lineno"> 284</span> <span class="comment">// the block followed by one of more length-prefixed blocks of compressed data."</span></div> |
| <div class="line"><a name="l00285"></a><span class="lineno"> 285</span> <span class="comment">// This is essentially blocks of blocks.</span></div> |
| <div class="line"><a name="l00286"></a><span class="lineno"> 286</span> <span class="comment">// The outer block consists of:</span></div> |
| <div class="line"><a name="l00287"></a><span class="lineno"> 287</span> <span class="comment">// - 4 byte little endian uncompressed_size</span></div> |
| <div class="line"><a name="l00288"></a><span class="lineno"> 288</span> <span class="comment">// < inner blocks ></span></div> |
| <div class="line"><a name="l00289"></a><span class="lineno"> 289</span> <span class="comment">// ... repeated until input_len is consumed ..</span></div> |
| <div class="line"><a name="l00290"></a><span class="lineno"> 290</span> <span class="comment">// The inner blocks have:</span></div> |
| <div class="line"><a name="l00291"></a><span class="lineno"> 291</span> <span class="comment">// - 4-byte little endian compressed_size</span></div> |
| <div class="line"><a name="l00292"></a><span class="lineno"> 292</span> <span class="comment">// < snappy compressed block ></span></div> |
| <div class="line"><a name="l00293"></a><span class="lineno"> 293</span> <span class="comment">// - 4-byte little endian compressed_size</span></div> |
| <div class="line"><a name="l00294"></a><span class="lineno"> 294</span> <span class="comment">// < snappy compressed block ></span></div> |
| <div class="line"><a name="l00295"></a><span class="lineno"> 295</span> <span class="comment">// ... repeated until uncompressed_size from outer block is consumed ...</span></div> |
| <div class="line"><a name="l00296"></a><span class="lineno"> 296</span> </div> |
| <div class="line"><a name="l00297"></a><span class="lineno"> 297</span> <span class="comment">// Utility function to decompress snappy block compressed data. If size_only is true,</span></div> |
| <div class="line"><a name="l00298"></a><span class="lineno"> 298</span> <span class="comment">// this function does not decompress but only computes the output size and writes</span></div> |
| <div class="line"><a name="l00299"></a><span class="lineno"> 299</span> <span class="comment">// the result to *output_len.</span></div> |
| <div class="line"><a name="l00300"></a><span class="lineno"> 300</span> <span class="comment">// If size_only is false, output must be preallocated to output_len and this needs to</span></div> |
| <div class="line"><a name="l00301"></a><span class="lineno"> 301</span> <span class="comment">// be exactly big enough to hold the decompressed output.</span></div> |
| <div class="line"><a name="l00302"></a><span class="lineno"> 302</span> <span class="comment">// size_only is a O(1) operations (just reads a single varint for each snappy block).</span></div> |
| <div class="line"><a name="l00303"></a><span class="lineno"><a class="line" href="decompress_8cc.html#a0e48ae15935a6410222ce478f5af498c"> 303</a></span> <span class="keyword">static</span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="decompress_8cc.html#a0e48ae15935a6410222ce478f5af498c">SnappyBlockDecompress</a>(int64_t input_len, <span class="keyword">const</span> uint8_t* input,</div> |
| <div class="line"><a name="l00304"></a><span class="lineno"> 304</span>  <span class="keywordtype">bool</span> size_only, int64_t* output_len, <span class="keywordtype">char</span>* output) {</div> |
| <div class="line"><a name="l00305"></a><span class="lineno"> 305</span>  int64_t uncompressed_total_len = 0;</div> |
| <div class="line"><a name="l00306"></a><span class="lineno"> 306</span>  <span class="keywordflow">while</span> (input_len > 0) {</div> |
| <div class="line"><a name="l00307"></a><span class="lineno"> 307</span>  uint32_t uncompressed_block_len = ReadWriteUtil::GetInt<uint32_t>(input);</div> |
| <div class="line"><a name="l00308"></a><span class="lineno"> 308</span>  input += <span class="keyword">sizeof</span>(uint32_t);</div> |
| <div class="line"><a name="l00309"></a><span class="lineno"> 309</span>  input_len -= <span class="keyword">sizeof</span>(uint32_t);</div> |
| <div class="line"><a name="l00310"></a><span class="lineno"> 310</span> </div> |
| <div class="line"><a name="l00311"></a><span class="lineno"> 311</span>  <span class="keywordflow">if</span> (uncompressed_block_len > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">Codec::MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00312"></a><span class="lineno"> 312</span>  <span class="keywordflow">if</span> (uncompressed_total_len == 0) {</div> |
| <div class="line"><a name="l00313"></a><span class="lineno"> 313</span>  <span class="comment">// TODO: is this check really robust?</span></div> |
| <div class="line"><a name="l00314"></a><span class="lineno"> 314</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(TErrorCode::SNAPPY_DECOMPRESS_INVALID_BLOCK_SIZE,</div> |
| <div class="line"><a name="l00315"></a><span class="lineno"> 315</span>  uncompressed_block_len);</div> |
| <div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  }</div> |
| <div class="line"><a name="l00317"></a><span class="lineno"> 317</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00318"></a><span class="lineno"> 318</span>  }</div> |
| <div class="line"><a name="l00319"></a><span class="lineno"> 319</span> </div> |
| <div class="line"><a name="l00320"></a><span class="lineno"> 320</span>  <span class="keywordflow">if</span> (!size_only) {</div> |
| <div class="line"><a name="l00321"></a><span class="lineno"> 321</span>  int64_t remaining_output_size = *output_len - uncompressed_total_len;</div> |
| <div class="line"><a name="l00322"></a><span class="lineno"> 322</span>  DCHECK_GE(remaining_output_size, uncompressed_block_len);</div> |
| <div class="line"><a name="l00323"></a><span class="lineno"> 323</span>  }</div> |
| <div class="line"><a name="l00324"></a><span class="lineno"> 324</span> </div> |
| <div class="line"><a name="l00325"></a><span class="lineno"> 325</span>  <span class="keywordflow">while</span> (uncompressed_block_len > 0) {</div> |
| <div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  <span class="comment">// Read the length of the next snappy compressed block.</span></div> |
| <div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  <span class="keywordtype">size_t</span> compressed_len = ReadWriteUtil::GetInt<uint32_t>(input);</div> |
| <div class="line"><a name="l00328"></a><span class="lineno"> 328</span>  input += <span class="keyword">sizeof</span>(uint32_t);</div> |
| <div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  input_len -= <span class="keyword">sizeof</span>(uint32_t);</div> |
| <div class="line"><a name="l00330"></a><span class="lineno"> 330</span> </div> |
| <div class="line"><a name="l00331"></a><span class="lineno"> 331</span>  <span class="keywordflow">if</span> (compressed_len == 0 || compressed_len > input_len) {</div> |
| <div class="line"><a name="l00332"></a><span class="lineno"> 332</span>  <span class="keywordflow">if</span> (uncompressed_total_len == 0) {</div> |
| <div class="line"><a name="l00333"></a><span class="lineno"> 333</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(TErrorCode::SNAPPY_DECOMPRESS_INVALID_COMPRESSED_LENGTH);</div> |
| <div class="line"><a name="l00334"></a><span class="lineno"> 334</span>  }</div> |
| <div class="line"><a name="l00335"></a><span class="lineno"> 335</span>  input_len = 0;</div> |
| <div class="line"><a name="l00336"></a><span class="lineno"> 336</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00337"></a><span class="lineno"> 337</span>  }</div> |
| <div class="line"><a name="l00338"></a><span class="lineno"> 338</span> </div> |
| <div class="line"><a name="l00339"></a><span class="lineno"> 339</span>  <span class="comment">// Read how big the output will be.</span></div> |
| <div class="line"><a name="l00340"></a><span class="lineno"> 340</span>  <span class="keywordtype">size_t</span> uncompressed_len;</div> |
| <div class="line"><a name="l00341"></a><span class="lineno"> 341</span>  <span class="keywordflow">if</span> (!snappy::GetUncompressedLength(reinterpret_cast<const char*>(input),</div> |
| <div class="line"><a name="l00342"></a><span class="lineno"> 342</span>  input_len, &uncompressed_len)) {</div> |
| <div class="line"><a name="l00343"></a><span class="lineno"> 343</span>  <span class="keywordflow">if</span> (uncompressed_total_len == 0) {</div> |
| <div class="line"><a name="l00344"></a><span class="lineno"> 344</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(TErrorCode::SNAPPY_DECOMPRESS_UNCOMPRESSED_LENGTH_FAILED);</div> |
| <div class="line"><a name="l00345"></a><span class="lineno"> 345</span>  }</div> |
| <div class="line"><a name="l00346"></a><span class="lineno"> 346</span>  input_len = 0;</div> |
| <div class="line"><a name="l00347"></a><span class="lineno"> 347</span>  <span class="keywordflow">break</span>;</div> |
| <div class="line"><a name="l00348"></a><span class="lineno"> 348</span>  }</div> |
| <div class="line"><a name="l00349"></a><span class="lineno"> 349</span>  DCHECK_GT(uncompressed_len, 0);</div> |
| <div class="line"><a name="l00350"></a><span class="lineno"> 350</span> </div> |
| <div class="line"><a name="l00351"></a><span class="lineno"> 351</span>  <span class="keywordflow">if</span> (!size_only) {</div> |
| <div class="line"><a name="l00352"></a><span class="lineno"> 352</span>  <span class="comment">// Decompress this snappy block</span></div> |
| <div class="line"><a name="l00353"></a><span class="lineno"> 353</span>  <span class="keywordflow">if</span> (!snappy::RawUncompress(reinterpret_cast<const char*>(input),</div> |
| <div class="line"><a name="l00354"></a><span class="lineno"> 354</span>  compressed_len, output)) {</div> |
| <div class="line"><a name="l00355"></a><span class="lineno"> 355</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(TErrorCode::SNAPPY_DECOMPRESS_RAW_UNCOMPRESS_FAILED);</div> |
| <div class="line"><a name="l00356"></a><span class="lineno"> 356</span>  }</div> |
| <div class="line"><a name="l00357"></a><span class="lineno"> 357</span>  output += uncompressed_len;</div> |
| <div class="line"><a name="l00358"></a><span class="lineno"> 358</span>  }</div> |
| <div class="line"><a name="l00359"></a><span class="lineno"> 359</span> </div> |
| <div class="line"><a name="l00360"></a><span class="lineno"> 360</span>  input += compressed_len;</div> |
| <div class="line"><a name="l00361"></a><span class="lineno"> 361</span>  input_len -= compressed_len;</div> |
| <div class="line"><a name="l00362"></a><span class="lineno"> 362</span>  uncompressed_block_len -= uncompressed_len;</div> |
| <div class="line"><a name="l00363"></a><span class="lineno"> 363</span>  uncompressed_total_len += uncompressed_len;</div> |
| <div class="line"><a name="l00364"></a><span class="lineno"> 364</span>  }</div> |
| <div class="line"><a name="l00365"></a><span class="lineno"> 365</span>  }</div> |
| <div class="line"><a name="l00366"></a><span class="lineno"> 366</span> </div> |
| <div class="line"><a name="l00367"></a><span class="lineno"> 367</span>  <span class="keywordflow">if</span> (size_only) {</div> |
| <div class="line"><a name="l00368"></a><span class="lineno"> 368</span>  *output_len = uncompressed_total_len;</div> |
| <div class="line"><a name="l00369"></a><span class="lineno"> 369</span>  } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (*output_len != uncompressed_total_len) {</div> |
| <div class="line"><a name="l00370"></a><span class="lineno"> 370</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(TErrorCode::SNAPPY_DECOMPRESS_DECOMPRESS_SIZE_INCORRECT);</div> |
| <div class="line"><a name="l00371"></a><span class="lineno"> 371</span>  }</div> |
| <div class="line"><a name="l00372"></a><span class="lineno"> 372</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00373"></a><span class="lineno"> 373</span> }</div> |
| <div class="line"><a name="l00374"></a><span class="lineno"> 374</span> </div> |
| <div class="line"><a name="l00375"></a><span class="lineno"><a class="line" href="classimpala_1_1SnappyBlockDecompressor.html#a43ed119f147b43de0ade821ebd0b2d97"> 375</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1SnappyBlockDecompressor.html#a43ed119f147b43de0ade821ebd0b2d97">SnappyBlockDecompressor::ProcessBlock</a>(<span class="keywordtype">bool</span> output_preallocated, int64_t input_len,</div> |
| <div class="line"><a name="l00376"></a><span class="lineno"> 376</span>  <span class="keyword">const</span> uint8_t* input, int64_t* output_len, uint8_t** output) {</div> |
| <div class="line"><a name="l00377"></a><span class="lineno"> 377</span>  <span class="keywordflow">if</span> (!output_preallocated) {</div> |
| <div class="line"><a name="l00378"></a><span class="lineno"> 378</span>  <span class="comment">// If we don't know the size beforehand, compute it.</span></div> |
| <div class="line"><a name="l00379"></a><span class="lineno"> 379</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="decompress_8cc.html#a0e48ae15935a6410222ce478f5af498c">SnappyBlockDecompress</a>(input_len, input, <span class="keyword">true</span>, output_len, NULL));</div> |
| <div class="line"><a name="l00380"></a><span class="lineno"> 380</span> </div> |
| <div class="line"><a name="l00381"></a><span class="lineno"> 381</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a> || <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> == NULL || <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> < *output_len) {</div> |
| <div class="line"><a name="l00382"></a><span class="lineno"> 382</span>  <span class="comment">// Need to allocate a new buffer</span></div> |
| <div class="line"><a name="l00383"></a><span class="lineno"> 383</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = *output_len;</div> |
| <div class="line"><a name="l00384"></a><span class="lineno"> 384</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a375ea0ba90166ebb055a3d0c9a8936c9">memory_pool_</a>-><a class="code" href="classimpala_1_1MemPool.html#a190eb28e39279c901eda3d831a9fe726">Allocate</a>(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00385"></a><span class="lineno"> 385</span>  }</div> |
| <div class="line"><a name="l00386"></a><span class="lineno"> 386</span>  *output = <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>;</div> |
| <div class="line"><a name="l00387"></a><span class="lineno"> 387</span>  }</div> |
| <div class="line"><a name="l00388"></a><span class="lineno"> 388</span> </div> |
| <div class="line"><a name="l00389"></a><span class="lineno"> 389</span>  <span class="keywordflow">if</span> (*output_len > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00390"></a><span class="lineno"> 390</span>  <span class="comment">// TODO: is this check really robust?</span></div> |
| <div class="line"><a name="l00391"></a><span class="lineno"> 391</span>  stringstream ss;</div> |
| <div class="line"><a name="l00392"></a><span class="lineno"> 392</span>  ss << <span class="stringliteral">"Decompressor: block size is too big. Data is likely corrupt. "</span></div> |
| <div class="line"><a name="l00393"></a><span class="lineno"> 393</span>  << <span class="stringliteral">"Size: "</span> << *output_len;</div> |
| <div class="line"><a name="l00394"></a><span class="lineno"> 394</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(ss.str());</div> |
| <div class="line"><a name="l00395"></a><span class="lineno"> 395</span>  }</div> |
| <div class="line"><a name="l00396"></a><span class="lineno"> 396</span> </div> |
| <div class="line"><a name="l00397"></a><span class="lineno"> 397</span>  <span class="keywordtype">char</span>* out_ptr = <span class="keyword">reinterpret_cast<</span><span class="keywordtype">char</span>*<span class="keyword">></span>(*output);</div> |
| <div class="line"><a name="l00398"></a><span class="lineno"> 398</span>  <a class="code" href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a>(<a class="code" href="decompress_8cc.html#a0e48ae15935a6410222ce478f5af498c">SnappyBlockDecompress</a>(input_len, input, <span class="keyword">false</span>, output_len, out_ptr));</div> |
| <div class="line"><a name="l00399"></a><span class="lineno"> 399</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00400"></a><span class="lineno"> 400</span> }</div> |
| <div class="line"><a name="l00401"></a><span class="lineno"> 401</span> </div> |
| <div class="line"><a name="l00402"></a><span class="lineno"><a class="line" href="classimpala_1_1SnappyDecompressor.html#a3abf2595a6f18f38e8b38bf62059b25a"> 402</a></span> <a class="code" href="classimpala_1_1SnappyDecompressor.html#a3abf2595a6f18f38e8b38bf62059b25a">SnappyDecompressor::SnappyDecompressor</a>(<a class="code" href="classimpala_1_1MemPool.html">MemPool</a>* mem_pool, <span class="keywordtype">bool</span> reuse_buffer)</div> |
| <div class="line"><a name="l00403"></a><span class="lineno"> 403</span>  : <a class="code" href="classimpala_1_1Codec.html">Codec</a>(mem_pool, reuse_buffer) {</div> |
| <div class="line"><a name="l00404"></a><span class="lineno"> 404</span> }</div> |
| <div class="line"><a name="l00405"></a><span class="lineno"> 405</span> </div> |
| <div class="line"><a name="l00406"></a><span class="lineno"><a class="line" href="classimpala_1_1SnappyDecompressor.html#a0e079954029cc491ea383e6bd5eae7ae"> 406</a></span> int64_t <a class="code" href="classimpala_1_1SnappyDecompressor.html#a0e079954029cc491ea383e6bd5eae7ae">SnappyDecompressor::MaxOutputLen</a>(int64_t input_len, <span class="keyword">const</span> uint8_t* input) {</div> |
| <div class="line"><a name="l00407"></a><span class="lineno"> 407</span>  DCHECK(input != NULL);</div> |
| <div class="line"><a name="l00408"></a><span class="lineno"> 408</span>  <span class="keywordtype">size_t</span> result;</div> |
| <div class="line"><a name="l00409"></a><span class="lineno"> 409</span>  <span class="keywordflow">if</span> (!snappy::GetUncompressedLength(reinterpret_cast<const char*>(input),</div> |
| <div class="line"><a name="l00410"></a><span class="lineno"> 410</span>  input_len, &result)) {</div> |
| <div class="line"><a name="l00411"></a><span class="lineno"> 411</span>  <span class="keywordflow">return</span> -1;</div> |
| <div class="line"><a name="l00412"></a><span class="lineno"> 412</span>  }</div> |
| <div class="line"><a name="l00413"></a><span class="lineno"> 413</span>  <span class="keywordflow">return</span> result;</div> |
| <div class="line"><a name="l00414"></a><span class="lineno"> 414</span> }</div> |
| <div class="line"><a name="l00415"></a><span class="lineno"> 415</span> </div> |
| <div class="line"><a name="l00416"></a><span class="lineno"><a class="line" href="classimpala_1_1SnappyDecompressor.html#aa505af1ca269568f1fc79484e37f7c4f"> 416</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1SnappyDecompressor.html#aa505af1ca269568f1fc79484e37f7c4f">SnappyDecompressor::ProcessBlock</a>(<span class="keywordtype">bool</span> output_preallocated, int64_t input_length,</div> |
| <div class="line"><a name="l00417"></a><span class="lineno"> 417</span>  <span class="keyword">const</span> uint8_t* input, int64_t* output_length, uint8_t** output) {</div> |
| <div class="line"><a name="l00418"></a><span class="lineno"> 418</span>  <span class="keywordflow">if</span> (!output_preallocated) {</div> |
| <div class="line"><a name="l00419"></a><span class="lineno"> 419</span>  int64_t uncompressed_length = <a class="code" href="classimpala_1_1SnappyDecompressor.html#a0e079954029cc491ea383e6bd5eae7ae">MaxOutputLen</a>(input_length, input);</div> |
| <div class="line"><a name="l00420"></a><span class="lineno"> 420</span>  <span class="keywordflow">if</span> (uncompressed_length < 0) <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Snappy: GetUncompressedLength failed"</span>);</div> |
| <div class="line"><a name="l00421"></a><span class="lineno"> 421</span>  <span class="keywordflow">if</span> (!<a class="code" href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">reuse_buffer_</a> || <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> == NULL || <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> < uncompressed_length) {</div> |
| <div class="line"><a name="l00422"></a><span class="lineno"> 422</span>  <a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> = uncompressed_length;</div> |
| <div class="line"><a name="l00423"></a><span class="lineno"> 423</span>  <span class="keywordflow">if</span> (<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a> > <a class="code" href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">MAX_BLOCK_SIZE</a>) {</div> |
| <div class="line"><a name="l00424"></a><span class="lineno"> 424</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Decompressor: block size is too big"</span>);</div> |
| <div class="line"><a name="l00425"></a><span class="lineno"> 425</span>  }</div> |
| <div class="line"><a name="l00426"></a><span class="lineno"> 426</span>  <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a> = <a class="code" href="classimpala_1_1Codec.html#a375ea0ba90166ebb055a3d0c9a8936c9">memory_pool_</a>-><a class="code" href="classimpala_1_1MemPool.html#a190eb28e39279c901eda3d831a9fe726">Allocate</a>(<a class="code" href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">buffer_length_</a>);</div> |
| <div class="line"><a name="l00427"></a><span class="lineno"> 427</span>  }</div> |
| <div class="line"><a name="l00428"></a><span class="lineno"> 428</span>  *output = <a class="code" href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">out_buffer_</a>;</div> |
| <div class="line"><a name="l00429"></a><span class="lineno"> 429</span>  *output_length = uncompressed_length;</div> |
| <div class="line"><a name="l00430"></a><span class="lineno"> 430</span>  }</div> |
| <div class="line"><a name="l00431"></a><span class="lineno"> 431</span> </div> |
| <div class="line"><a name="l00432"></a><span class="lineno"> 432</span>  <span class="keywordflow">if</span> (!snappy::RawUncompress(reinterpret_cast<const char*>(input),</div> |
| <div class="line"><a name="l00433"></a><span class="lineno"> 433</span>  static_cast<size_t>(input_length), reinterpret_cast<char*>(*output))) {</div> |
| <div class="line"><a name="l00434"></a><span class="lineno"> 434</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Snappy: RawUncompress failed"</span>);</div> |
| <div class="line"><a name="l00435"></a><span class="lineno"> 435</span>  }</div> |
| <div class="line"><a name="l00436"></a><span class="lineno"> 436</span> </div> |
| <div class="line"><a name="l00437"></a><span class="lineno"> 437</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00438"></a><span class="lineno"> 438</span> }</div> |
| <div class="line"><a name="l00439"></a><span class="lineno"> 439</span> </div> |
| <div class="line"><a name="l00440"></a><span class="lineno"><a class="line" href="classimpala_1_1Lz4Decompressor.html#a5b5fba68f171d46d71459191fae38e90"> 440</a></span> <a class="code" href="classimpala_1_1Lz4Decompressor.html#a5b5fba68f171d46d71459191fae38e90">Lz4Decompressor::Lz4Decompressor</a>(<a class="code" href="classimpala_1_1MemPool.html">MemPool</a>* mem_pool, <span class="keywordtype">bool</span> reuse_buffer)</div> |
| <div class="line"><a name="l00441"></a><span class="lineno"> 441</span>  : <a class="code" href="classimpala_1_1Codec.html">Codec</a>(mem_pool, reuse_buffer) {</div> |
| <div class="line"><a name="l00442"></a><span class="lineno"> 442</span> }</div> |
| <div class="line"><a name="l00443"></a><span class="lineno"> 443</span> </div> |
| <div class="line"><a name="l00444"></a><span class="lineno"><a class="line" href="classimpala_1_1Lz4Decompressor.html#ab82dacc81c74062801099636953d455a"> 444</a></span> int64_t <a class="code" href="classimpala_1_1Lz4Decompressor.html#ab82dacc81c74062801099636953d455a">Lz4Decompressor::MaxOutputLen</a>(int64_t input_len, <span class="keyword">const</span> uint8_t* input) {</div> |
| <div class="line"><a name="l00445"></a><span class="lineno"> 445</span>  DCHECK(input != NULL) << <span class="stringliteral">"Passed null input to Lz4 Decompressor"</span>;</div> |
| <div class="line"><a name="l00446"></a><span class="lineno"> 446</span>  <span class="keywordflow">return</span> -1;</div> |
| <div class="line"><a name="l00447"></a><span class="lineno"> 447</span> }</div> |
| <div class="line"><a name="l00448"></a><span class="lineno"> 448</span> </div> |
| <div class="line"><a name="l00449"></a><span class="lineno"><a class="line" href="classimpala_1_1Lz4Decompressor.html#a2ac5eae95e0478116e0602798ffe511e"> 449</a></span> <a class="code" href="classimpala_1_1Status.html">Status</a> <a class="code" href="classimpala_1_1Lz4Decompressor.html#a2ac5eae95e0478116e0602798ffe511e">Lz4Decompressor::ProcessBlock</a>(<span class="keywordtype">bool</span> output_preallocated, int64_t input_length,</div> |
| <div class="line"><a name="l00450"></a><span class="lineno"> 450</span>  <span class="keyword">const</span> uint8_t* input, int64_t* output_length, uint8_t** output) {</div> |
| <div class="line"><a name="l00451"></a><span class="lineno"> 451</span>  DCHECK(output_preallocated) << <span class="stringliteral">"Lz4 Codec implementation must have allocated output"</span>;</div> |
| <div class="line"><a name="l00452"></a><span class="lineno"> 452</span>  <span class="comment">// LZ4_uncompress will cause a segmentation fault if passed a NULL output.</span></div> |
| <div class="line"><a name="l00453"></a><span class="lineno"> 453</span>  <span class="keywordflow">if</span>(*output_length == 0) <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00454"></a><span class="lineno"> 454</span>  <span class="keywordflow">if</span> (LZ4_uncompress(reinterpret_cast<const char*>(input),</div> |
| <div class="line"><a name="l00455"></a><span class="lineno"> 455</span>  reinterpret_cast<char*>(*output), *output_length) != input_length) {</div> |
| <div class="line"><a name="l00456"></a><span class="lineno"> 456</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html">Status</a>(<span class="stringliteral">"Lz4: uncompress failed"</span>);</div> |
| <div class="line"><a name="l00457"></a><span class="lineno"> 457</span>  }</div> |
| <div class="line"><a name="l00458"></a><span class="lineno"> 458</span> </div> |
| <div class="line"><a name="l00459"></a><span class="lineno"> 459</span>  <span class="keywordflow">return</span> <a class="code" href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">Status::OK</a>;</div> |
| <div class="line"><a name="l00460"></a><span class="lineno"> 460</span> }</div> |
| <div class="ttc" id="classimpala_1_1Lz4Decompressor_html_ab82dacc81c74062801099636953d455a"><div class="ttname"><a href="classimpala_1_1Lz4Decompressor.html#ab82dacc81c74062801099636953d455a">impala::Lz4Decompressor::MaxOutputLen</a></div><div class="ttdeci">virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t *input=NULL)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00444">decompress.cc:444</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_aef1273f520daba6e1552371d9adf2596"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#aef1273f520daba6e1552371d9adf2596">impala::GzipDecompressor::ProcessBlock</a></div><div class="ttdeci">virtual Status ProcessBlock(bool output_preallocated, int64_t input_length, const uint8_t *input, int64_t *output_length, uint8_t **output)</div><div class="ttdoc">Process a block of data, either compressing or decompressing it. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00118">decompress.cc:118</a></div></div> |
| <div class="ttc" id="classimpala_1_1Lz4Decompressor_html_a2ac5eae95e0478116e0602798ffe511e"><div class="ttname"><a href="classimpala_1_1Lz4Decompressor.html#a2ac5eae95e0478116e0602798ffe511e">impala::Lz4Decompressor::ProcessBlock</a></div><div class="ttdeci">virtual Status ProcessBlock(bool output_preallocated, int64_t input_length, const uint8_t *input, int64_t *output_length, uint8_t **output)</div><div class="ttdoc">Process a block of data, either compressing or decompressing it. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00449">decompress.cc:449</a></div></div> |
| <div class="ttc" id="classimpala_1_1SnappyBlockDecompressor_html_ab35ca003f800b7eaf215cf8be3064ffa"><div class="ttname"><a href="classimpala_1_1SnappyBlockDecompressor.html#ab35ca003f800b7eaf215cf8be3064ffa">impala::SnappyBlockDecompressor::SnappyBlockDecompressor</a></div><div class="ttdeci">SnappyBlockDecompressor(MemPool *mem_pool, bool reuse_buffer)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00274">decompress.cc:274</a></div></div> |
| <div class="ttc" id="classimpala_1_1BzipDecompressor_html_ad52d8b778cb9e1696e72d407f726bb59"><div class="ttname"><a href="classimpala_1_1BzipDecompressor.html#ad52d8b778cb9e1696e72d407f726bb59">impala::BzipDecompressor::BzipDecompressor</a></div><div class="ttdeci">BzipDecompressor(MemPool *mem_pool, bool reuse_buffer)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00209">decompress.cc:209</a></div></div> |
| <div class="ttc" id="status_8h_html_a85f7d0e774e15eb35b74f53264305e16"><div class="ttname"><a href="status_8h.html#a85f7d0e774e15eb35b74f53264305e16">RETURN_IF_ERROR</a></div><div class="ttdeci">#define RETURN_IF_ERROR(stmt)</div><div class="ttdoc">some generally useful macros </div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00242">status.h:242</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a11083f42122c1ae176019d5b392a3b69"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a11083f42122c1ae176019d5b392a3b69">impala::GzipDecompressor::~GzipDecompressor</a></div><div class="ttdeci">virtual ~GzipDecompressor()</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00041">decompress.cc:41</a></div></div> |
| <div class="ttc" id="classimpala_1_1BzipDecompressor_html_a7b9057b22f6a91dc3abbe8bde73d3e31"><div class="ttname"><a href="classimpala_1_1BzipDecompressor.html#a7b9057b22f6a91dc3abbe8bde73d3e31">impala::BzipDecompressor::ProcessBlock</a></div><div class="ttdeci">virtual Status ProcessBlock(bool output_preallocated, int64_t input_length, const uint8_t *input, int64_t *output_length, uint8_t **output)</div><div class="ttdoc">Process a block of data, either compressing or decompressing it. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00217">decompress.cc:217</a></div></div> |
| <div class="ttc" id="classimpala_1_1MemPool_html_ab6774a8c0694da25e200601506d26237"><div class="ttname"><a href="classimpala_1_1MemPool.html#ab6774a8c0694da25e200601506d26237">impala::MemPool::AcquireData</a></div><div class="ttdeci">void AcquireData(MemPool *src, bool keep_current)</div><div class="ttdef"><b>Definition:</b> <a href="mem-pool_8cc_source.html#l00161">mem-pool.cc:161</a></div></div> |
| <div class="ttc" id="decompress_8cc_html_a0e48ae15935a6410222ce478f5af498c"><div class="ttname"><a href="decompress_8cc.html#a0e48ae15935a6410222ce478f5af498c">SnappyBlockDecompress</a></div><div class="ttdeci">static Status SnappyBlockDecompress(int64_t input_len, const uint8_t *input, bool size_only, int64_t *output_len, char *output)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00303">decompress.cc:303</a></div></div> |
| <div class="ttc" id="classimpala_1_1BzipDecompressor_html_a7b3c707cf38b61683fb70267e76ffeaf"><div class="ttname"><a href="classimpala_1_1BzipDecompressor.html#a7b3c707cf38b61683fb70267e76ffeaf">impala::BzipDecompressor::MaxOutputLen</a></div><div class="ttdeci">virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t *input=NULL)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00213">decompress.cc:213</a></div></div> |
| <div class="ttc" id="logging_8h_html"><div class="ttname"><a href="logging_8h.html">logging.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a93bd812c02e9838a8397a78cabe494cf"><div class="ttname"><a href="classimpala_1_1Codec.html#a93bd812c02e9838a8397a78cabe494cf">impala::Codec::reuse_buffer_</a></div><div class="ttdeci">bool reuse_buffer_</div><div class="ttdoc">Can we reuse the output buffer or do we need to allocate on each call? </div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00161">codec.h:161</a></div></div> |
| <div class="ttc" id="classimpala_1_1SnappyBlockDecompressor_html_a0aa0fb233daafcc313f6e0be8dd0575f"><div class="ttname"><a href="classimpala_1_1SnappyBlockDecompressor.html#a0aa0fb233daafcc313f6e0be8dd0575f">impala::SnappyBlockDecompressor::MaxOutputLen</a></div><div class="ttdeci">virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t *input=NULL)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00278">decompress.cc:278</a></div></div> |
| <div class="ttc" id="decompress_8h_html"><div class="ttname"><a href="decompress_8h.html">decompress.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html"><div class="ttname"><a href="classimpala_1_1Status.html">impala::Status</a></div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00081">status.h:81</a></div></div> |
| <div class="ttc" id="classimpala_1_1Lz4Decompressor_html_a5b5fba68f171d46d71459191fae38e90"><div class="ttname"><a href="classimpala_1_1Lz4Decompressor.html#a5b5fba68f171d46d71459191fae38e90">impala::Lz4Decompressor::Lz4Decompressor</a></div><div class="ttdeci">Lz4Decompressor(MemPool *mem_pool=NULL, bool reuse_buffer=false)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00440">decompress.cc:440</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a962735dde29923ed2049f927cecce94a"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a962735dde29923ed2049f927cecce94a">impala::GzipDecompressor::WINDOW_BITS</a></div><div class="ttdeci">static const int WINDOW_BITS</div><div class="ttdoc">These are magic numbers from zlib.h. Not clear why they are not defined there. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8h_source.html#l00051">decompress.h:51</a></div></div> |
| <div class="ttc" id="classimpala_1_1MemPool_html"><div class="ttname"><a href="classimpala_1_1MemPool.html">impala::MemPool</a></div><div class="ttdef"><b>Definition:</b> <a href="mem-pool_8h_source.html#l00077">mem-pool.h:77</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a976226c3d06c869a2e1c282ec11a4b3d"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a976226c3d06c869a2e1c282ec11a4b3d">impala::GzipDecompressor::DebugStreamState</a></div><div class="ttdeci">std::string DebugStreamState() const </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00060">decompress.cc:60</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a1e72ca38a6edbf62588957e6460e5743"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a1e72ca38a6edbf62588957e6460e5743">impala::GzipDecompressor::DETECT_CODEC</a></div><div class="ttdeci">static const int DETECT_CODEC</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8h_source.html#l00052">decompress.h:52</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_aba118f063dade9e76efcddb2cd2ad3bf"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#aba118f063dade9e76efcddb2cd2ad3bf">impala::GzipDecompressor::stream_</a></div><div class="ttdeci">z_stream stream_</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8h_source.html#l00048">decompress.h:48</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a079688b7a4dde5a4e60467b449901e12"><div class="ttname"><a href="classimpala_1_1Codec.html#a079688b7a4dde5a4e60467b449901e12">impala::Codec::MAX_BLOCK_SIZE</a></div><div class="ttdeci">static const int MAX_BLOCK_SIZE</div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00140">codec.h:140</a></div></div> |
| <div class="ttc" id="logging_8h_html_a6ccc2106c47622db4e52a401a415fc58"><div class="ttname"><a href="logging_8h.html#a6ccc2106c47622db4e52a401a415fc58">VLOG_ROW</a></div><div class="ttdeci">#define VLOG_ROW</div><div class="ttdef"><b>Definition:</b> <a href="logging_8h_source.html#l00059">logging.h:59</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a365af17fa61441862d200bc295ffc734"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a365af17fa61441862d200bc295ffc734">impala::GzipDecompressor::ProcessBlockStreaming</a></div><div class="ttdeci">virtual Status ProcessBlockStreaming(int64_t input_length, const uint8_t *input, int64_t *input_bytes_read, int64_t *output_length, uint8_t **output, bool *eos)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00071">decompress.cc:71</a></div></div> |
| <div class="ttc" id="runtime-state_8h_html"><div class="ttname"><a href="runtime-state_8h.html">runtime-state.h</a></div></div> |
| <div class="ttc" id="read-write-util_8h_html"><div class="ttname"><a href="read-write-util_8h.html">read-write-util.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1SnappyBlockDecompressor_html_a43ed119f147b43de0ade821ebd0b2d97"><div class="ttname"><a href="classimpala_1_1SnappyBlockDecompressor.html#a43ed119f147b43de0ade821ebd0b2d97">impala::SnappyBlockDecompressor::ProcessBlock</a></div><div class="ttdeci">virtual Status ProcessBlock(bool output_preallocated, int64_t input_length, const uint8_t *input, int64_t *output_length, uint8_t **output)</div><div class="ttdoc">Process a block of data, either compressing or decompressing it. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00375">decompress.cc:375</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_afcd266a7fd72165ffeb2e25159374889"><div class="ttname"><a href="classimpala_1_1Codec.html#afcd266a7fd72165ffeb2e25159374889">impala::Codec::out_buffer_</a></div><div class="ttdeci">uint8_t * out_buffer_</div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00165">codec.h:165</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a82fbdfbac9ea0b9d14004ebb0e501058"><div class="ttname"><a href="classimpala_1_1Codec.html#a82fbdfbac9ea0b9d14004ebb0e501058">impala::Codec::buffer_length_</a></div><div class="ttdeci">int64_t buffer_length_</div><div class="ttdoc">Length of the output buffer. </div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00168">codec.h:168</a></div></div> |
| <div class="ttc" id="classimpala_1_1SnappyDecompressor_html_a0e079954029cc491ea383e6bd5eae7ae"><div class="ttname"><a href="classimpala_1_1SnappyDecompressor.html#a0e079954029cc491ea383e6bd5eae7ae">impala::SnappyDecompressor::MaxOutputLen</a></div><div class="ttdeci">virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t *input=NULL)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00406">decompress.cc:406</a></div></div> |
| <div class="ttc" id="classimpala_1_1Status_html_a580565665ea944eb64f3f495b1bee1e0"><div class="ttname"><a href="classimpala_1_1Status.html#a580565665ea944eb64f3f495b1bee1e0">impala::Status::OK</a></div><div class="ttdeci">static const Status OK</div><div class="ttdef"><b>Definition:</b> <a href="status_8h_source.html#l00087">status.h:87</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html"><div class="ttname"><a href="classimpala_1_1Codec.html">impala::Codec</a></div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00041">codec.h:41</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a7ec5c623611416c9a873ea242d0d5035"><div class="ttname"><a href="classimpala_1_1Codec.html#a7ec5c623611416c9a873ea242d0d5035">impala::Codec::temp_memory_pool_</a></div><div class="ttdeci">boost::scoped_ptr< MemPool > temp_memory_pool_</div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00158">codec.h:158</a></div></div> |
| <div class="ttc" id="names_8h_html"><div class="ttname"><a href="names_8h.html">names.h</a></div></div> |
| <div class="ttc" id="classimpala_1_1SnappyDecompressor_html_aa505af1ca269568f1fc79484e37f7c4f"><div class="ttname"><a href="classimpala_1_1SnappyDecompressor.html#aa505af1ca269568f1fc79484e37f7c4f">impala::SnappyDecompressor::ProcessBlock</a></div><div class="ttdeci">virtual Status ProcessBlock(bool output_preallocated, int64_t input_length, const uint8_t *input, int64_t *output_length, uint8_t **output)</div><div class="ttdoc">Process a block of data, either compressing or decompressing it. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00416">decompress.cc:416</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_ac6919e8e4c45879b01e56489b861fb8f"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#ac6919e8e4c45879b01e56489b861fb8f">impala::GzipDecompressor::is_deflate_</a></div><div class="ttdeci">bool is_deflate_</div><div class="ttdoc">If set assume deflate format, otherwise zlib or gzip. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8h_source.html#l00046">decompress.h:46</a></div></div> |
| <div class="ttc" id="decompress_8cc_html_a9448edeed1e4c2c2e69391d4cbbb256e"><div class="ttname"><a href="decompress_8cc.html#a9448edeed1e4c2c2e69391d4cbbb256e">STREAM_GZIP_OUT_BUF_SIZE</a></div><div class="ttdeci">const int64_t STREAM_GZIP_OUT_BUF_SIZE</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00033">decompress.cc:33</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a6433c26c71c0fdb82f9688324313f54c"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a6433c26c71c0fdb82f9688324313f54c">impala::GzipDecompressor::MaxOutputLen</a></div><div class="ttdeci">virtual int64_t MaxOutputLen(int64_t input_len, const uint8_t *input=NULL)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00056">decompress.cc:56</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_aca8878ef51a6e6bd670d3bdde5bc05ba"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#aca8878ef51a6e6bd670d3bdde5bc05ba">impala::GzipDecompressor::GzipDecompressor</a></div><div class="ttdeci">GzipDecompressor(MemPool *mem_pool=NULL, bool reuse_buffer=false, bool is_deflate=false)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00035">decompress.cc:35</a></div></div> |
| <div class="ttc" id="classimpala_1_1GzipDecompressor_html_a2e27e1fc7d43e02e2b48b9bb78db5931"><div class="ttname"><a href="classimpala_1_1GzipDecompressor.html#a2e27e1fc7d43e02e2b48b9bb78db5931">impala::GzipDecompressor::Init</a></div><div class="ttdeci">virtual Status Init()</div><div class="ttdoc">Initialize the codec. This should only be called once. </div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00045">decompress.cc:45</a></div></div> |
| <div class="ttc" id="classimpala_1_1Codec_html_a375ea0ba90166ebb055a3d0c9a8936c9"><div class="ttname"><a href="classimpala_1_1Codec.html#a375ea0ba90166ebb055a3d0c9a8936c9">impala::Codec::memory_pool_</a></div><div class="ttdeci">MemPool * memory_pool_</div><div class="ttdoc">Pool to allocate the buffer to hold transformed data. </div><div class="ttdef"><b>Definition:</b> <a href="codec_8h_source.html#l00154">codec.h:154</a></div></div> |
| <div class="ttc" id="classimpala_1_1MemPool_html_a190eb28e39279c901eda3d831a9fe726"><div class="ttname"><a href="classimpala_1_1MemPool.html#a190eb28e39279c901eda3d831a9fe726">impala::MemPool::Allocate</a></div><div class="ttdeci">uint8_t * Allocate(int size)</div><div class="ttdef"><b>Definition:</b> <a href="mem-pool_8h_source.html#l00092">mem-pool.h:92</a></div></div> |
| <div class="ttc" id="classimpala_1_1SnappyDecompressor_html_a3abf2595a6f18f38e8b38bf62059b25a"><div class="ttname"><a href="classimpala_1_1SnappyDecompressor.html#a3abf2595a6f18f38e8b38bf62059b25a">impala::SnappyDecompressor::SnappyDecompressor</a></div><div class="ttdeci">SnappyDecompressor(MemPool *mem_pool=NULL, bool reuse_buffer=false)</div><div class="ttdef"><b>Definition:</b> <a href="decompress_8cc_source.html#l00402">decompress.cc:402</a></div></div> |
| </div><!-- fragment --></div><!-- contents --> |
| </div><!-- doc-content --> |
| <!-- start footer part --> |
| <div id="nav-path" class="navpath"><!-- id is needed for treeview function! --> |
| <ul> |
| <li class="navelem"><a class="el" href="dir_e5d120be6b5e8a44336cbfd013b25604.html">be</a></li><li class="navelem"><a class="el" href="dir_68cf7cafb51a962d5bc4848b83cab0de.html">src</a></li><li class="navelem"><a class="el" href="dir_6939c8639261a9303da2cde491b82898.html">util</a></li><li class="navelem"><a class="el" href="decompress_8cc.html">decompress.cc</a></li> |
| <li class="footer">Generated on Thu May 7 2015 16:10:37 for Impala by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.6 </li> |
| </ul> |
| </div> |
| </body> |
| </html> |
