| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <title>MADlib: sketch.sql_in Source File</title> |
| |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <script src="../mathjax/MathJax.js"> |
| MathJax.Hub.Config({ |
| extensions: ["tex2jax.js", "TeX/AMSmath.js", "TeX/AMSsymbols.js"], |
| jax: ["input/TeX","output/HTML-CSS"], |
| }); |
| </script> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div! --> |
| |
| |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| |
| |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">MADlib |
|  <span id="projectnumber">0.6</span> <span style="font-size:10pt; font-style:italic"><a href="../latest/./sketch_8sql__in_source.html"> A newer version is available</a></span> |
| </div> |
| <div id="projectbrief">User Documentation</div> |
| </td> |
| |
| |
| |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| |
| <!-- Generated by Doxygen 1.7.5.1 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="modules.html"><span>Modules</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| initNavTree('sketch_8sql__in.html',''); |
| </script> |
| <div id="doc-content"> |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">sketch.sql_in</div> </div> |
| </div> |
| <div class="contents"> |
| <a href="sketch_8sql__in.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* ----------------------------------------------------------------------- */</span><span class="comment">/**</span> |
| <a name="l00002"></a>00002 <span class="comment"> *</span> |
| <a name="l00003"></a>00003 <span class="comment"> * @file sketch.sql_in</span> |
| <a name="l00004"></a>00004 <span class="comment"> *</span> |
| <a name="l00005"></a>00005 <span class="comment"> * @brief SQL functions for sketch-based approximations of descriptive statistics</span> |
| <a name="l00006"></a>00006 <span class="comment"> * @date April 2011</span> |
| <a name="l00007"></a>00007 <span class="comment"> *</span> |
| <a name="l00008"></a>00008 <span class="comment"> * @sa For a brief introduction to sketches, see the module description grp_sketches</span> |
| <a name="l00009"></a>00009 <span class="comment"> *</span> |
| <a name="l00010"></a>00010 <span class="comment"> */</span><span class="comment">/* ----------------------------------------------------------------------- */</span> |
| <a name="l00011"></a>00011 |
| <a name="l00012"></a>00012 m4_include(`SQLCommon.m4<span class="stringliteral">')</span> |
| <a name="l00013"></a>00013 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00014"></a>00014 <span class="comment">/**</span> |
| <a name="l00015"></a>00015 <span class="comment">@addtogroup grp_sketches</span> |
| <a name="l00016"></a>00016 <span class="comment"></span> |
| <a name="l00017"></a>00017 <span class="comment">@about</span> |
| <a name="l00018"></a>00018 <span class="comment"></span> |
| <a name="l00019"></a>00019 <span class="comment">Sketches (sometimes called "synopsis data structures") are small randomized</span> |
| <a name="l00020"></a>00020 <span class="comment">in-memory data structures that capture statistical properties of a large set</span> |
| <a name="l00021"></a>00021 <span class="comment">of values (e.g. a column of a table). Sketches can be formed in a single</span> |
| <a name="l00022"></a>00022 <span class="comment">pass of the data, and used to approximate a variety of descriptive statistics.</span> |
| <a name="l00023"></a>00023 <span class="comment"></span> |
| <a name="l00024"></a>00024 <span class="comment">We implement sketches as SQL User-Defined Aggregates (UDAs). Because they</span> |
| <a name="l00025"></a>00025 <span class="comment">are single-pass, small-space and parallelized, a single query can</span> |
| <a name="l00026"></a>00026 <span class="comment">use many sketches to gather summary statistics on many columns of a table efficiently.</span> |
| <a name="l00027"></a>00027 <span class="comment"></span> |
| <a name="l00028"></a>00028 <span class="comment">This module currently implements user-defined aggregates based on three main sketch methods:</span> |
| <a name="l00029"></a>00029 <span class="comment"> - <i>Flajolet-Martin (FM)</i> sketches for approximating <c>COUNT(DISTINCT)</c>.</span> |
| <a name="l00030"></a>00030 <span class="comment"> - <i>Count-Min (CM)</i> sketches, which can be used to approximate a number of descriptive statistics including</span> |
| <a name="l00031"></a>00031 <span class="comment"> - <c>COUNT(*)</c> of rows whose column value matches a given value in a set</span> |
| <a name="l00032"></a>00032 <span class="comment"> - <c>COUNT(*)</c> of rows whose column value falls in a range (*)</span> |
| <a name="l00033"></a>00033 <span class="comment"> - order statistics including <i>median</i> and <i>centiles</i> (*)</span> |
| <a name="l00034"></a>00034 <span class="comment"> - <i>histograms</i>: both <i>equi-width</i> and <i>equi-depth</i> (*)</span> |
| <a name="l00035"></a>00035 <span class="comment"> - <i>Most Frequent Value (MFV)</i> sketches, which output the most</span> |
| <a name="l00036"></a>00036 <span class="comment">frequently-occuring values in a column, along with their associated counts.</span> |
| <a name="l00037"></a>00037 <span class="comment"></span> |
| <a name="l00038"></a>00038 <span class="comment"> <i>Note:</i> Features marked with a star (*) only work for discrete types that</span> |
| <a name="l00039"></a>00039 <span class="comment"> can be cast to int8.</span> |
| <a name="l00040"></a>00040 <span class="comment"></span> |
| <a name="l00041"></a>00041 <span class="comment">@implementation</span> |
| <a name="l00042"></a>00042 <span class="comment">The sketch methods consists of a number of SQL UDAs (user defined aggregates)</span> |
| <a name="l00043"></a>00043 <span class="comment">and UDFs (user defined functions), to be used directly in SQL queries.</span> |
| <a name="l00044"></a>00044 <span class="comment">*/</span> |
| <a name="l00045"></a>00045 <span class="comment"></span> |
| <a name="l00046"></a>00046 <span class="comment">/**</span> |
| <a name="l00047"></a>00047 <span class="comment">@addtogroup grp_fmsketch</span> |
| <a name="l00048"></a>00048 <span class="comment"></span> |
| <a name="l00049"></a>00049 <span class="comment">@about</span> |
| <a name="l00050"></a>00050 <span class="comment">Flajolet-Martin's distinct count estimation</span> |
| <a name="l00051"></a>00051 <span class="comment">implemented as a user-defined aggregate.</span> |
| <a name="l00052"></a>00052 <span class="comment"></span> |
| <a name="l00053"></a>00053 <span class="comment">@usage</span> |
| <a name="l00054"></a>00054 <span class="comment">- Get the number of distinct values in a designated column.</span> |
| <a name="l00055"></a>00055 <span class="comment"> <pre>SELECT \ref fmsketch_dcount(<em>col_name</em>) FROM table_name;</pre></span> |
| <a name="l00056"></a>00056 <span class="comment"></span> |
| <a name="l00057"></a>00057 <span class="comment">@implementation</span> |
| <a name="l00058"></a>00058 <span class="comment">\ref fmsketch_dcount can be run on a column of any type.</span> |
| <a name="l00059"></a>00059 <span class="comment">It returns an approximation to the number of distinct values</span> |
| <a name="l00060"></a>00060 <span class="comment">(a la <c>COUNT(DISTINCT x)</c>), but faster and approximate.</span> |
| <a name="l00061"></a>00061 <span class="comment">Like any aggregate, it can be combined with a GROUP BY clause to do distinct</span> |
| <a name="l00062"></a>00062 <span class="comment">counts per group.</span> |
| <a name="l00063"></a>00063 <span class="comment"></span> |
| <a name="l00064"></a>00064 <span class="comment">@examp</span> |
| <a name="l00065"></a>00065 <span class="comment">-# Generate some data:</span> |
| <a name="l00066"></a>00066 <span class="comment">\verbatim</span> |
| <a name="l00067"></a>00067 <span class="comment">sql> CREATE TABLE data(class INT, a1 INT);</span> |
| <a name="l00068"></a>00068 <span class="comment">sql> INSERT INTO data SELECT 1,1 FROM generate_series(1,10000);</span> |
| <a name="l00069"></a>00069 <span class="comment">sql> INSERT INTO data SELECT 1,2 FROM generate_series(1,15000);</span> |
| <a name="l00070"></a>00070 <span class="comment">sql> INSERT INTO data SELECT 1,3 FROM generate_series(1,10000);</span> |
| <a name="l00071"></a>00071 <span class="comment">sql> INSERT INTO data SELECT 2,5 FROM generate_series(1,1000);</span> |
| <a name="l00072"></a>00072 <span class="comment">sql> INSERT INTO data SELECT 2,6 FROM generate_series(1,1000);</span> |
| <a name="l00073"></a>00073 <span class="comment">\endverbatim</span> |
| <a name="l00074"></a>00074 <span class="comment">-# Find distinct number of values for each class</span> |
| <a name="l00075"></a>00075 <span class="comment">\verbatim</span> |
| <a name="l00076"></a>00076 <span class="comment">sql> SELECT class,fmsketch_dcount(a1) FROM data GROUP BY data.class;</span> |
| <a name="l00077"></a>00077 <span class="comment">class | fmsketch_dcount</span> |
| <a name="l00078"></a>00078 <span class="comment">-------+-----------------</span> |
| <a name="l00079"></a>00079 <span class="comment"> 2 | 2</span> |
| <a name="l00080"></a>00080 <span class="comment"> 1 | 3</span> |
| <a name="l00081"></a>00081 <span class="comment">(2 rows)</span> |
| <a name="l00082"></a>00082 <span class="comment">\endverbatim</span> |
| <a name="l00083"></a>00083 <span class="comment"></span> |
| <a name="l00084"></a>00084 <span class="comment">@literature</span> |
| <a name="l00085"></a>00085 <span class="comment">[1] P. Flajolet and N.G. Martin. Probabilistic counting algorithms for data base applications, Journal of Computer and System Sciences 31(2), pp 182-209, 1985. http://algo.inria.fr/flajolet/Publications/FlMa85.pdf</span> |
| <a name="l00086"></a>00086 <span class="comment"></span> |
| <a name="l00087"></a>00087 <span class="comment">@sa File sketch.sql_in documenting the SQL function.</span> |
| <a name="l00088"></a>00088 <span class="comment"></span> |
| <a name="l00089"></a>00089 <span class="comment">*/</span> |
| <a name="l00090"></a>00090 <span class="comment"></span> |
| <a name="l00091"></a>00091 <span class="comment">/**</span> |
| <a name="l00092"></a>00092 <span class="comment">@addtogroup grp_countmin</span> |
| <a name="l00093"></a>00093 <span class="comment"></span> |
| <a name="l00094"></a>00094 <span class="comment">@about</span> |
| <a name="l00095"></a>00095 <span class="comment">This module implements Cormode-Muthukrishnan <i>CountMin</i> sketches</span> |
| <a name="l00096"></a>00096 <span class="comment">on integer values, implemented as a user-defined aggregate. It also provides</span> |
| <a name="l00097"></a>00097 <span class="comment">scalar functions over the sketches to produce approximate counts, order</span> |
| <a name="l00098"></a>00098 <span class="comment">statistics, and histograms.</span> |
| <a name="l00099"></a>00099 <span class="comment"></span> |
| <a name="l00100"></a>00100 <span class="comment">@usage</span> |
| <a name="l00101"></a>00101 <span class="comment"></span> |
| <a name="l00102"></a>00102 <span class="comment">- Get a sketch of a selected column specified by <em>col_name</em>.</span> |
| <a name="l00103"></a>00103 <span class="comment"> <pre>SELECT \ref cmsketch(<em>col_name</em>) FROM table_name;</pre></span> |
| <a name="l00104"></a>00104 <span class="comment"></span> |
| <a name="l00105"></a>00105 <span class="comment">- Get the number of rows where <em>col_name = p</em>, computed from the sketch</span> |
| <a name="l00106"></a>00106 <span class="comment"> obtained from <tt>cmsketch</tt>.</span> |
| <a name="l00107"></a>00107 <span class="comment"> <pre>SELECT \ref cmsketch_count(<em>cmsketch</em>,<em>p</em>) FROM table_name;</pre></span> |
| <a name="l00108"></a>00108 <span class="comment"></span> |
| <a name="l00109"></a>00109 <span class="comment">- Get the number of rows where <em>col_name</em> is between <em>m</em> and <em>n</em> inclusive.</span> |
| <a name="l00110"></a>00110 <span class="comment"> <pre>SELECT \ref cmsketch_rangecount(<em>cmsketch</em>,<em>m</em>,<em>n</em>) FROM table_name;</pre></span> |
| <a name="l00111"></a>00111 <span class="comment"></span> |
| <a name="l00112"></a>00112 <span class="comment">- Get the <em>k</em>th percentile of <em>col_name</em> where <em>count</em> specifies number of rows. <em>k</em> should be an integer between 1 to 99.</span> |
| <a name="l00113"></a>00113 <span class="comment"> <pre>SELECT \ref cmsketch_centile(<em>cmsketch</em>,<em>k</em>,<em>count</em>) FROM table_name;</pre></span> |
| <a name="l00114"></a>00114 <span class="comment"></span> |
| <a name="l00115"></a>00115 <span class="comment">- Get the median of <em>col_name</em> where <em>count</em> specifies number of rows. This is equivalent to <tt>\ref cmsketch_centile(<em>cmsketch</em>,50,<em>count</em>)</tt>.</span> |
| <a name="l00116"></a>00116 <span class="comment"> <pre>SELECT \ref cmsketch_median(<em>cmsketch</em>,<em>count</em>) FROM table_name;</pre></span> |
| <a name="l00117"></a>00117 <span class="comment"></span> |
| <a name="l00118"></a>00118 <span class="comment">- Get an n-bucket histogram for values between min and max for the column where each bucket has approximately the same width. The output is a text string containing triples {lo, hi, count} representing the buckets; counts are approximate.</span> |
| <a name="l00119"></a>00119 <span class="comment"> <pre>SELECT \ref cmsketch_width_histogram(<em>cmsketch</em>,<em>min</em>,<em>max</em>,<em>n</em>) FROM table_name;</pre></span> |
| <a name="l00120"></a>00120 <span class="comment"></span> |
| <a name="l00121"></a>00121 <span class="comment">- Get an n-bucket histogram for the column where each bucket has approximately the same count. The output is a text string containing triples {lo, hi, count} representing the buckets; counts are approximate. Note that an equi-depth histogram is equivalent to a spanning set of equi-spaced centiles.</span> |
| <a name="l00122"></a>00122 <span class="comment"> <pre>SELECT \ref cmsketch_depth_histogram(<em>cmsketch</em>,<em>n</em>) FROM table_name;</pre></span> |
| <a name="l00123"></a>00123 <span class="comment"></span> |
| <a name="l00124"></a>00124 <span class="comment">@examp</span> |
| <a name="l00125"></a>00125 <span class="comment"></span> |
| <a name="l00126"></a>00126 <span class="comment">-# Generate some data</span> |
| <a name="l00127"></a>00127 <span class="comment">\verbatim</span> |
| <a name="l00128"></a>00128 <span class="comment">sql> CREATE TABLE data(class INT, a1 INT);</span> |
| <a name="l00129"></a>00129 <span class="comment">sql> INSERT INTO data SELECT 1,1 FROM generate_series(1,10000);</span> |
| <a name="l00130"></a>00130 <span class="comment">sql> INSERT INTO data SELECT 1,2 FROM generate_series(1,15000);</span> |
| <a name="l00131"></a>00131 <span class="comment">sql> INSERT INTO data SELECT 1,3 FROM generate_series(1,10000);</span> |
| <a name="l00132"></a>00132 <span class="comment">sql> INSERT INTO data SELECT 2,5 FROM generate_series(1,1000);</span> |
| <a name="l00133"></a>00133 <span class="comment">sql> INSERT INTO data SELECT 2,6 FROM generate_series(1,1000);</span> |
| <a name="l00134"></a>00134 <span class="comment">\endverbatim</span> |
| <a name="l00135"></a>00135 <span class="comment">-# Count number of rows where a1 = 2 in each class</span> |
| <a name="l00136"></a>00136 <span class="comment">\verbatim</span> |
| <a name="l00137"></a>00137 <span class="comment">sql> SELECT class,cmsketch_count(cmsketch(a1),2) FROM data GROUP BY data.class;</span> |
| <a name="l00138"></a>00138 <span class="comment"> class | cmsketch_count</span> |
| <a name="l00139"></a>00139 <span class="comment">-------+----------------</span> |
| <a name="l00140"></a>00140 <span class="comment"> 2 | 0</span> |
| <a name="l00141"></a>00141 <span class="comment"> 1 | 15000</span> |
| <a name="l00142"></a>00142 <span class="comment">(2 rows)</span> |
| <a name="l00143"></a>00143 <span class="comment">\endverbatim</span> |
| <a name="l00144"></a>00144 <span class="comment">-# Count number of rows where a1 is between 3 and 6</span> |
| <a name="l00145"></a>00145 <span class="comment">\verbatim</span> |
| <a name="l00146"></a>00146 <span class="comment">sql> SELECT class,cmsketch_rangecount(cmsketch(a1),3,6) FROM data GROUP BY data.class;</span> |
| <a name="l00147"></a>00147 <span class="comment"> class | cmsketch_rangecount</span> |
| <a name="l00148"></a>00148 <span class="comment">-------+---------------------</span> |
| <a name="l00149"></a>00149 <span class="comment"> 2 | 2000</span> |
| <a name="l00150"></a>00150 <span class="comment"> 1 | 10000</span> |
| <a name="l00151"></a>00151 <span class="comment">(2 rows)</span> |
| <a name="l00152"></a>00152 <span class="comment">\endverbatim</span> |
| <a name="l00153"></a>00153 <span class="comment">-# Compute the 90th percentile of all of a1</span> |
| <a name="l00154"></a>00154 <span class="comment">\verbatim</span> |
| <a name="l00155"></a>00155 <span class="comment">sql> SELECT cmsketch_centile(cmsketch(a1),90,count(*)) FROM data;</span> |
| <a name="l00156"></a>00156 <span class="comment"> cmsketch_centile</span> |
| <a name="l00157"></a>00157 <span class="comment">------------------</span> |
| <a name="l00158"></a>00158 <span class="comment"> 3</span> |
| <a name="l00159"></a>00159 <span class="comment">(1 row)</span> |
| <a name="l00160"></a>00160 <span class="comment">\endverbatim</span> |
| <a name="l00161"></a>00161 <span class="comment">-# Produce an equi-width histogram with 2 bins between 0 and 10</span> |
| <a name="l00162"></a>00162 <span class="comment">\verbatim</span> |
| <a name="l00163"></a>00163 <span class="comment">sql> SELECT cmsketch_width_histogram(cmsketch(a1),0,10,2) FROM data;</span> |
| <a name="l00164"></a>00164 <span class="comment"> cmsketch_width_histogram</span> |
| <a name="l00165"></a>00165 <span class="comment">------------------------------------</span> |
| <a name="l00166"></a>00166 <span class="comment"> [[0L, 4L, 35000], [5L, 10L, 2000]]</span> |
| <a name="l00167"></a>00167 <span class="comment">(1 row)</span> |
| <a name="l00168"></a>00168 <span class="comment">\endverbatim</span> |
| <a name="l00169"></a>00169 <span class="comment">-# Produce an equi-depth histogram of a1 with 2 bins of approximately equal depth</span> |
| <a name="l00170"></a>00170 <span class="comment">\verbatim</span> |
| <a name="l00171"></a>00171 <span class="comment">sql> SELECT cmsketch_depth_histogram(cmsketch(a1),2) FROM data;</span> |
| <a name="l00172"></a>00172 <span class="comment"> cmsketch_depth_histogram</span> |
| <a name="l00173"></a>00173 <span class="comment">-----------------------------------------------------------------------</span> |
| <a name="l00174"></a>00174 <span class="comment"> [[-9223372036854775807L, 1, 10000], [2, 9223372036854775807L, 27000]]</span> |
| <a name="l00175"></a>00175 <span class="comment">(1 row)</span> |
| <a name="l00176"></a>00176 <span class="comment">\endverbatim</span> |
| <a name="l00177"></a>00177 <span class="comment"></span> |
| <a name="l00178"></a>00178 <span class="comment">@literature</span> |
| <a name="l00179"></a>00179 <span class="comment"></span> |
| <a name="l00180"></a>00180 <span class="comment">[1] G. Cormode and S. Muthukrishnan. An improved data stream summary: The count-min sketch and its applications. LATIN 2004, J. Algorithm 55(1): 58-75 (2005) . http://dimacs.rutgers.edu/~graham/pubs/html/CormodeMuthukrishnan04CMLatin.html</span> |
| <a name="l00181"></a>00181 <span class="comment"></span> |
| <a name="l00182"></a>00182 <span class="comment">[2] G. Cormode. Encyclopedia entry on 'Count-Min Sketch'. In L. Liu and M. T. Ozsu, editors, Encyclopedia of Database Systems, pages 511-516. Springer, 2009. http://dimacs.rutgers.edu/~graham/pubs/html/Cormode09b.html</span> |
| <a name="l00183"></a>00183 <span class="comment"></span> |
| <a name="l00184"></a>00184 <span class="comment">@sa File sketch.sql_in documenting the SQL functions.</span> |
| <a name="l00185"></a>00185 <span class="comment">\n\n Module grp_quantile for a different implementation of quantile function.</span> |
| <a name="l00186"></a>00186 <span class="comment"></span> |
| <a name="l00187"></a>00187 <span class="comment">*/</span> |
| <a name="l00188"></a>00188 <span class="comment"></span> |
| <a name="l00189"></a>00189 <span class="comment">/**</span> |
| <a name="l00190"></a>00190 <span class="comment">@addtogroup grp_mfvsketch</span> |
| <a name="l00191"></a>00191 <span class="comment"></span> |
| <a name="l00192"></a>00192 <span class="comment">@about</span> |
| <a name="l00193"></a>00193 <span class="comment">MFVSketch: Most Frequent Values variant of CountMin sketch, implemented</span> |
| <a name="l00194"></a>00194 <span class="comment">as a UDA.</span> |
| <a name="l00195"></a>00195 <span class="comment"></span> |
| <a name="l00196"></a>00196 <span class="comment">@usage</span> |
| <a name="l00197"></a>00197 <span class="comment">Produces an n-bucket histogram for a column where each bucket counts one of the</span> |
| <a name="l00198"></a>00198 <span class="comment">most frequent values in the column. The output is an array of doubles {value, count}</span> |
| <a name="l00199"></a>00199 <span class="comment">in descending order of frequency; counts are approximated via CountMin sketches.</span> |
| <a name="l00200"></a>00200 <span class="comment">Ties are handled arbitrarily.</span> |
| <a name="l00201"></a>00201 <span class="comment"><pre>SELECT \ref mfvsketch_top_histogram(<em>col_name</em>,n) FROM table_name;</pre></span> |
| <a name="l00202"></a>00202 <span class="comment"><pre>SELECT \ref mfvsketch_top_histogram(<em>col_name</em>,n) FROM table_name;</pre></span> |
| <a name="l00203"></a>00203 <span class="comment"></span> |
| <a name="l00204"></a>00204 <span class="comment">The MFV frequent-value UDA comes in two different versions:</span> |
| <a name="l00205"></a>00205 <span class="comment">- a faithful implementation that preserves the approximation guarantees</span> |
| <a name="l00206"></a>00206 <span class="comment">of Cormode/Muthukrishnan,</span> |
| <a name="l00207"></a>00207 <span class="comment">- and a "quick and dirty" version that can do parallel aggregation in Greenplum</span> |
| <a name="l00208"></a>00208 <span class="comment">at the expense of missing some of the most frequent values.</span> |
| <a name="l00209"></a>00209 <span class="comment"></span> |
| <a name="l00210"></a>00210 <span class="comment">In PostgreSQL the two UDAs are identical. In Greenplum, the quick version should</span> |
| <a name="l00211"></a>00211 <span class="comment">produce good results unless the number of values requested is very small,</span> |
| <a name="l00212"></a>00212 <span class="comment">or the distribution is very flat.</span> |
| <a name="l00213"></a>00213 <span class="comment"></span> |
| <a name="l00214"></a>00214 <span class="comment">@examp</span> |
| <a name="l00215"></a>00215 <span class="comment"></span> |
| <a name="l00216"></a>00216 <span class="comment">-# Generate some data</span> |
| <a name="l00217"></a>00217 <span class="comment">\verbatim</span> |
| <a name="l00218"></a>00218 <span class="comment">sql> CREATE TABLE data(class INT, a1 INT);</span> |
| <a name="l00219"></a>00219 <span class="comment">sql> INSERT INTO data SELECT 1,1 FROM generate_series(1,10000);</span> |
| <a name="l00220"></a>00220 <span class="comment">sql> INSERT INTO data SELECT 1,2 FROM generate_series(1,15000);</span> |
| <a name="l00221"></a>00221 <span class="comment">sql> INSERT INTO data SELECT 1,3 FROM generate_series(1,10000);</span> |
| <a name="l00222"></a>00222 <span class="comment">sql> INSERT INTO data SELECT 2,5 FROM generate_series(1,1000);</span> |
| <a name="l00223"></a>00223 <span class="comment">sql> INSERT INTO data SELECT 2,6 FROM generate_series(1,1000);</span> |
| <a name="l00224"></a>00224 <span class="comment">\endverbatim</span> |
| <a name="l00225"></a>00225 <span class="comment">-# Produce histogram of 5 bins and return the most frequent value and associated</span> |
| <a name="l00226"></a>00226 <span class="comment">count in each bin:</span> |
| <a name="l00227"></a>00227 <span class="comment">\verbatim</span> |
| <a name="l00228"></a>00228 <span class="comment">sql> SELECT mfvsketch_top_histogram(a1,5) FROM data;</span> |
| <a name="l00229"></a>00229 <span class="comment"></span> |
| <a name="l00230"></a>00230 <span class="comment"> mfvsketch_top_histogram</span> |
| <a name="l00231"></a>00231 <span class="comment">--------------------------------------------------------------</span> |
| <a name="l00232"></a>00232 <span class="comment">[0:4][0:1]={{2,15000},{1,10000},{3,10000},{5,1000},{6,1000}}</span> |
| <a name="l00233"></a>00233 <span class="comment">(1 row)</span> |
| <a name="l00234"></a>00234 <span class="comment">\endverbatim</span> |
| <a name="l00235"></a>00235 <span class="comment"></span> |
| <a name="l00236"></a>00236 <span class="comment">@literature</span> |
| <a name="l00237"></a>00237 <span class="comment">This method is not usually called an MFV sketch in the literature; it</span> |
| <a name="l00238"></a>00238 <span class="comment">is a natural extension of the CountMin sketch.</span> |
| <a name="l00239"></a>00239 <span class="comment"></span> |
| <a name="l00240"></a>00240 <span class="comment">@sa File sketch.sql_in documenting the SQL functions.</span> |
| <a name="l00241"></a>00241 <span class="comment">\n\n Module grp_countmin.</span> |
| <a name="l00242"></a>00242 <span class="comment">*/</span> |
| <a name="l00243"></a>00243 |
| <a name="l00244"></a>00244 -- FM Sketch Functions |
| <a name="l00245"></a>00245 DROP FUNCTION IF EXISTS MADLIB_SCHEMA.big_or(bitmap1 bytea, bitmap2 bytea) CASCADE; |
| <a name="l00246"></a>00246 CREATE FUNCTION MADLIB_SCHEMA.big_or(bitmap1 bytea, bitmap2 bytea) |
| <a name="l00247"></a>00247 RETURNS bytea |
| <a name="l00248"></a>00248 AS 'MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00249"></a>00249 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00250"></a>00250 <span class="stringliteral"></span> |
| <a name="l00251"></a>00251 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_trans(bitmaps bytea, input anyelement) CASCADE;</span> |
| <a name="l00252"></a>00252 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_trans(bitmaps bytea, input anyelement)</span> |
| <a name="l00253"></a>00253 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00254"></a>00254 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00255"></a>00255 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00256"></a>00256 <span class="stringliteral"></span> |
| <a name="l00257"></a>00257 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_count_distinct(bitmaps bytea) CASCADE;</span> |
| <a name="l00258"></a>00258 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_count_distinct(bitmaps bytea)</span> |
| <a name="l00259"></a>00259 <span class="stringliteral">RETURNS int8</span> |
| <a name="l00260"></a>00260 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00261"></a>00261 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00262"></a>00262 <span class="stringliteral"></span> |
| <a name="l00263"></a>00263 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_merge(bitmaps1 bytea, bitmaps2 bytea) CASCADE;</span> |
| <a name="l00264"></a>00264 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_merge(bitmaps1 bytea, bitmaps2 bytea)</span> |
| <a name="l00265"></a>00265 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00266"></a>00266 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00267"></a>00267 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00268"></a>00268 <span class="stringliteral"></span> |
| <a name="l00269"></a>00269 <span class="stringliteral">DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fmsketch_dcount(anyelement);</span> |
| <a name="l00270"></a>00270 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00271"></a>00271 <span class="comment">/**</span> |
| <a name="l00272"></a>00272 <span class="comment"> * @brief Flajolet-Martin's distinct count estimation</span> |
| <a name="l00273"></a>00273 <span class="comment"> * @param column name</span> |
| <a name="l00274"></a>00274 <span class="comment"> */</span> |
| <a name="l00275"></a>00275 CREATE AGGREGATE MADLIB_SCHEMA.fmsketch_dcount(/*+ column */ anyelement) |
| <a name="l00276"></a>00276 ( |
| <a name="l00277"></a>00277 sfunc = MADLIB_SCHEMA.__fmsketch_trans, |
| <a name="l00278"></a>00278 stype = bytea, |
| <a name="l00279"></a>00279 finalfunc = MADLIB_SCHEMA.__fmsketch_count_distinct, |
| <a name="l00280"></a>00280 m4_ifdef(`__GREENPLUM__',`prefunc = MADLIB_SCHEMA.__fmsketch_merge,<span class="stringliteral">')</span> |
| <a name="l00281"></a>00281 <span class="stringliteral"> initcond = '</span><span class="stringliteral">'</span> |
| <a name="l00282"></a>00282 <span class="stringliteral">);</span> |
| <a name="l00283"></a>00283 <span class="stringliteral"></span> |
| <a name="l00284"></a>00284 <span class="stringliteral"></span> |
| <a name="l00285"></a>00285 <span class="stringliteral">-- CM Sketch Functions</span> |
| <a name="l00286"></a>00286 <span class="stringliteral"></span> |
| <a name="l00287"></a>00287 <span class="stringliteral">-- We register __cmsketch_int8_trans for varying numbers of arguments to support</span> |
| <a name="l00288"></a>00288 <span class="stringliteral">-- a variety of agg function signatures. The first 2 args are used to</span> |
| <a name="l00289"></a>00289 <span class="stringliteral">-- aggregate; the remaining args are carried along unchanged inside the</span> |
| <a name="l00290"></a>00290 <span class="stringliteral">-- return structure for the use of the UDA finalizer.</span> |
| <a name="l00291"></a>00291 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8) CASCADE;</span> |
| <a name="l00292"></a>00292 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8)</span> |
| <a name="l00293"></a>00293 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00294"></a>00294 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00295"></a>00295 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00296"></a>00296 <span class="stringliteral"></span> |
| <a name="l00297"></a><a class="code" href="sketch_8sql__in.html#a5a7e077028d5e0441552a77db3586db1">00297</a> <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8) CASCADE;</span> |
| <a name="l00298"></a>00298 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8)</span> |
| <a name="l00299"></a>00299 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00300"></a>00300 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00301"></a>00301 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00302"></a>00302 <span class="stringliteral"></span> |
| <a name="l00303"></a>00303 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8, int8) CASCADE;</span> |
| <a name="l00304"></a>00304 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8, arg2 int8)</span> |
| <a name="l00305"></a>00305 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00306"></a>00306 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00307"></a>00307 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00308"></a>00308 <span class="stringliteral"></span> |
| <a name="l00309"></a>00309 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8, int8, int8) CASCADE;</span> |
| <a name="l00310"></a>00310 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8, arg2 int8, arg3 int8)</span> |
| <a name="l00311"></a>00311 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00312"></a>00312 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00313"></a>00313 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00314"></a>00314 <span class="stringliteral"></span> |
| <a name="l00315"></a>00315 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_final(bytea) CASCADE;</span> |
| <a name="l00316"></a>00316 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_final(counters bytea)</span> |
| <a name="l00317"></a>00317 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00318"></a>00318 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00319"></a>00319 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00320"></a>00320 <span class="stringliteral"></span> |
| <a name="l00321"></a>00321 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_base64_final(bytea) CASCADE;</span> |
| <a name="l00322"></a>00322 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_base64_final(sketch bytea)</span> |
| <a name="l00323"></a>00323 <span class="stringliteral">RETURNS text</span> |
| <a name="l00324"></a>00324 <span class="stringliteral">AS $$</span> |
| <a name="l00325"></a>00325 <span class="stringliteral">select encode(MADLIB_SCHEMA.__cmsketch_final($1), '</span>base64<span class="stringliteral">');</span> |
| <a name="l00326"></a>00326 <span class="stringliteral">$$ LANGUAGE SQL;</span> |
| <a name="l00327"></a>00327 <span class="stringliteral"></span> |
| <a name="l00328"></a>00328 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_merge(bytea, bytea) CASCADE;</span> |
| <a name="l00329"></a>00329 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_merge(bytea, bytea)</span> |
| <a name="l00330"></a>00330 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00331"></a>00331 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00332"></a>00332 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00333"></a>00333 <span class="stringliteral"></span> |
| <a name="l00334"></a>00334 <span class="stringliteral">DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.cmsketch(int8);</span><span class="comment"></span> |
| <a name="l00335"></a>00335 <span class="comment">/**</span> |
| <a name="l00336"></a>00336 <span class="comment"> *@brief <c>cmsketch</c> is a UDA that can be run on columns of type int8,</span> |
| <a name="l00337"></a>00337 <span class="comment"> * or any column that can be cast to an int8. It produces a base64 string</span> |
| <a name="l00338"></a>00338 <span class="comment"> * representing a CountMin sketch: a large array of counters that is intended</span> |
| <a name="l00339"></a>00339 <span class="comment"> * to be passed into a UDF like <c>cmsketch_width_histogram</c> described below.</span> |
| <a name="l00340"></a>00340 <span class="comment"> */</span> |
| <a name="l00341"></a>00341 CREATE AGGREGATE MADLIB_SCHEMA.cmsketch(/*+ column */ INT8) |
| <a name="l00342"></a>00342 ( |
| <a name="l00343"></a>00343 sfunc = MADLIB_SCHEMA.__cmsketch_int8_trans, |
| <a name="l00344"></a>00344 stype = bytea, |
| <a name="l00345"></a>00345 finalfunc = MADLIB_SCHEMA.__cmsketch_base64_final, |
| <a name="l00346"></a>00346 m4_ifdef(`__GREENPLUM__', `prefunc = MADLIB_SCHEMA.__cmsketch_merge,<span class="stringliteral">')</span> |
| <a name="l00347"></a>00347 <span class="stringliteral"> initcond = '</span><span class="stringliteral">'</span> |
| <a name="l00348"></a>00348 <span class="stringliteral">);</span> |
| <a name="l00349"></a>00349 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00350"></a>00350 <span class="comment">/**</span> |
| <a name="l00351"></a>00351 <span class="comment"> @brief <c>cmsketch_count</c> is a scalar UDF to compute the approximate</span> |
| <a name="l00352"></a>00352 <span class="comment"> number of occurences of a value in a column summarized by a cmsketch. Takes</span> |
| <a name="l00353"></a>00353 <span class="comment"> the results of the <c>cmsketch</c> aggregate as its first argument, and the</span> |
| <a name="l00354"></a>00354 <span class="comment"> desired value as the second.</span> |
| <a name="l00355"></a>00355 <span class="comment"> */</span> |
| <a name="l00356"></a>00356 DROP FUNCTION IF EXISTS MADLIB_SCHEMA.cmsketch_count(text, int8) CASCADE; |
| <a name="l00357"></a>00357 CREATE FUNCTION MADLIB_SCHEMA.cmsketch_count(sketches64 text, val int8) |
| <a name="l00358"></a>00358 RETURNS int8 |
| <a name="l00359"></a>00359 AS $$ |
| <a name="l00360"></a>00360 PythonFunctionBodyOnly(`sketch', `countmin<span class="stringliteral">')</span> |
| <a name="l00361"></a>00361 <span class="stringliteral"> # schema_madlib comes from PythonFunctionBodyOnly</span> |
| <a name="l00362"></a>00362 <span class="stringliteral"> return countmin.count(sketches64, val)</span> |
| <a name="l00363"></a><a class="code" href="sketch_8sql__in.html#a323b932332f2db302469f14dc5fae369">00363</a> <span class="stringliteral">$$ LANGUAGE plpythonu;</span> |
| <a name="l00364"></a>00364 <span class="stringliteral"></span> |
| <a name="l00365"></a>00365 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00366"></a>00366 <span class="comment">/**</span> |
| <a name="l00367"></a>00367 <span class="comment"> @brief <c>cmsketch_rangecount</c> is a scalar UDF to approximate the number</span> |
| <a name="l00368"></a>00368 <span class="comment"> of occurrences of values in the range <c>[lo,hi]</c> inclusive, given a</span> |
| <a name="l00369"></a>00369 <span class="comment"> cmsketch of a column. Takes the results</span> |
| <a name="l00370"></a>00370 <span class="comment"> of the <c>cmsketch</c> aggregate as its first argument, and the desired range</span> |
| <a name="l00371"></a>00371 <span class="comment"> boundaries as the second and third.</span> |
| <a name="l00372"></a>00372 <span class="comment"> */</span> |
| <a name="l00373"></a>00373 DROP FUNCTION IF EXISTS MADLIB_SCHEMA.cmsketch_rangecount(text, int8, int8) CASCADE; |
| <a name="l00374"></a>00374 CREATE FUNCTION MADLIB_SCHEMA.cmsketch_rangecount(sketches64 text, bot int8, top int8) |
| <a name="l00375"></a>00375 RETURNS int8 |
| <a name="l00376"></a>00376 AS $$ |
| <a name="l00377"></a>00377 PythonFunctionBodyOnly(`sketch', `countmin<span class="stringliteral">')</span> |
| <a name="l00378"></a>00378 <span class="stringliteral"> # schema_madlib comes from PythonFunctionBodyOnly</span> |
| <a name="l00379"></a><a class="code" href="sketch_8sql__in.html#a3498d2c778d1289154f61d34e84c609e">00379</a> <span class="stringliteral"> return countmin.rangecount(sketches64, bot, top)</span> |
| <a name="l00380"></a>00380 <span class="stringliteral">$$ LANGUAGE plpythonu;</span> |
| <a name="l00381"></a>00381 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00382"></a>00382 <span class="comment">/**</span> |
| <a name="l00383"></a>00383 <span class="comment"> @brief <c>cmsketch_centile</c> is a scalar UDF to compute a centile value</span> |
| <a name="l00384"></a>00384 <span class="comment"> from a cmsketch. Takes the results of the <c>cmsketch</c> aggregate as its</span> |
| <a name="l00385"></a>00385 <span class="comment"> first argument, a number between 1 and 99 as the desired centile in the</span> |
| <a name="l00386"></a>00386 <span class="comment"> second, and the count of the column as the third. Produces a value from the</span> |
| <a name="l00387"></a>00387 <span class="comment"> sketched column that is approximately at the centile's position in sorted</span> |
| <a name="l00388"></a>00388 <span class="comment"> order.</span> |
| <a name="l00389"></a>00389 <span class="comment"> */</span> |
| <a name="l00390"></a>00390 DROP FUNCTION IF EXISTS MADLIB_SCHEMA.cmsketch_centile(text, int8, int8) CASCADE; |
| <a name="l00391"></a>00391 CREATE FUNCTION MADLIB_SCHEMA.cmsketch_centile(sketches64 text, centile int8, cnt int8) |
| <a name="l00392"></a>00392 RETURNS int8 |
| <a name="l00393"></a>00393 AS $$ |
| <a name="l00394"></a>00394 PythonFunctionBodyOnly(`sketch', `countmin<span class="stringliteral">')</span> |
| <a name="l00395"></a>00395 <span class="stringliteral"> # schema_madlib comes from PythonFunctionBodyOnly</span> |
| <a name="l00396"></a><a class="code" href="sketch_8sql__in.html#aeff9e36cfb3338c4e405d4ac77d3968c">00396</a> <span class="stringliteral"> return countmin.centile(sketches64, centile, cnt)</span> |
| <a name="l00397"></a>00397 <span class="stringliteral">$$ LANGUAGE plpythonu;</span> |
| <a name="l00398"></a>00398 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00399"></a>00399 <span class="comment">/**</span> |
| <a name="l00400"></a>00400 <span class="comment"> @brief <c>cmsketch_median</c> is a scalar UDF to compute a median value</span> |
| <a name="l00401"></a>00401 <span class="comment"> from a cmsketch. Takes the results of the <c>cmsketch</c> aggregate as its</span> |
| <a name="l00402"></a>00402 <span class="comment"> first argument, and the count as the second. Produces a value from the</span> |
| <a name="l00403"></a>00403 <span class="comment"> sketched column that is approximately at the halfway position in sorted</span> |
| <a name="l00404"></a>00404 <span class="comment"> order.</span> |
| <a name="l00405"></a>00405 <span class="comment"> */</span> |
| <a name="l00406"></a>00406 DROP FUNCTION IF EXISTS MADLIB_SCHEMA.cmsketch_median(text, int8) CASCADE; |
| <a name="l00407"></a>00407 CREATE FUNCTION MADLIB_SCHEMA.cmsketch_median(sketches64 text, cnt int8) |
| <a name="l00408"></a>00408 RETURNS int8 |
| <a name="l00409"></a>00409 AS $$ |
| <a name="l00410"></a>00410 PythonFunctionBodyOnly(`sketch', `countmin<span class="stringliteral">')</span> |
| <a name="l00411"></a>00411 <span class="stringliteral"> # schema_madlib comes from PythonFunctionBodyOnly</span> |
| <a name="l00412"></a>00412 <span class="stringliteral"> return countmin.centile(sketches64, 50, cnt)</span> |
| <a name="l00413"></a><a class="code" href="sketch_8sql__in.html#a2f2ab2fe3244515f5f73d49690e73b39">00413</a> <span class="stringliteral">$$ LANGUAGE plpythonu;</span> |
| <a name="l00414"></a>00414 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00415"></a>00415 <span class="comment">/**</span> |
| <a name="l00416"></a>00416 <span class="comment"> \brief <c>cmsketch_width_histogram</c> is a scalar UDF that takes three aggregates of a column -- cmsketch, min and max-- as well as a number of buckets, and produces an n-bucket histogram for the column where each bucket has approximately the same width. The output is a text string containing triples {lo, hi, count} representing the buckets; counts are approximate.</span> |
| <a name="l00417"></a>00417 <span class="comment"> */</span> |
| <a name="l00418"></a>00418 DROP FUNCTION IF EXISTS MADLIB_SCHEMA.cmsketch_width_histogram(text, /*+ min */int8, /*+ max */int8, /*+ nbuckets */ int4) CASCADE; |
| <a name="l00419"></a>00419 CREATE FUNCTION MADLIB_SCHEMA.cmsketch_width_histogram(sketches64 text, themin int8, themax int8, nbuckets int4) |
| <a name="l00420"></a>00420 RETURNS text |
| <a name="l00421"></a>00421 AS $$ |
| <a name="l00422"></a>00422 PythonFunctionBodyOnly(`sketch', `countmin<span class="stringliteral">')</span> |
| <a name="l00423"></a>00423 <span class="stringliteral"> # schema_madlib comes from PythonFunctionBodyOnly</span> |
| <a name="l00424"></a>00424 <span class="stringliteral"> return countmin.width_histogram( sketches64, themin, themax, nbuckets)</span> |
| <a name="l00425"></a>00425 <span class="stringliteral">$$ LANGUAGE plpythonu;</span> |
| <a name="l00426"></a>00426 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00427"></a>00427 <span class="comment">/** @brief <c>cmsketch_depth_histogram</c> is a UDA that takes a cmsketch and a number of buckets n, and produces an n-bucket histogram for the column where each bucket has approximately the same count. The output is a text string containing triples {lo, hi, count} representing the buckets; counts are approximate. Note that an equi-depth histogram is equivalent to a spanning set of equi-spaced centiles.</span> |
| <a name="l00428"></a>00428 <span class="comment">*/</span> |
| <a name="l00429"></a><a class="code" href="sketch_8sql__in.html#a0ef6428fa7ba4b7f7b1f633e6f8003ea">00429</a> DROP FUNCTION IF EXISTS MADLIB_SCHEMA.cmsketch_depth_histogram(text, /*+ nbuckets */ int4) CASCADE; |
| <a name="l00430"></a>00430 CREATE FUNCTION MADLIB_SCHEMA.cmsketch_depth_histogram(sketches64 text, nbuckets int4) |
| <a name="l00431"></a>00431 RETURNS text |
| <a name="l00432"></a>00432 AS $$ |
| <a name="l00433"></a>00433 PythonFunctionBodyOnly(`sketch', `countmin<span class="stringliteral">')</span> |
| <a name="l00434"></a>00434 <span class="stringliteral"> # schema_madlib comes from PythonFunctionBodyOnly</span> |
| <a name="l00435"></a>00435 <span class="stringliteral"> return countmin.depth_histogram(sketches64, nbuckets)</span> |
| <a name="l00436"></a>00436 <span class="stringliteral">$$ LANGUAGE plpythonu;</span> |
| <a name="l00437"></a>00437 <span class="stringliteral"></span> |
| <a name="l00438"></a>00438 <span class="stringliteral">-- MFV Sketch functions</span> |
| <a name="l00439"></a>00439 <span class="stringliteral"></span> |
| <a name="l00440"></a>00440 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_trans(bytea, anyelement, int4) CASCADE;</span> |
| <a name="l00441"></a><a class="code" href="sketch_8sql__in.html#a8482f62849adf40a2c7df78c23ea33a4">00441</a> <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_trans(bytea, anyelement, int4)</span> |
| <a name="l00442"></a>00442 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00443"></a>00443 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00444"></a>00444 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00445"></a>00445 <span class="stringliteral"></span> |
| <a name="l00446"></a>00446 <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_final(bytea) CASCADE;</span> |
| <a name="l00447"></a>00447 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_final(bytea)</span> |
| <a name="l00448"></a>00448 <span class="stringliteral">RETURNS text[][]</span> |
| <a name="l00449"></a>00449 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00450"></a>00450 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00451"></a>00451 <span class="stringliteral"></span> |
| <a name="l00452"></a><a class="code" href="sketch_8sql__in.html#a9e6d30f20b724b96249cc4a0f67a279e">00452</a> <span class="stringliteral">DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_merge(bytea, bytea) CASCADE;</span> |
| <a name="l00453"></a>00453 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_merge(bytea, bytea)</span> |
| <a name="l00454"></a>00454 <span class="stringliteral">RETURNS bytea</span> |
| <a name="l00455"></a>00455 <span class="stringliteral">AS '</span>MODULE_PATHNAME<span class="stringliteral">'</span> |
| <a name="l00456"></a>00456 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00457"></a>00457 <span class="stringliteral"></span> |
| <a name="l00458"></a>00458 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__sketch_rightmost_one(bytea, integer, integer)</span> |
| <a name="l00459"></a>00459 <span class="stringliteral">RETURNS integer AS '</span>MODULE_PATHNAME<span class="stringliteral">', '</span>sketch_rightmost_one<span class="stringliteral">' LANGUAGE C STRICT;</span> |
| <a name="l00460"></a>00460 <span class="stringliteral"></span> |
| <a name="l00461"></a>00461 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__sketch_leftmost_zero(bytea, integer, integer)</span> |
| <a name="l00462"></a>00462 <span class="stringliteral">RETURNS integer AS '</span>MODULE_PATHNAME<span class="stringliteral">', '</span>sketch_leftmost_zero<span class="stringliteral">' LANGUAGE C STRICT;</span> |
| <a name="l00463"></a>00463 <span class="stringliteral"></span> |
| <a name="l00464"></a>00464 <span class="stringliteral">CREATE FUNCTION MADLIB_SCHEMA.__sketch_array_set_bit_in_place(bytea, integer, integer, integer, integer)</span> |
| <a name="l00465"></a>00465 <span class="stringliteral">RETURNS bytea AS '</span>MODULE_PATHNAME<span class="stringliteral">', '</span>sketch_array_set_bit_in_place<span class="stringliteral">' LANGUAGE C STRICT;</span> |
| <a name="l00466"></a>00466 <span class="stringliteral"></span> |
| <a name="l00467"></a>00467 <span class="stringliteral">DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.mfvsketch_top_histogram( anyelement, int4);</span><span class="comment"></span> |
| <a name="l00468"></a>00468 <span class="comment">/**</span> |
| <a name="l00469"></a>00469 <span class="comment"> * @brief Produces an n-bucket histogram for a column where each bucket counts</span> |
| <a name="l00470"></a>00470 <span class="comment"> * one of the most frequent values in the column. The output is an array of</span> |
| <a name="l00471"></a>00471 <span class="comment"> * doubles {value, count} in descending order of frequency; counts are</span> |
| <a name="l00472"></a>00472 <span class="comment"> * approximated via CountMin sketches. Ties are handled arbitrarily.</span> |
| <a name="l00473"></a>00473 <span class="comment">*/</span> |
| <a name="l00474"></a>00474 CREATE AGGREGATE MADLIB_SCHEMA.mfvsketch_top_histogram(/*+ column */ anyelement, /*+ number_of_buckets */ int4) |
| <a name="l00475"></a>00475 ( |
| <a name="l00476"></a>00476 sfunc = MADLIB_SCHEMA.__mfvsketch_trans, |
| <a name="l00477"></a>00477 stype = bytea, |
| <a name="l00478"></a>00478 finalfunc = MADLIB_SCHEMA.__mfvsketch_final, |
| <a name="l00479"></a>00479 initcond = '<span class="stringliteral">'</span> |
| <a name="l00480"></a>00480 <span class="stringliteral">);</span> |
| <a name="l00481"></a>00481 <span class="stringliteral"></span> |
| <a name="l00482"></a>00482 <span class="stringliteral">DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.mfvsketch_quick_histogram(anyelement, int4);</span><span class="comment"></span> |
| <a name="l00483"></a>00483 <span class="comment">/**</span> |
| <a name="l00484"></a>00484 <span class="comment"> * @brief On Postgres it works the same way as \ref mfvsketch_top_histogram but,</span> |
| <a name="l00485"></a>00485 <span class="comment"> * in Greenplum it does parallel aggregation to provide a "quick and dirty" answer.</span> |
| <a name="l00486"></a>00486 <span class="comment">*/</span> |
| <a name="l00487"></a>00487 CREATE AGGREGATE MADLIB_SCHEMA.mfvsketch_quick_histogram(/*+ column */ anyelement, /*+ number_of_buckets */ int4) |
| <a name="l00488"></a>00488 ( |
| <a name="l00489"></a>00489 sfunc = MADLIB_SCHEMA.__mfvsketch_trans, |
| <a name="l00490"></a>00490 stype = bytea, |
| <a name="l00491"></a>00491 finalfunc = MADLIB_SCHEMA.__mfvsketch_final, |
| <a name="l00492"></a>00492 m4_ifdef(`__GREENPLUM__', `prefunc = MADLIB_SCHEMA.__mfvsketch_merge,<span class="stringliteral">')</span> |
| <a name="l00493"></a>00493 <span class="stringliteral"> initcond = '</span><span class="stringliteral">'</span> |
| <a name="l00494"></a>00494 <span class="stringliteral">);</span> |
| </pre></div></div> |
| </div> |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="sketch_8sql__in.html">sketch.sql_in</a> </li> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Functions</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| |
| <li class="footer">Generated on Tue Apr 2 2013 14:57:03 for MADlib by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.5.1 </li> |
| </ul> |
| </div> |
| |
| |
| </body> |
| </html> |
