| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <title>MADlib: lda.sql_in Source File</title> |
| |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <script src="../mathjax/MathJax.js"> |
| MathJax.Hub.Config({ |
| extensions: ["tex2jax.js", "TeX/AMSmath.js", "TeX/AMSsymbols.js"], |
| jax: ["input/TeX","output/HTML-CSS"], |
| }); |
| </script> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div! --> |
| |
| |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| |
| |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">MADlib |
|  <span id="projectnumber">0.7</span> <span style="font-size:10pt; font-style:italic"><a href="../latest/./lda_8sql__in_source.html"> A newer version is available</a></span> |
| </div> |
| <div id="projectbrief">User Documentation</div> |
| </td> |
| |
| |
| |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| |
| <!-- Generated by Doxygen 1.7.5.1 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="modules.html"><span>Modules</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| initNavTree('lda_8sql__in.html',''); |
| </script> |
| <div id="doc-content"> |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">lda.sql_in</div> </div> |
| </div> |
| <div class="contents"> |
| <a href="lda_8sql__in.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* ----------------------------------------------------------------------- */</span><span class="comment">/** </span> |
| <a name="l00002"></a>00002 <span class="comment"> *</span> |
| <a name="l00003"></a>00003 <span class="comment"> * @file lda.sql_in</span> |
| <a name="l00004"></a>00004 <span class="comment"> *</span> |
| <a name="l00005"></a>00005 <span class="comment"> * @brief SQL functions for Latent Dirichlet Allocation</span> |
| <a name="l00006"></a>00006 <span class="comment"> * @date Dec 2012</span> |
| <a name="l00007"></a>00007 <span class="comment"> *</span> |
| <a name="l00008"></a>00008 <span class="comment"> * @sa For an introduction to Latent Dirichlet Allocation models, see the </span> |
| <a name="l00009"></a>00009 <span class="comment"> module description \ref grp_lda.</span> |
| <a name="l00010"></a>00010 <span class="comment"> *</span> |
| <a name="l00011"></a>00011 <span class="comment"> */</span><span class="comment">/* ------------------------------------------------------------------------*/</span> |
| <a name="l00012"></a>00012 |
| <a name="l00013"></a>00013 m4_include(`SQLCommon.m4<span class="stringliteral">')</span> |
| <a name="l00014"></a>00014 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00015"></a>00015 <span class="comment">/**</span> |
| <a name="l00016"></a>00016 <span class="comment"></span> |
| <a name="l00017"></a>00017 <span class="comment">@addtogroup grp_lda</span> |
| <a name="l00018"></a>00018 <span class="comment"></span> |
| <a name="l00019"></a>00019 <span class="comment">@about</span> |
| <a name="l00020"></a>00020 <span class="comment"></span> |
| <a name="l00021"></a>00021 <span class="comment">Latent Dirichlet Allocation (LDA) is an interesting generative probabilistic</span> |
| <a name="l00022"></a>00022 <span class="comment">model for natural texts and has received a lot of attention in recent years. </span> |
| <a name="l00023"></a>00023 <span class="comment">The model is quite versatile, having found uses in problems like automated </span> |
| <a name="l00024"></a>00024 <span class="comment">topic discovery, collaborative filtering, and document classification.</span> |
| <a name="l00025"></a>00025 <span class="comment"></span> |
| <a name="l00026"></a>00026 <span class="comment">The LDA model posits that each document is associated with a mixture of various</span> |
| <a name="l00027"></a>00027 <span class="comment">topics (e.g. a document is related to Topic 1 with probability 0.7, and Topic 2</span> |
| <a name="l00028"></a>00028 <span class="comment">with probability 0.3), and that each word in the document is attributable to</span> |
| <a name="l00029"></a>00029 <span class="comment">one of the document's topics. There is a (symmetric) Dirichlet prior with</span> |
| <a name="l00030"></a>00030 <span class="comment">parameter \f$ \alpha \f$ on each document's topic mixture. In addition, there</span> |
| <a name="l00031"></a>00031 <span class="comment">is another (symmetric) Dirichlet prior with parameter \f$ \beta \f$ on the</span> |
| <a name="l00032"></a>00032 <span class="comment">distribution of words for each topic.</span> |
| <a name="l00033"></a>00033 <span class="comment"></span> |
| <a name="l00034"></a>00034 <span class="comment">The following generative process then defines a distribution over a corpus of</span> |
| <a name="l00035"></a>00035 <span class="comment">documents. </span> |
| <a name="l00036"></a>00036 <span class="comment"></span> |
| <a name="l00037"></a>00037 <span class="comment">- Sample for each topic \f$ i \f$, a per-topic word</span> |
| <a name="l00038"></a>00038 <span class="comment">distribution \f$ \phi_i \f$ from the Dirichlet(\f$\beta\f$) prior. </span> |
| <a name="l00039"></a>00039 <span class="comment"></span> |
| <a name="l00040"></a>00040 <span class="comment">- For each document:</span> |
| <a name="l00041"></a>00041 <span class="comment"> - Sample a document length N from a suitable distribution, say, Poisson.</span> |
| <a name="l00042"></a>00042 <span class="comment"> - Sample a topic mixture \f$ \theta \f$ for the document from the</span> |
| <a name="l00043"></a>00043 <span class="comment">Dirichlet(\f$\alpha\f$) distribution. </span> |
| <a name="l00044"></a>00044 <span class="comment"> - For each of the N words:</span> |
| <a name="l00045"></a>00045 <span class="comment"> - Sample a topic \f$ z_n \f$ from the multinomial topic distribution \f$</span> |
| <a name="l00046"></a>00046 <span class="comment"> \theta \f$. </span> |
| <a name="l00047"></a>00047 <span class="comment"> - Sample a word \f$ w_n \f$ from the multinomial word distribution \f$</span> |
| <a name="l00048"></a>00048 <span class="comment"> \phi_{z_n} \f$ associated with topic \f$ z_n \f$.</span> |
| <a name="l00049"></a>00049 <span class="comment"></span> |
| <a name="l00050"></a>00050 <span class="comment">In practice, only the words in each document are observable. The topic mixture</span> |
| <a name="l00051"></a>00051 <span class="comment">of each document and the topic for each word in each document are latent</span> |
| <a name="l00052"></a>00052 <span class="comment">unobservable variables that need to be inferred from the observables, and this</span> |
| <a name="l00053"></a>00053 <span class="comment">is the problem people refer to when they talk about the inference problem for</span> |
| <a name="l00054"></a>00054 <span class="comment">LDA. Exact inference is intractable, but several approximate inference</span> |
| <a name="l00055"></a>00055 <span class="comment">algorithms for LDA have been developed. The simple and effective Gibbs sampling</span> |
| <a name="l00056"></a>00056 <span class="comment">algorithm described in Griffiths and Steyvers [2] appears to be the current</span> |
| <a name="l00057"></a>00057 <span class="comment">algorithm of choice.</span> |
| <a name="l00058"></a>00058 <span class="comment"></span> |
| <a name="l00059"></a>00059 <span class="comment">This implementation provides a parallel and scalable in-database solution for</span> |
| <a name="l00060"></a>00060 <span class="comment">LDA based on Gibbs sampling. Different with the implementations based on MPI or</span> |
| <a name="l00061"></a>00061 <span class="comment">Hadoop Map/Reduce, this implementation builds upon the shared-nothing MPP</span> |
| <a name="l00062"></a>00062 <span class="comment">databases and enables high-performance in-database analytics.</span> |
| <a name="l00063"></a>00063 <span class="comment"></span> |
| <a name="l00064"></a>00064 <span class="comment">@input</span> |
| <a name="l00065"></a>00065 <span class="comment">The \b corpus/dataset to be analyzed is expected to be of the following form:</span> |
| <a name="l00066"></a>00066 <span class="comment"><pre>{TABLE|VIEW} <em>data_table</em> (</span> |
| <a name="l00067"></a>00067 <span class="comment"> <em>docid</em> INTEGER,</span> |
| <a name="l00068"></a>00068 <span class="comment"> <em>wordid</em> INTEGER,</span> |
| <a name="l00069"></a>00069 <span class="comment"> <em>count</em> INTEGER</span> |
| <a name="l00070"></a>00070 <span class="comment">)</pre></span> |
| <a name="l00071"></a>00071 <span class="comment">where \c docid refers to the document ID, \c wordid is the word ID (the index</span> |
| <a name="l00072"></a>00072 <span class="comment">of a word in the vocabulary), and \c count is the number of occurence of the</span> |
| <a name="l00073"></a>00073 <span class="comment">word in the document. </span> |
| <a name="l00074"></a>00074 <span class="comment"></span> |
| <a name="l00075"></a>00075 <span class="comment">The \b vocabulary/dictionary that indexes all the words found in the corpus is</span> |
| <a name="l00076"></a>00076 <span class="comment">of the following form:</span> |
| <a name="l00077"></a>00077 <span class="comment"><pre>{TABLE|VIEW} <em>vocab_table</em> (</span> |
| <a name="l00078"></a>00078 <span class="comment"> <em>wordid</em> INTEGER,</span> |
| <a name="l00079"></a>00079 <span class="comment"> <em>word</em> TEXT,</span> |
| <a name="l00080"></a>00080 <span class="comment">)</pre></span> |
| <a name="l00081"></a>00081 <span class="comment">where \c wordid refers the word ID (the index of a word in the vocabulary) and</span> |
| <a name="l00082"></a>00082 <span class="comment">\c word is the actual word.</span> |
| <a name="l00083"></a>00083 <span class="comment"></span> |
| <a name="l00084"></a>00084 <span class="comment">@usage</span> |
| <a name="l00085"></a>00085 <span class="comment">- The training (i.e. topic inference) can be done with the following function:</span> |
| <a name="l00086"></a>00086 <span class="comment"> <pre></span> |
| <a name="l00087"></a>00087 <span class="comment"> SELECT \ref lda_train(</span> |
| <a name="l00088"></a>00088 <span class="comment"> <em>'data_table'</em>,</span> |
| <a name="l00089"></a>00089 <span class="comment"> <em>'model_table'</em>,</span> |
| <a name="l00090"></a>00090 <span class="comment"> <em>'output_data_table'</em>, </span> |
| <a name="l00091"></a>00091 <span class="comment"> <em>voc_size</em>, </span> |
| <a name="l00092"></a>00092 <span class="comment"> <em>topic_num</em>,</span> |
| <a name="l00093"></a>00093 <span class="comment"> <em>iter_num</em>, </span> |
| <a name="l00094"></a>00094 <span class="comment"> <em>alpha</em>, </span> |
| <a name="l00095"></a>00095 <span class="comment"> <em>beta</em>)</span> |
| <a name="l00096"></a>00096 <span class="comment"> </pre></span> |
| <a name="l00097"></a>00097 <span class="comment"> </span> |
| <a name="l00098"></a>00098 <span class="comment"> This function stores the resulting model in <tt><em>model_table</em></tt>.</span> |
| <a name="l00099"></a>00099 <span class="comment"> The table has only 1 row and is in the following form:</span> |
| <a name="l00100"></a>00100 <span class="comment"> <pre>{TABLE} <em>model_table</em> (</span> |
| <a name="l00101"></a>00101 <span class="comment"> <em>voc_size</em> INTEGER,</span> |
| <a name="l00102"></a>00102 <span class="comment"> <em>topic_num</em> INTEGER,</span> |
| <a name="l00103"></a>00103 <span class="comment"> <em>alpha</em> FLOAT,</span> |
| <a name="l00104"></a>00104 <span class="comment"> <em>beta</em> FLOAT,</span> |
| <a name="l00105"></a>00105 <span class="comment"> <em>model</em> INTEGER[][])</span> |
| <a name="l00106"></a>00106 <span class="comment"> </pre></span> |
| <a name="l00107"></a>00107 <span class="comment"></span> |
| <a name="l00108"></a>00108 <span class="comment"> This function also stores the topic counts and the topic assignments in</span> |
| <a name="l00109"></a>00109 <span class="comment"> each document in <tt><em>output_data_table</em></tt>. The table is in the</span> |
| <a name="l00110"></a>00110 <span class="comment"> following form:</span> |
| <a name="l00111"></a>00111 <span class="comment"> <pre>{TABLE} <em>output_data_table</em> (</span> |
| <a name="l00112"></a>00112 <span class="comment"> <em>docid</em> INTEGER,</span> |
| <a name="l00113"></a>00113 <span class="comment"> <em>wordcount</em> INTEGER,</span> |
| <a name="l00114"></a>00114 <span class="comment"> <em>words</em> INTEGER[],</span> |
| <a name="l00115"></a>00115 <span class="comment"> <em>counts</em> INTEGER[],</span> |
| <a name="l00116"></a>00116 <span class="comment"> <em>topic_count</em> INTEGER[],</span> |
| <a name="l00117"></a>00117 <span class="comment"> <em>topic_assignment</em> INTEGER[])</span> |
| <a name="l00118"></a>00118 <span class="comment"> </pre></span> |
| <a name="l00119"></a>00119 <span class="comment"></span> |
| <a name="l00120"></a>00120 <span class="comment">- The prediction (i.e. labelling of test documents using a learned LDA model)</span> |
| <a name="l00121"></a>00121 <span class="comment"> can be done with the following function: </span> |
| <a name="l00122"></a>00122 <span class="comment"> <pre></span> |
| <a name="l00123"></a>00123 <span class="comment"> SELECT \ref lda_predict(</span> |
| <a name="l00124"></a>00124 <span class="comment"> <em>'data_table'</em>,</span> |
| <a name="l00125"></a>00125 <span class="comment"> <em>'model_table'</em>,</span> |
| <a name="l00126"></a>00126 <span class="comment"> <em>'output_table'</em>);</span> |
| <a name="l00127"></a>00127 <span class="comment"> </pre></span> |
| <a name="l00128"></a>00128 <span class="comment"> </span> |
| <a name="l00129"></a>00129 <span class="comment"> This function stores the prediction results in</span> |
| <a name="l00130"></a>00130 <span class="comment"> <tt><em>output_table</em></tt>. Each row in the table stores the topic</span> |
| <a name="l00131"></a>00131 <span class="comment"> distribution and the topic assignments for a docuemnt in the dataset. And</span> |
| <a name="l00132"></a>00132 <span class="comment"> the table is in the following form: </span> |
| <a name="l00133"></a>00133 <span class="comment"> <pre>{TABLE} <em>output_table</em> (</span> |
| <a name="l00134"></a>00134 <span class="comment"> <em>docid</em> INTEGER,</span> |
| <a name="l00135"></a>00135 <span class="comment"> <em>wordcount</em> INTEGER,</span> |
| <a name="l00136"></a>00136 <span class="comment"> <em>words</em> INTEGER,</span> |
| <a name="l00137"></a>00137 <span class="comment"> <em>counts</em> INTEGER,</span> |
| <a name="l00138"></a>00138 <span class="comment"> <em>topic_count</em> INTEGER[],</span> |
| <a name="l00139"></a>00139 <span class="comment"> <em>topic_assignment</em> INTEGER[])</span> |
| <a name="l00140"></a>00140 <span class="comment"> </pre></span> |
| <a name="l00141"></a>00141 <span class="comment"></span> |
| <a name="l00142"></a>00142 <span class="comment">- This module also provides a function for computing the perplexity:</span> |
| <a name="l00143"></a>00143 <span class="comment"> <pre></span> |
| <a name="l00144"></a>00144 <span class="comment"> SELECT \ref lda_get_perplexity(</span> |
| <a name="l00145"></a>00145 <span class="comment"> <em>'model_table'</em>,</span> |
| <a name="l00146"></a>00146 <span class="comment"> <em>'output_data_table'</em>);</span> |
| <a name="l00147"></a>00147 <span class="comment"> </pre></span> |
| <a name="l00148"></a>00148 <span class="comment"></span> |
| <a name="l00149"></a>00149 <span class="comment">@implementation</span> |
| <a name="l00150"></a>00150 <span class="comment">The input format for this module is very common in many machine learning</span> |
| <a name="l00151"></a>00151 <span class="comment">packages written in various lanugages, which allows users to generate</span> |
| <a name="l00152"></a>00152 <span class="comment">datasets using any existing document preprocessing tools or import existing</span> |
| <a name="l00153"></a>00153 <span class="comment">dataset very conveniently. Internally, the input data will be validated and then</span> |
| <a name="l00154"></a>00154 <span class="comment">converted to the following format for efficiency: </span> |
| <a name="l00155"></a>00155 <span class="comment"> <pre>{TABLE} <em>__internal_data_table__</em> (</span> |
| <a name="l00156"></a>00156 <span class="comment"> <em>docid</em> INTEGER,</span> |
| <a name="l00157"></a>00157 <span class="comment"> <em>wordcount</em> INTEGER,</span> |
| <a name="l00158"></a>00158 <span class="comment"> <em>words</em> INTEGER[],</span> |
| <a name="l00159"></a>00159 <span class="comment"> <em>counts</em> INTEGER[])</span> |
| <a name="l00160"></a>00160 <span class="comment"> </pre></span> |
| <a name="l00161"></a>00161 <span class="comment">where \c docid is the document ID, \c wordcount is the count of words in the</span> |
| <a name="l00162"></a>00162 <span class="comment">document, \c words is the list of unique words in the document, and \c counts</span> |
| <a name="l00163"></a>00163 <span class="comment">is the list of number of occurence of each unique word in the document. The</span> |
| <a name="l00164"></a>00164 <span class="comment">convertion can be done with the help of aggregation functions very easily.</span> |
| <a name="l00165"></a>00165 <span class="comment"></span> |
| <a name="l00166"></a>00166 <span class="comment">@examp</span> |
| <a name="l00167"></a>00167 <span class="comment"></span> |
| <a name="l00168"></a>00168 <span class="comment">We now give a usage example.</span> |
| <a name="l00169"></a>00169 <span class="comment"></span> |
| <a name="l00170"></a>00170 <span class="comment">- As a first step, we need to prepare a dataset and vocabulary in the appropriate structure.</span> |
| <a name="l00171"></a>00171 <span class="comment"> \code</span> |
| <a name="l00172"></a>00172 <span class="comment"> CREATE TABLE my_vocab(wordid INT4, word TEXT)</span> |
| <a name="l00173"></a>00173 <span class="comment"> m4_ifdef(`__GREENPLUM__',`DISTRIBUTED BY (wordid)');</span> |
| <a name="l00174"></a>00174 <span class="comment"> </span> |
| <a name="l00175"></a>00175 <span class="comment"> INSERT INTO my_vocab VALUES</span> |
| <a name="l00176"></a>00176 <span class="comment"> (0, 'code'), (1, 'data'), (2, 'graph'), (3, 'image'), (4, 'input'), (5,</span> |
| <a name="l00177"></a>00177 <span class="comment"> 'layer'), (6, 'learner'), (7, 'loss'), (8, 'model'), (9, 'network'), (10,</span> |
| <a name="l00178"></a>00178 <span class="comment"> 'neuron'), (11, 'object'), (12, 'output'), (13, 'rate'), (14, 'set'), (15,</span> |
| <a name="l00179"></a>00179 <span class="comment"> 'signal'), (16, 'sparse'), (17, 'spatial'), (18, 'system'), (19, 'training');</span> |
| <a name="l00180"></a>00180 <span class="comment"> </span> |
| <a name="l00181"></a>00181 <span class="comment"> CREATE TABLE my_training </span> |
| <a name="l00182"></a>00182 <span class="comment"> (</span> |
| <a name="l00183"></a>00183 <span class="comment"> docid INT4, </span> |
| <a name="l00184"></a>00184 <span class="comment"> wordid INT4, </span> |
| <a name="l00185"></a>00185 <span class="comment"> count INT4</span> |
| <a name="l00186"></a>00186 <span class="comment"> )</span> |
| <a name="l00187"></a>00187 <span class="comment"> m4_ifdef(`__GREENPLUM__',`DISTRIBUTED BY (docid)');</span> |
| <a name="l00188"></a>00188 <span class="comment"> </span> |
| <a name="l00189"></a>00189 <span class="comment"> INSERT INTO my_training VALUES</span> |
| <a name="l00190"></a>00190 <span class="comment"> (0, 0, 2),(0, 3, 2),(0, 5, 1),(0, 7, 1),(0, 8, 1),(0, 9, 1),(0, 11, 1),(0, 13,</span> |
| <a name="l00191"></a>00191 <span class="comment"> 1), (1, 0, 1),(1, 3, 1),(1, 4, 1),(1, 5, 1),(1, 6, 1),(1, 7, 1),(1, 10, 1),(1,</span> |
| <a name="l00192"></a>00192 <span class="comment"> 14, 1),(1, 17, 1),(1, 18, 1), (2, 4, 2),(2, 5, 1),(2, 6, 2),(2, 12, 1),(2, 13,</span> |
| <a name="l00193"></a>00193 <span class="comment"> 1),(2, 15, 1),(2, 18, 2), (3, 0, 1),(3, 1, 2),(3, 12, 3),(3, 16, 1),(3, 17,</span> |
| <a name="l00194"></a>00194 <span class="comment"> 2),(3, 19, 1), (4, 1, 1),(4, 2, 1),(4, 3, 1),(4, 5, 1),(4, 6, 1),(4, 10, 1),(4,</span> |
| <a name="l00195"></a>00195 <span class="comment"> 11, 1),(4, 14, 1),(4, 18, 1),(4, 19, 1), (5, 0, 1),(5, 2, 1),(5, 5, 1),(5, 7,</span> |
| <a name="l00196"></a>00196 <span class="comment"> 1),(5, 10, 1),(5, 12, 1),(5, 16, 1),(5, 18, 1),(5, 19, 2), (6, 1, 1),(6, 3,</span> |
| <a name="l00197"></a>00197 <span class="comment"> 1),(6, 12, 2),(6, 13, 1),(6, 14, 2),(6, 15, 1),(6, 16, 1),(6, 17, 1), (7, 0,</span> |
| <a name="l00198"></a>00198 <span class="comment"> 1),(7, 2, 1),(7, 4, 1),(7, 5, 1),(7, 7, 2),(7, 8, 1),(7, 11, 1),(7, 14, 1),(7,</span> |
| <a name="l00199"></a>00199 <span class="comment"> 16, 1), (8, 2, 1),(8, 4, 4),(8, 6, 2),(8, 11, 1),(8, 15, 1),(8, 18, 1),</span> |
| <a name="l00200"></a>00200 <span class="comment"> (9, 0, 1),(9, 1, 1),(9, 4, 1),(9, 9, 2),(9, 12, 2),(9, 15, 1),(9, 18, 1),(9,</span> |
| <a name="l00201"></a>00201 <span class="comment"> 19, 1);</span> |
| <a name="l00202"></a>00202 <span class="comment"> </span> |
| <a name="l00203"></a>00203 <span class="comment"> </span> |
| <a name="l00204"></a>00204 <span class="comment"> CREATE TABLE my_testing </span> |
| <a name="l00205"></a>00205 <span class="comment"> (</span> |
| <a name="l00206"></a>00206 <span class="comment"> docid INT4, </span> |
| <a name="l00207"></a>00207 <span class="comment"> wordid INT4, </span> |
| <a name="l00208"></a>00208 <span class="comment"> count INT4</span> |
| <a name="l00209"></a>00209 <span class="comment"> )</span> |
| <a name="l00210"></a>00210 <span class="comment"> m4_ifdef(`__GREENPLUM__',`DISTRIBUTED BY (docid)');</span> |
| <a name="l00211"></a>00211 <span class="comment"> </span> |
| <a name="l00212"></a>00212 <span class="comment"> INSERT INTO my_testing VALUES</span> |
| <a name="l00213"></a>00213 <span class="comment"> (0, 0, 2),(0, 8, 1),(0, 9, 1),(0, 10, 1),(0, 12, 1),(0, 15, 2),(0, 18, 1),(0,</span> |
| <a name="l00214"></a>00214 <span class="comment"> 19, 1), (1, 0, 1),(1, 2, 1),(1, 5, 1),(1, 7, 1),(1, 12, 2),(1, 13, 1),(1, 16,</span> |
| <a name="l00215"></a>00215 <span class="comment"> 1),(1, 17, 1),(1, 18, 1), (2, 0, 1),(2, 1, 1),(2, 2, 1),(2, 3, 1),(2, 4, 1),(2,</span> |
| <a name="l00216"></a>00216 <span class="comment"> 5, 1),(2, 6, 1),(2, 12, 1),(2, 14, 1),(2, 18, 1), (3, 2, 2),(3, 6, 2),(3, 7,</span> |
| <a name="l00217"></a>00217 <span class="comment"> 1),(3, 9, 1),(3, 11, 2),(3, 14, 1),(3, 15, 1), (4, 1, 1),(4, 2, 2),(4, 3,</span> |
| <a name="l00218"></a>00218 <span class="comment"> 1),(4, 5, 2),(4, 6, 1),(4, 11, 1),(4, 18, 2);</span> |
| <a name="l00219"></a>00219 <span class="comment"> \endcode</span> |
| <a name="l00220"></a>00220 <span class="comment"></span> |
| <a name="l00221"></a>00221 <span class="comment">- To perform training, we call the lda_train() function with the</span> |
| <a name="l00222"></a>00222 <span class="comment">appropriate parameters. Here is an example.</span> |
| <a name="l00223"></a>00223 <span class="comment"> \code</span> |
| <a name="l00224"></a>00224 <span class="comment"> SELECT MADLib.lda_train(</span> |
| <a name="l00225"></a>00225 <span class="comment"> 'my_training', 'my_model', 'my_outdata', 20, 5, 10, 5, 0.01);</span> |
| <a name="l00226"></a>00226 <span class="comment"> \endcode</span> |
| <a name="l00227"></a>00227 <span class="comment"></span> |
| <a name="l00228"></a>00228 <span class="comment"> After a successful run of the lda_train() function, two tables will be</span> |
| <a name="l00229"></a>00229 <span class="comment"> generated, one for storing the learned models, and another for storing the</span> |
| <a name="l00230"></a>00230 <span class="comment"> output data table.</span> |
| <a name="l00231"></a>00231 <span class="comment"></span> |
| <a name="l00232"></a>00232 <span class="comment"> To get the detailed information about the learned model, we can run the</span> |
| <a name="l00233"></a>00233 <span class="comment"> following commands:</span> |
| <a name="l00234"></a>00234 <span class="comment"></span> |
| <a name="l00235"></a>00235 <span class="comment"> - The topic description by top-k words</span> |
| <a name="l00236"></a>00236 <span class="comment"> \code</span> |
| <a name="l00237"></a>00237 <span class="comment"> SELECT * FROM MADLib.lda_get_topic_desc(</span> |
| <a name="l00238"></a>00238 <span class="comment"> 'my_model', 'my_vocab', 'my_topic_desc', 15);</span> |
| <a name="l00239"></a>00239 <span class="comment"> \endcode</span> |
| <a name="l00240"></a>00240 <span class="comment"></span> |
| <a name="l00241"></a>00241 <span class="comment"> - The per-topic word counts</span> |
| <a name="l00242"></a>00242 <span class="comment"> \code</span> |
| <a name="l00243"></a>00243 <span class="comment"> SELECT MADLib.lda_get_topic_word_count(</span> |
| <a name="l00244"></a>00244 <span class="comment"> 'my_model', 'my_topic_word_count');</span> |
| <a name="l00245"></a>00245 <span class="comment"> \endcode</span> |
| <a name="l00246"></a>00246 <span class="comment"></span> |
| <a name="l00247"></a>00247 <span class="comment"> - The per-word topic counts</span> |
| <a name="l00248"></a>00248 <span class="comment"> \code</span> |
| <a name="l00249"></a>00249 <span class="comment"> SELECT MADLib.lda_get_word_topic_count(</span> |
| <a name="l00250"></a>00250 <span class="comment"> 'my_model', 'my_word_topic_count');</span> |
| <a name="l00251"></a>00251 <span class="comment"> \endcode</span> |
| <a name="l00252"></a>00252 <span class="comment"></span> |
| <a name="l00253"></a>00253 <span class="comment"> To get the topic counts and the topic assignments for each doucment, we</span> |
| <a name="l00254"></a>00254 <span class="comment"> can run the following commands:</span> |
| <a name="l00255"></a>00255 <span class="comment"></span> |
| <a name="l00256"></a>00256 <span class="comment"> - The per-document topic counts:</span> |
| <a name="l00257"></a>00257 <span class="comment"> \code</span> |
| <a name="l00258"></a>00258 <span class="comment"> SELECT </span> |
| <a name="l00259"></a>00259 <span class="comment"> docid, topic_count </span> |
| <a name="l00260"></a>00260 <span class="comment"> FROM my_outdata;</span> |
| <a name="l00261"></a>00261 <span class="comment"> \endcode</span> |
| <a name="l00262"></a>00262 <span class="comment"></span> |
| <a name="l00263"></a>00263 <span class="comment"> - The per-document topic assignments:</span> |
| <a name="l00264"></a>00264 <span class="comment"> \code</span> |
| <a name="l00265"></a>00265 <span class="comment"> SELECT </span> |
| <a name="l00266"></a>00266 <span class="comment"> docid, words, counts, topic_assignment </span> |
| <a name="l00267"></a>00267 <span class="comment"> FROM my_outdata;</span> |
| <a name="l00268"></a>00268 <span class="comment"> \endcode</span> |
| <a name="l00269"></a>00269 <span class="comment"> By scanning \c words, \c counts, and \c topic_assignment together, we can</span> |
| <a name="l00270"></a>00270 <span class="comment"> get the topic assignment for each word in a document.</span> |
| <a name="l00271"></a>00271 <span class="comment"></span> |
| <a name="l00272"></a>00272 <span class="comment">- To use a learned LDA model for prediction (i.e. to label new documents), we can use the following command: </span> |
| <a name="l00273"></a>00273 <span class="comment"> \code</span> |
| <a name="l00274"></a>00274 <span class="comment"> SELECT MADLib.lda_predict(</span> |
| <a name="l00275"></a>00275 <span class="comment"> 'my_testing', 'my_model', 'my_pred');</span> |
| <a name="l00276"></a>00276 <span class="comment"> \endcode</span> |
| <a name="l00277"></a>00277 <span class="comment"></span> |
| <a name="l00278"></a>00278 <span class="comment"> After a successful run of the lda_predict() function, the prediction</span> |
| <a name="l00279"></a>00279 <span class="comment"> results will be generated and stored in <em>my_pred</em>. This table has</span> |
| <a name="l00280"></a>00280 <span class="comment"> the same schema as the <em>my_outdata</em> generated by the lda_train()</span> |
| <a name="l00281"></a>00281 <span class="comment"> function.</span> |
| <a name="l00282"></a>00282 <span class="comment"> </span> |
| <a name="l00283"></a>00283 <span class="comment"> To get te the topic counts and the topic assignments for each doucment, we</span> |
| <a name="l00284"></a>00284 <span class="comment"> can run the following commands:</span> |
| <a name="l00285"></a>00285 <span class="comment"></span> |
| <a name="l00286"></a>00286 <span class="comment"> - The per-document topic counts:</span> |
| <a name="l00287"></a>00287 <span class="comment"> \code</span> |
| <a name="l00288"></a>00288 <span class="comment"> SELECT </span> |
| <a name="l00289"></a>00289 <span class="comment"> docid, topic_count </span> |
| <a name="l00290"></a>00290 <span class="comment"> FROM my_pred;</span> |
| <a name="l00291"></a>00291 <span class="comment"> \endcode</span> |
| <a name="l00292"></a>00292 <span class="comment"></span> |
| <a name="l00293"></a>00293 <span class="comment"> - The per-document topic assignments:</span> |
| <a name="l00294"></a>00294 <span class="comment"> \code</span> |
| <a name="l00295"></a>00295 <span class="comment"> SELECT </span> |
| <a name="l00296"></a>00296 <span class="comment"> docid, words, counts, topic_assignment </span> |
| <a name="l00297"></a>00297 <span class="comment"> FROM my_pred;</span> |
| <a name="l00298"></a>00298 <span class="comment"> \endcode</span> |
| <a name="l00299"></a>00299 <span class="comment"> By scanning \c words, \c counts, and \c topic_assignment together, we can</span> |
| <a name="l00300"></a>00300 <span class="comment"> get the topic assignment for each word in a document.</span> |
| <a name="l00301"></a>00301 <span class="comment"></span> |
| <a name="l00302"></a>00302 <span class="comment">- To compute the perplexity, we can use the following command:</span> |
| <a name="l00303"></a>00303 <span class="comment"> \code</span> |
| <a name="l00304"></a>00304 <span class="comment"> SELECT MADLib.lda_get_perplexity(</span> |
| <a name="l00305"></a>00305 <span class="comment"> 'my_model', 'my_pred');</span> |
| <a name="l00306"></a>00306 <span class="comment"> \endcode</span> |
| <a name="l00307"></a>00307 <span class="comment"></span> |
| <a name="l00308"></a>00308 <span class="comment">@literature</span> |
| <a name="l00309"></a>00309 <span class="comment"></span> |
| <a name="l00310"></a>00310 <span class="comment">[1] D.M. Blei, A.Y. Ng, M.I. Jordan, <em>Latent Dirichlet Allocation</em>,</span> |
| <a name="l00311"></a>00311 <span class="comment"> Journal of Machine Learning Research, vol. 3, pp. 993-1022, 2003.</span> |
| <a name="l00312"></a>00312 <span class="comment"></span> |
| <a name="l00313"></a>00313 <span class="comment">[2] T. Griffiths and M. Steyvers, <em>Finding scientific topics</em>, PNAS,</span> |
| <a name="l00314"></a>00314 <span class="comment"> vol. 101, pp. 5228-5235, 2004.</span> |
| <a name="l00315"></a>00315 <span class="comment"></span> |
| <a name="l00316"></a>00316 <span class="comment">[3] Y. Wang, H. Bai, M. Stanton, W-Y. Chen, and E.Y. Chang, <em>lda: Parallel</span> |
| <a name="l00317"></a>00317 <span class="comment"> Dirichlet Allocation for Large-scale Applications</em>, AAIM, 2009.</span> |
| <a name="l00318"></a>00318 <span class="comment"></span> |
| <a name="l00319"></a>00319 <span class="comment">[4] http://en.wikipedia.org/wiki/Latent_Dirichlet_allocation</span> |
| <a name="l00320"></a>00320 <span class="comment"></span> |
| <a name="l00321"></a>00321 <span class="comment">[5] J. Chang, Collapsed Gibbs sampling methods for topic models, R manual,</span> |
| <a name="l00322"></a>00322 <span class="comment"> 2010.</span> |
| <a name="l00323"></a>00323 <span class="comment"></span> |
| <a name="l00324"></a>00324 <span class="comment">@sa File lda.sql_in documenting the SQL functions.</span> |
| <a name="l00325"></a>00325 <span class="comment">*/</span> |
| <a name="l00326"></a>00326 |
| <a name="l00327"></a>00327 -- UDT for summarizing a UDF call |
| <a name="l00328"></a>00328 DROP TYPE IF EXISTS MADLIB_SCHEMA.lda_result; |
| <a name="l00329"></a>00329 CREATE TYPE MADLIB_SCHEMA.lda_result AS |
| <a name="l00330"></a>00330 ( |
| <a name="l00331"></a>00331 output_table TEXT, |
| <a name="l00332"></a>00332 description TEXT |
| <a name="l00333"></a>00333 ); |
| <a name="l00334"></a>00334 <span class="comment"></span> |
| <a name="l00335"></a>00335 <span class="comment">/**</span> |
| <a name="l00336"></a>00336 <span class="comment"> * @brief This UDF provides an entry for the lda training process.</span> |
| <a name="l00337"></a>00337 <span class="comment"> * @param data_table Table storing the training dataset, each row is in</span> |
| <a name="l00338"></a>00338 <span class="comment"> * the form of <docid, wordid, count> where docid,</span> |
| <a name="l00339"></a>00339 <span class="comment"> * wordid, and count are all non-negative integers.</span> |
| <a name="l00340"></a>00340 <span class="comment"> * @param voc_size Size of the vocabulary (Note that the wordid should</span> |
| <a name="l00341"></a>00341 <span class="comment"> * be continous integers starting from 0 to voc_size -</span> |
| <a name="l00342"></a>00342 <span class="comment"> * 1. A data validation rountine will be called to</span> |
| <a name="l00343"></a>00343 <span class="comment"> * validate the dataset.)</span> |
| <a name="l00344"></a>00344 <span class="comment"> * @param topic_num Number of topics (e.g. 100)</span> |
| <a name="l00345"></a>00345 <span class="comment"> * @param iter_num Number of iterations (e.g. 60)</span> |
| <a name="l00346"></a>00346 <span class="comment"> * @param alpha Dirichlet parameter for the per-doc topic multinomial</span> |
| <a name="l00347"></a>00347 <span class="comment"> * (e.g. 50/topic_num)</span> |
| <a name="l00348"></a>00348 <span class="comment"> * @param beta Dirichlet parameter for the per-topic word multinomial</span> |
| <a name="l00349"></a>00349 <span class="comment"> * (e.g. 0.01)</span> |
| <a name="l00350"></a>00350 <span class="comment"> * @param model_table Table storing the learned models (voc_size, topic_num, </span> |
| <a name="l00351"></a>00351 <span class="comment"> * alpha, beta, per-word topic counts, and</span> |
| <a name="l00352"></a>00352 <span class="comment"> * corpus-level topic counts)</span> |
| <a name="l00353"></a>00353 <span class="comment"> * @param output_data_table Table storing the output data table in the form of</span> |
| <a name="l00354"></a>00354 <span class="comment"> * <docid, wordcount, words, counts, topic_count,</span> |
| <a name="l00355"></a>00355 <span class="comment"> * topic_assignment></span> |
| <a name="l00356"></a>00356 <span class="comment"> **/</span> |
| <a name="l00357"></a>00357 CREATE OR REPLACE FUNCTION |
| <a name="l00358"></a>00358 MADLIB_SCHEMA.lda_train |
| <a name="l00359"></a>00359 ( |
| <a name="l00360"></a>00360 data_table TEXT, |
| <a name="l00361"></a>00361 model_table TEXT, |
| <a name="l00362"></a>00362 output_data_table TEXT, |
| <a name="l00363"></a>00363 voc_size INT4, |
| <a name="l00364"></a>00364 topic_num INT4, |
| <a name="l00365"></a>00365 iter_num INT4, |
| <a name="l00366"></a>00366 alpha FLOAT8, |
| <a name="l00367"></a>00367 beta FLOAT8 |
| <a name="l00368"></a>00368 ) |
| <a name="l00369"></a>00369 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00370"></a>00370 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00371"></a>00371 <span class="stringliteral"> lda.lda_train(</span> |
| <a name="l00372"></a>00372 <span class="stringliteral"> schema_madlib, data_table, model_table, output_data_table, voc_size,</span> |
| <a name="l00373"></a>00373 <span class="stringliteral"> topic_num, iter_num, alpha, beta</span> |
| <a name="l00374"></a>00374 <span class="stringliteral"> )</span> |
| <a name="l00375"></a>00375 <span class="stringliteral"> return [[model_table, '</span>model table<span class="stringliteral">'], </span> |
| <a name="l00376"></a>00376 <span class="stringliteral"> [output_data_table, '</span>output data table<span class="stringliteral">']]</span> |
| <a name="l00377"></a>00377 <span class="stringliteral">$$ LANGUAGE PLPYTHONU STRICT;</span> |
| <a name="l00378"></a>00378 <span class="stringliteral"></span> |
| <a name="l00379"></a>00379 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00380"></a>00380 <span class="comment">/**</span> |
| <a name="l00381"></a>00381 <span class="comment"> * @brief This UDF provides an entry for the lda predicton process.</span> |
| <a name="l00382"></a>00382 <span class="comment"> * @param data_table Table storing the testing dataset, each row is in the </span> |
| <a name="l00383"></a>00383 <span class="comment"> * form of <docid, wordid, count></span> |
| <a name="l00384"></a>00384 <span class="comment"> * where docid, wordid, and count are all non-negative </span> |
| <a name="l00385"></a>00385 <span class="comment"> * integers.</span> |
| <a name="l00386"></a>00386 <span class="comment"> * @param model_table Table storing the learned models</span> |
| <a name="l00387"></a>00387 <span class="comment"> * @param output_table Table storing per-document topic counts and topic </span> |
| <a name="l00388"></a>00388 <span class="comment"> * assignments</span> |
| <a name="l00389"></a>00389 <span class="comment"> * @note default iter_num = 20</span> |
| <a name="l00390"></a>00390 <span class="comment"> **/</span> |
| <a name="l00391"></a>00391 CREATE OR REPLACE FUNCTION |
| <a name="l00392"></a>00392 MADLIB_SCHEMA.lda_predict |
| <a name="l00393"></a>00393 ( |
| <a name="l00394"></a>00394 data_table TEXT, |
| <a name="l00395"></a>00395 model_table TEXT, |
| <a name="l00396"></a>00396 output_table TEXT |
| <a name="l00397"></a>00397 ) |
| <a name="l00398"></a>00398 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00399"></a>00399 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00400"></a>00400 <span class="stringliteral"> lda.lda_predict(schema_madlib, data_table, model_table, output_table)</span> |
| <a name="l00401"></a>00401 <span class="stringliteral"> return [[</span> |
| <a name="l00402"></a>00402 <span class="stringliteral"> output_table, </span> |
| <a name="l00403"></a>00403 <span class="stringliteral"> '</span>per-doc topic distribution and per-word topic assignments<span class="stringliteral">']]</span> |
| <a name="l00404"></a>00404 <span class="stringliteral">$$ LANGUAGE PLPYTHONU STRICT;</span> |
| <a name="l00405"></a>00405 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00406"></a>00406 <span class="comment">/**</span> |
| <a name="l00407"></a>00407 <span class="comment"> * @brief A overloaded version which allows users to specify iter_num.</span> |
| <a name="l00408"></a>00408 <span class="comment"> **/</span> |
| <a name="l00409"></a>00409 CREATE OR REPLACE FUNCTION |
| <a name="l00410"></a>00410 MADLIB_SCHEMA.lda_predict |
| <a name="l00411"></a>00411 ( |
| <a name="l00412"></a>00412 data_table TEXT, |
| <a name="l00413"></a>00413 model_table TEXT, |
| <a name="l00414"></a>00414 output_table TEXT, |
| <a name="l00415"></a>00415 iter_num INT4 |
| <a name="l00416"></a>00416 ) |
| <a name="l00417"></a>00417 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00418"></a>00418 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00419"></a>00419 <span class="stringliteral"> lda.lda_predict(</span> |
| <a name="l00420"></a>00420 <span class="stringliteral"> schema_madlib, data_table, model_table, output_table, iter_num)</span> |
| <a name="l00421"></a>00421 <span class="stringliteral"> return [[</span> |
| <a name="l00422"></a>00422 <span class="stringliteral"> output_table, </span> |
| <a name="l00423"></a>00423 <span class="stringliteral"> '</span>per-doc topic distribution and per-word topic assignments<span class="stringliteral">']]</span> |
| <a name="l00424"></a>00424 <span class="stringliteral">$$ LANGUAGE PLPYTHONU STRICT;</span><span class="comment"></span> |
| <a name="l00425"></a>00425 <span class="comment">/**</span> |
| <a name="l00426"></a>00426 <span class="comment"> * @brief This UDF computes the per-topic word counts.</span> |
| <a name="l00427"></a>00427 <span class="comment"> * @param model_table The model table generated by the training process</span> |
| <a name="l00428"></a>00428 <span class="comment"> * @param output_table The output table storing the per-topic word counts</span> |
| <a name="l00429"></a>00429 <span class="comment"> **/</span> |
| <a name="l00430"></a>00430 CREATE OR REPLACE FUNCTION |
| <a name="l00431"></a>00431 MADLIB_SCHEMA.lda_get_topic_word_count |
| <a name="l00432"></a>00432 ( |
| <a name="l00433"></a>00433 model_table TEXT, |
| <a name="l00434"></a>00434 output_table TEXT |
| <a name="l00435"></a>00435 ) |
| <a name="l00436"></a>00436 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00437"></a>00437 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00438"></a>00438 <span class="stringliteral"> lda.get_topic_word_count(schema_madlib, model_table, output_table)</span> |
| <a name="l00439"></a>00439 <span class="stringliteral"> return [[output_table, '</span>per-topic word counts<span class="stringliteral">']]</span> |
| <a name="l00440"></a>00440 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00441"></a>00441 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00442"></a>00442 <span class="comment">/**</span> |
| <a name="l00443"></a>00443 <span class="comment"> * @brief This UDF computes the per-word topic counts.</span> |
| <a name="l00444"></a>00444 <span class="comment"> * @param model_table The model table generated by the training process</span> |
| <a name="l00445"></a>00445 <span class="comment"> * @param dist_table The output table storing the per-word topic counts</span> |
| <a name="l00446"></a>00446 <span class="comment"> **/</span> |
| <a name="l00447"></a>00447 CREATE OR REPLACE FUNCTION |
| <a name="l00448"></a>00448 MADLIB_SCHEMA.lda_get_word_topic_count |
| <a name="l00449"></a>00449 ( |
| <a name="l00450"></a>00450 model_table TEXT, |
| <a name="l00451"></a>00451 output_table TEXT |
| <a name="l00452"></a>00452 ) |
| <a name="l00453"></a>00453 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00454"></a>00454 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00455"></a>00455 <span class="stringliteral"> lda.get_word_topic_count(schema_madlib, model_table, output_table)</span> |
| <a name="l00456"></a>00456 <span class="stringliteral"> return [[output_table, '</span>per-word topic counts<span class="stringliteral">']]</span> |
| <a name="l00457"></a>00457 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00458"></a>00458 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00459"></a>00459 <span class="comment">/**</span> |
| <a name="l00460"></a>00460 <span class="comment"> * @brief This UDF gets the description for each topic (top-k words) </span> |
| <a name="l00461"></a>00461 <span class="comment"> * @param model_table The model table generated by the training process</span> |
| <a name="l00462"></a>00462 <span class="comment"> * @param vocab_table The vocabulary table (<wordid, word>)</span> |
| <a name="l00463"></a>00463 <span class="comment"> * @param top_k The number of top words for each topic description </span> |
| <a name="l00464"></a>00464 <span class="comment"> * @param desc_table The output table for storing the per-topic description </span> |
| <a name="l00465"></a>00465 <span class="comment"> **/</span> |
| <a name="l00466"></a>00466 CREATE OR REPLACE FUNCTION |
| <a name="l00467"></a>00467 MADLIB_SCHEMA.lda_get_topic_desc |
| <a name="l00468"></a>00468 ( |
| <a name="l00469"></a>00469 model_table TEXT, |
| <a name="l00470"></a>00470 vocab_table TEXT, |
| <a name="l00471"></a>00471 desc_table TEXT, |
| <a name="l00472"></a>00472 top_k INT4 |
| <a name="l00473"></a>00473 ) |
| <a name="l00474"></a>00474 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00475"></a>00475 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00476"></a>00476 <span class="stringliteral"> lda.get_topic_desc(</span> |
| <a name="l00477"></a>00477 <span class="stringliteral"> schema_madlib, model_table, vocab_table, desc_table, top_k)</span> |
| <a name="l00478"></a>00478 <span class="stringliteral"> return [[</span> |
| <a name="l00479"></a>00479 <span class="stringliteral"> desc_table, </span> |
| <a name="l00480"></a>00480 <span class="stringliteral"> """topic description, use "ORDER BY topicid, prob DESC" to check the</span> |
| <a name="l00481"></a>00481 <span class="stringliteral"> results"""]]</span> |
| <a name="l00482"></a>00482 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00483"></a>00483 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00484"></a>00484 <span class="comment">/**</span> |
| <a name="l00485"></a>00485 <span class="comment"> * @brief This UDF assigns topics to words in a document randomly.</span> |
| <a name="l00486"></a>00486 <span class="comment"> * @param word_count The number of words in the document</span> |
| <a name="l00487"></a>00487 <span class="comment"> * @param topic_num The number of topics (specified by the user)</span> |
| <a name="l00488"></a>00488 <span class="comment"> * @return The topic counts and topic assignments </span> |
| <a name="l00489"></a>00489 <span class="comment"> **/</span> |
| <a name="l00490"></a>00490 CREATE OR REPLACE FUNCTION |
| <a name="l00491"></a>00491 MADLIB_SCHEMA.__lda_random_assign |
| <a name="l00492"></a>00492 ( |
| <a name="l00493"></a>00493 word_count INT4, |
| <a name="l00494"></a>00494 topic_num INT4 |
| <a name="l00495"></a>00495 ) |
| <a name="l00496"></a>00496 RETURNS INT4[] |
| <a name="l00497"></a>00497 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_random_assign<span class="stringliteral">'</span> |
| <a name="l00498"></a>00498 <span class="stringliteral">LANGUAGE C STRICT;</span> |
| <a name="l00499"></a>00499 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00500"></a>00500 <span class="comment">/**</span> |
| <a name="l00501"></a>00501 <span class="comment"> * @brief This UDF learns the topics of words in a document and is the main</span> |
| <a name="l00502"></a>00502 <span class="comment"> * step of a Gibbs sampling iteration. The model parameter (including the</span> |
| <a name="l00503"></a>00503 <span class="comment"> * per-word topic counts and corpus-level topic counts) is passed to this</span> |
| <a name="l00504"></a>00504 <span class="comment"> * function in the first call and then transfered to the rest calls through</span> |
| <a name="l00505"></a>00505 <span class="comment"> * fcinfo->flinfo->fn_extra to allow the immediate update.</span> |
| <a name="l00506"></a>00506 <span class="comment"> * @param words The set of unique words in the document</span> |
| <a name="l00507"></a>00507 <span class="comment"> * @param counts The counts of each unique words in the document</span> |
| <a name="l00508"></a>00508 <span class="comment"> * (sum(counts) = word_count)</span> |
| <a name="l00509"></a>00509 <span class="comment"> * @param doc_topic The current per-doc topic counts and topic</span> |
| <a name="l00510"></a>00510 <span class="comment"> * assignments</span> |
| <a name="l00511"></a>00511 <span class="comment"> * @param model The current model (including the per-word topic counts</span> |
| <a name="l00512"></a>00512 <span class="comment"> * and the corpus-level topic counts)</span> |
| <a name="l00513"></a>00513 <span class="comment"> * @param alpha The Dirichlet parameter for per-document topic multinomial </span> |
| <a name="l00514"></a>00514 <span class="comment"> * @param beta The Dirichlet parameter for per-topic word multinomial</span> |
| <a name="l00515"></a>00515 <span class="comment"> * @param voc_size The size of vocabulary</span> |
| <a name="l00516"></a>00516 <span class="comment"> * @param topic_num The number of topics</span> |
| <a name="l00517"></a>00517 <span class="comment"> * @return The learned topic counts and topic assignments </span> |
| <a name="l00518"></a>00518 <span class="comment"> **/</span> |
| <a name="l00519"></a>00519 CREATE OR REPLACE FUNCTION |
| <a name="l00520"></a>00520 MADLIB_SCHEMA.__lda_gibbs_sample |
| <a name="l00521"></a>00521 ( |
| <a name="l00522"></a>00522 words INT4[], |
| <a name="l00523"></a>00523 counts INT4[], |
| <a name="l00524"></a>00524 doc_topic INT4[], |
| <a name="l00525"></a>00525 model INT4[], |
| <a name="l00526"></a>00526 alpha FLOAT8, |
| <a name="l00527"></a>00527 beta FLOAT8, |
| <a name="l00528"></a>00528 voc_size INT4, |
| <a name="l00529"></a>00529 topic_num INT4, |
| <a name="l00530"></a>00530 iter_num INT4 |
| <a name="l00531"></a>00531 ) |
| <a name="l00532"></a>00532 RETURNS INT4[] |
| <a name="l00533"></a>00533 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_gibbs_sample<span class="stringliteral">'</span> |
| <a name="l00534"></a>00534 <span class="stringliteral">LANGUAGE C;</span> |
| <a name="l00535"></a>00535 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00536"></a>00536 <span class="comment">/**</span> |
| <a name="l00537"></a>00537 <span class="comment"> * @brief This UDF is the sfunc for the aggregator computing the topic counts</span> |
| <a name="l00538"></a>00538 <span class="comment"> * for each word and the topic count in the whole corpus. It scans the topic</span> |
| <a name="l00539"></a>00539 <span class="comment"> * assignments in a document and updates the topic counts.</span> |
| <a name="l00540"></a>00540 <span class="comment"> * @param state The topic counts</span> |
| <a name="l00541"></a>00541 <span class="comment"> * @param words The unique words in the document</span> |
| <a name="l00542"></a>00542 <span class="comment"> * @param counts The counts of each unique words in the document</span> |
| <a name="l00543"></a>00543 <span class="comment"> * (sum(counts) = word_count)</span> |
| <a name="l00544"></a>00544 <span class="comment"> * @param topic_assignment The topic assignments in the document</span> |
| <a name="l00545"></a>00545 <span class="comment"> * @param topic_num The number of topics</span> |
| <a name="l00546"></a>00546 <span class="comment"> * @return The updated state</span> |
| <a name="l00547"></a>00547 <span class="comment"> **/</span> |
| <a name="l00548"></a>00548 CREATE OR REPLACE FUNCTION |
| <a name="l00549"></a>00549 MADLIB_SCHEMA.__lda_count_topic_sfunc |
| <a name="l00550"></a>00550 ( |
| <a name="l00551"></a>00551 state INT4[], |
| <a name="l00552"></a>00552 words INT4[], |
| <a name="l00553"></a>00553 counts INT4[], |
| <a name="l00554"></a>00554 topic_assignment INT4[], |
| <a name="l00555"></a>00555 voc_size INT4, |
| <a name="l00556"></a>00556 topic_num INT4 |
| <a name="l00557"></a>00557 ) |
| <a name="l00558"></a>00558 RETURNS INT4[] |
| <a name="l00559"></a>00559 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_count_topic_sfunc<span class="stringliteral">'</span> |
| <a name="l00560"></a>00560 <span class="stringliteral">LANGUAGE C;</span> |
| <a name="l00561"></a>00561 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00562"></a>00562 <span class="comment">/**</span> |
| <a name="l00563"></a>00563 <span class="comment"> * @brief This UDF is the prefunc for the aggregator computing the per-word</span> |
| <a name="l00564"></a>00564 <span class="comment"> * topic counts.</span> |
| <a name="l00565"></a>00565 <span class="comment"> * @param state1 The local word topic counts</span> |
| <a name="l00566"></a>00566 <span class="comment"> * @param state2 The local word topic counts</span> |
| <a name="l00567"></a>00567 <span class="comment"> * @return The element-wise sum of two local states</span> |
| <a name="l00568"></a>00568 <span class="comment"> **/</span> |
| <a name="l00569"></a>00569 CREATE OR REPLACE FUNCTION |
| <a name="l00570"></a>00570 MADLIB_SCHEMA.__lda_count_topic_prefunc |
| <a name="l00571"></a>00571 ( |
| <a name="l00572"></a>00572 state1 INT4[], |
| <a name="l00573"></a>00573 state2 INT4[] |
| <a name="l00574"></a>00574 ) |
| <a name="l00575"></a>00575 RETURNS INT4[] |
| <a name="l00576"></a>00576 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_count_topic_prefunc<span class="stringliteral">'</span> |
| <a name="l00577"></a>00577 <span class="stringliteral">LANGUAGE C IMMUTABLE STRICT;</span> |
| <a name="l00578"></a>00578 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00579"></a>00579 <span class="comment">/**</span> |
| <a name="l00580"></a>00580 <span class="comment"> * @brief This uda computes the word topic counts by scanning and summing</span> |
| <a name="l00581"></a>00581 <span class="comment"> * up topic assignments in each document.</span> |
| <a name="l00582"></a>00582 <span class="comment"> * @param words The unique words in the document</span> |
| <a name="l00583"></a>00583 <span class="comment"> * @param counts The counts of each unique words in the document</span> |
| <a name="l00584"></a>00584 <span class="comment"> * @param topic_assignment The topic assignments in the document</span> |
| <a name="l00585"></a>00585 <span class="comment"> * @param voc_size The size of vocabulary</span> |
| <a name="l00586"></a>00586 <span class="comment"> * @param topic_num The number of topics</span> |
| <a name="l00587"></a>00587 <span class="comment"> * @return The word topic counts (a 1-d array embeding a 2-d array)</span> |
| <a name="l00588"></a>00588 <span class="comment"> **/</span> |
| <a name="l00589"></a>00589 DROP AGGREGATE IF EXISTS |
| <a name="l00590"></a>00590 MADLIB_SCHEMA.__lda_count_topic_agg |
| <a name="l00591"></a>00591 ( |
| <a name="l00592"></a>00592 INT4[], |
| <a name="l00593"></a>00593 INT4[], |
| <a name="l00594"></a>00594 INT4[], |
| <a name="l00595"></a>00595 INT4, |
| <a name="l00596"></a>00596 INT4 |
| <a name="l00597"></a>00597 ); |
| <a name="l00598"></a>00598 CREATE AGGREGATE |
| <a name="l00599"></a>00599 MADLIB_SCHEMA.__lda_count_topic_agg |
| <a name="l00600"></a>00600 ( |
| <a name="l00601"></a>00601 INT4[], |
| <a name="l00602"></a>00602 INT4[], |
| <a name="l00603"></a>00603 INT4[], |
| <a name="l00604"></a>00604 INT4, |
| <a name="l00605"></a>00605 INT4 |
| <a name="l00606"></a>00606 ) |
| <a name="l00607"></a>00607 ( |
| <a name="l00608"></a>00608 stype = INT4[], |
| <a name="l00609"></a>00609 sfunc = MADLIB_SCHEMA.__lda_count_topic_sfunc |
| <a name="l00610"></a>00610 m4_ifdef( |
| <a name="l00611"></a>00611 `GREENPLUM', |
| <a name="l00612"></a>00612 `, prefunc = MADLIB_SCHEMA.__lda_count_topic_prefunc<span class="stringliteral">'</span> |
| <a name="l00613"></a>00613 <span class="stringliteral"> )</span> |
| <a name="l00614"></a>00614 <span class="stringliteral">);</span> |
| <a name="l00615"></a>00615 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00616"></a>00616 <span class="comment">/**</span> |
| <a name="l00617"></a>00617 <span class="comment"> * @brief This UDF computes the perplexity given the output data table and the</span> |
| <a name="l00618"></a>00618 <span class="comment"> * model table.</span> |
| <a name="l00619"></a>00619 <span class="comment"> * @param model_table The model table generated by lda_train</span> |
| <a name="l00620"></a>00620 <span class="comment"> * @param output_table The output data table generated by lda_predict</span> |
| <a name="l00621"></a>00621 <span class="comment"> * @return The perplexity</span> |
| <a name="l00622"></a>00622 <span class="comment"> **/</span> |
| <a name="l00623"></a>00623 CREATE OR REPLACE FUNCTION |
| <a name="l00624"></a>00624 MADLIB_SCHEMA.lda_get_perplexity |
| <a name="l00625"></a>00625 ( |
| <a name="l00626"></a>00626 model_table TEXT, |
| <a name="l00627"></a>00627 output_data_table TEXT |
| <a name="l00628"></a>00628 ) |
| <a name="l00629"></a>00629 RETURNS FLOAT8 AS $$ |
| <a name="l00630"></a>00630 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00631"></a>00631 <span class="stringliteral"> return lda.get_perplexity(</span> |
| <a name="l00632"></a>00632 <span class="stringliteral"> schema_madlib, model_table, output_data_table)</span> |
| <a name="l00633"></a>00633 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00634"></a>00634 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00635"></a>00635 <span class="comment">/**</span> |
| <a name="l00636"></a>00636 <span class="comment"> * @brief This UDF is the sfunc for the aggregator computing the perpleixty.</span> |
| <a name="l00637"></a>00637 <span class="comment"> * @param state The cached model plus perplexity</span> |
| <a name="l00638"></a>00638 <span class="comment"> * @param words The unique words in the document</span> |
| <a name="l00639"></a>00639 <span class="comment"> * @param counts The counts of each unique words in the document</span> |
| <a name="l00640"></a>00640 <span class="comment"> * @param doc_topic The topic counts in the document</span> |
| <a name="l00641"></a>00641 <span class="comment"> * @param model The learned model</span> |
| <a name="l00642"></a>00642 <span class="comment"> * @param alpha The Dirichlet parameter for per-document topic multinomial </span> |
| <a name="l00643"></a>00643 <span class="comment"> * @param beta The Dirichlet parameter for per-topic word multinomial</span> |
| <a name="l00644"></a>00644 <span class="comment"> * @param voc_size The size of vocabulary</span> |
| <a name="l00645"></a>00645 <span class="comment"> * @param topic_num The number of topics</span> |
| <a name="l00646"></a>00646 <span class="comment"> * @return The updated state </span> |
| <a name="l00647"></a>00647 <span class="comment"> **/</span> |
| <a name="l00648"></a>00648 CREATE OR REPLACE FUNCTION |
| <a name="l00649"></a>00649 MADLIB_SCHEMA.__lda_perplexity_sfunc |
| <a name="l00650"></a>00650 ( |
| <a name="l00651"></a>00651 state INT4[], |
| <a name="l00652"></a>00652 words INT4[], |
| <a name="l00653"></a>00653 counts INT4[], |
| <a name="l00654"></a>00654 doc_topic INT4[], |
| <a name="l00655"></a>00655 model INT4[][], |
| <a name="l00656"></a>00656 alpha FLOAT8, |
| <a name="l00657"></a>00657 beta FLOAT8, |
| <a name="l00658"></a>00658 voc_size INT4, |
| <a name="l00659"></a>00659 topic_num INT4 |
| <a name="l00660"></a>00660 ) |
| <a name="l00661"></a>00661 RETURNS INT4[] |
| <a name="l00662"></a>00662 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_perplexity_sfunc<span class="stringliteral">'</span> |
| <a name="l00663"></a>00663 <span class="stringliteral">LANGUAGE C IMMUTABLE;</span> |
| <a name="l00664"></a>00664 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00665"></a>00665 <span class="comment">/**</span> |
| <a name="l00666"></a>00666 <span class="comment"> * @brief This UDF is the prefunc for the aggregator computing the perplexity.</span> |
| <a name="l00667"></a>00667 <span class="comment"> * @param state1 The local state </span> |
| <a name="l00668"></a>00668 <span class="comment"> * @param state2 The local state </span> |
| <a name="l00669"></a>00669 <span class="comment"> * @return The merged state </span> |
| <a name="l00670"></a>00670 <span class="comment"> **/</span> |
| <a name="l00671"></a>00671 CREATE OR REPLACE FUNCTION |
| <a name="l00672"></a>00672 MADLIB_SCHEMA.__lda_perplexity_prefunc |
| <a name="l00673"></a>00673 ( |
| <a name="l00674"></a>00674 state1 INT4[], |
| <a name="l00675"></a>00675 state2 INT4[] |
| <a name="l00676"></a>00676 ) |
| <a name="l00677"></a>00677 RETURNS INT4[] |
| <a name="l00678"></a>00678 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_perplexity_prefunc<span class="stringliteral">'</span> |
| <a name="l00679"></a>00679 <span class="stringliteral">LANGUAGE C IMMUTABLE STRICT;</span> |
| <a name="l00680"></a>00680 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00681"></a>00681 <span class="comment">/**</span> |
| <a name="l00682"></a>00682 <span class="comment"> * @brief This UDF is the finalfunc for the aggregator computing the perplexity.</span> |
| <a name="l00683"></a>00683 <span class="comment"> * @param state The merged state </span> |
| <a name="l00684"></a>00684 <span class="comment"> * @return The perpleixty</span> |
| <a name="l00685"></a>00685 <span class="comment"> **/</span> |
| <a name="l00686"></a>00686 CREATE OR REPLACE FUNCTION |
| <a name="l00687"></a>00687 MADLIB_SCHEMA.__lda_perplexity_ffunc |
| <a name="l00688"></a>00688 ( |
| <a name="l00689"></a>00689 state INT4[] |
| <a name="l00690"></a>00690 ) |
| <a name="l00691"></a>00691 RETURNS FLOAT8 |
| <a name="l00692"></a>00692 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_perplexity_ffunc<span class="stringliteral">'</span> |
| <a name="l00693"></a>00693 <span class="stringliteral">LANGUAGE C IMMUTABLE STRICT;</span> |
| <a name="l00694"></a>00694 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00695"></a>00695 <span class="comment">/**</span> |
| <a name="l00696"></a>00696 <span class="comment"> * @brief This aggregator computes the perpleixty.</span> |
| <a name="l00697"></a>00697 <span class="comment"> * @param words The unique words in the document</span> |
| <a name="l00698"></a>00698 <span class="comment"> * @param counts The counts of each unique words in the document</span> |
| <a name="l00699"></a>00699 <span class="comment"> * @param doc_topic The topic counts in the document</span> |
| <a name="l00700"></a>00700 <span class="comment"> * @param model The learned model</span> |
| <a name="l00701"></a>00701 <span class="comment"> * @param alpha The Dirichlet parameter for per-document topic multinomial </span> |
| <a name="l00702"></a>00702 <span class="comment"> * @param beta The Dirichlet parameter for per-topic word multinomial</span> |
| <a name="l00703"></a>00703 <span class="comment"> * @param voc_size The size of vocabulary</span> |
| <a name="l00704"></a>00704 <span class="comment"> * @param topic_num The number of topics</span> |
| <a name="l00705"></a>00705 <span class="comment"> * @return The updated perplexity</span> |
| <a name="l00706"></a>00706 <span class="comment"> **/</span> |
| <a name="l00707"></a>00707 DROP AGGREGATE IF EXISTS |
| <a name="l00708"></a>00708 MADLIB_SCHEMA.__lda_perplexity_agg |
| <a name="l00709"></a>00709 ( |
| <a name="l00710"></a>00710 INT4[], |
| <a name="l00711"></a>00711 INT4[], |
| <a name="l00712"></a>00712 INT4[], |
| <a name="l00713"></a>00713 INT4[], |
| <a name="l00714"></a>00714 FLOAT8, |
| <a name="l00715"></a>00715 FLOAT8, |
| <a name="l00716"></a>00716 INT4, |
| <a name="l00717"></a>00717 INT4 |
| <a name="l00718"></a>00718 ); |
| <a name="l00719"></a>00719 CREATE AGGREGATE |
| <a name="l00720"></a>00720 MADLIB_SCHEMA.__lda_perplexity_agg |
| <a name="l00721"></a>00721 ( |
| <a name="l00722"></a>00722 INT4[], |
| <a name="l00723"></a>00723 INT4[], |
| <a name="l00724"></a>00724 INT4[], |
| <a name="l00725"></a>00725 INT4[], |
| <a name="l00726"></a>00726 FLOAT8, |
| <a name="l00727"></a>00727 FLOAT8, |
| <a name="l00728"></a>00728 INT4, |
| <a name="l00729"></a>00729 INT4 |
| <a name="l00730"></a>00730 ) |
| <a name="l00731"></a>00731 ( |
| <a name="l00732"></a>00732 stype = INT4[], |
| <a name="l00733"></a>00733 sfunc = MADLIB_SCHEMA.__lda_perplexity_sfunc, |
| <a name="l00734"></a>00734 finalfunc = MADLIB_SCHEMA.__lda_perplexity_ffunc |
| <a name="l00735"></a>00735 m4_ifdef( |
| <a name="l00736"></a>00736 `GREENPLUM', |
| <a name="l00737"></a>00737 `, prefunc = MADLIB_SCHEMA.__lda_perplexity_prefunc<span class="stringliteral">'</span> |
| <a name="l00738"></a>00738 <span class="stringliteral"> )</span> |
| <a name="l00739"></a>00739 <span class="stringliteral">);</span> |
| <a name="l00740"></a>00740 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00741"></a>00741 <span class="comment">/**</span> |
| <a name="l00742"></a>00742 <span class="comment"> * @brief Unnest a 2-D array into a set of 1-D arrays </span> |
| <a name="l00743"></a>00743 <span class="comment"> * @param arr The 2-D array to be unnested</span> |
| <a name="l00744"></a>00744 <span class="comment"> * @return The unnested 1-D arrays</span> |
| <a name="l00745"></a>00745 <span class="comment"> **/</span> |
| <a name="l00746"></a>00746 CREATE OR REPLACE FUNCTION |
| <a name="l00747"></a>00747 MADLIB_SCHEMA.__lda_util_unnest |
| <a name="l00748"></a>00748 ( |
| <a name="l00749"></a>00749 arr INT4[][] |
| <a name="l00750"></a>00750 ) |
| <a name="l00751"></a>00751 RETURNS SETOF INT4[] |
| <a name="l00752"></a>00752 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_unnest<span class="stringliteral">'</span> |
| <a name="l00753"></a>00753 <span class="stringliteral">LANGUAGE C IMMUTABLE STRICT;</span> |
| <a name="l00754"></a>00754 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00755"></a>00755 <span class="comment">/**</span> |
| <a name="l00756"></a>00756 <span class="comment"> * @brief Transpose a 2-D array</span> |
| <a name="l00757"></a>00757 <span class="comment"> * @param matrix The input 2-D array</span> |
| <a name="l00758"></a>00758 <span class="comment"> * @param The transposed array</span> |
| <a name="l00759"></a>00759 <span class="comment"> **/</span> |
| <a name="l00760"></a>00760 CREATE OR REPLACE FUNCTION |
| <a name="l00761"></a>00761 MADLIB_SCHEMA.__lda_util_transpose |
| <a name="l00762"></a>00762 ( |
| <a name="l00763"></a>00763 matrix INT4[][] |
| <a name="l00764"></a>00764 ) |
| <a name="l00765"></a>00765 RETURNS INT4[][] |
| <a name="l00766"></a>00766 AS 'MODULE_PATHNAME<span class="stringliteral">', '</span>lda_transpose<span class="stringliteral">'</span> |
| <a name="l00767"></a>00767 <span class="stringliteral">LANGUAGE C IMMUTABLE STRICT;</span> |
| <a name="l00768"></a>00768 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00769"></a>00769 <span class="comment">/**</span> |
| <a name="l00770"></a>00770 <span class="comment"> * @brief L1 normalization with smoothing</span> |
| <a name="l00771"></a>00771 <span class="comment"> * @param arr The array to be normalized</span> |
| <a name="l00772"></a>00772 <span class="comment"> * @param smooth The smoothing parameter</span> |
| <a name="l00773"></a>00773 <span class="comment"> * @return The normalized vector</span> |
| <a name="l00774"></a>00774 <span class="comment"> **/</span> |
| <a name="l00775"></a>00775 CREATE OR REPLACE FUNCTION |
| <a name="l00776"></a>00776 MADLIB_SCHEMA.__lda_util_norm_with_smoothing |
| <a name="l00777"></a>00777 ( |
| <a name="l00778"></a>00778 arr FLOAT8[], |
| <a name="l00779"></a>00779 smooth FLOAT8 |
| <a name="l00780"></a>00780 ) |
| <a name="l00781"></a>00781 RETURNS FLOAT8[] AS $$ |
| <a name="l00782"></a>00782 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00783"></a>00783 <span class="stringliteral"> return lda.l1_norm_with_smoothing(arr, smooth)</span> |
| <a name="l00784"></a>00784 <span class="stringliteral">$$ LANGUAGE PLPYTHONU STRICT;</span> |
| <a name="l00785"></a>00785 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00786"></a>00786 <span class="comment">/**</span> |
| <a name="l00787"></a>00787 <span class="comment"> * @brief This UDF returns the index of elements in a sorted order</span> |
| <a name="l00788"></a>00788 <span class="comment"> * @param arr The array to be sorted</span> |
| <a name="l00789"></a>00789 <span class="comment"> * @return The index of elements</span> |
| <a name="l00790"></a>00790 <span class="comment"> **/</span> |
| <a name="l00791"></a>00791 CREATE OR REPLACE FUNCTION |
| <a name="l00792"></a>00792 MADLIB_SCHEMA.__lda_util_index_sort |
| <a name="l00793"></a>00793 ( |
| <a name="l00794"></a>00794 arr FLOAT8[] |
| <a name="l00795"></a>00795 ) |
| <a name="l00796"></a>00796 RETURNS INT4[] AS $$ |
| <a name="l00797"></a>00797 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00798"></a>00798 <span class="stringliteral"> return lda.index_sort(arr)</span> |
| <a name="l00799"></a>00799 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00800"></a>00800 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00801"></a>00801 <span class="comment">/**</span> |
| <a name="l00802"></a>00802 <span class="comment"> * @brief This UDF checks the vocabulary and converts non-continous wordids into</span> |
| <a name="l00803"></a>00803 <span class="comment"> * continuous integers ranging from 0 to voc_size - 1.</span> |
| <a name="l00804"></a>00804 <span class="comment"> * @param vocab_table The vocabulary table in the form of </span> |
| <a name="l00805"></a>00805 <span class="comment"> <wordid::int4, word::text></span> |
| <a name="l00806"></a>00806 <span class="comment"> * @param output_vocab_table The regularized vocabulary table </span> |
| <a name="l00807"></a>00807 <span class="comment"> **/</span> |
| <a name="l00808"></a>00808 CREATE OR REPLACE FUNCTION |
| <a name="l00809"></a>00809 MADLIB_SCHEMA.__lda_util_norm_vocab |
| <a name="l00810"></a>00810 ( |
| <a name="l00811"></a>00811 vocab_table TEXT, |
| <a name="l00812"></a>00812 output_vocab_table TEXT |
| <a name="l00813"></a>00813 ) |
| <a name="l00814"></a>00814 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00815"></a>00815 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00816"></a>00816 <span class="stringliteral"> lda.norm_vocab(vocab_table, output_vocab_table)</span> |
| <a name="l00817"></a>00817 <span class="stringliteral"> return [[output_vocab_table,'</span>normalized vocbulary table<span class="stringliteral">']]</span> |
| <a name="l00818"></a>00818 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00819"></a>00819 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00820"></a>00820 <span class="comment">/**</span> |
| <a name="l00821"></a>00821 <span class="comment"> * @brief This UDF converts the data table according to the normalized</span> |
| <a name="l00822"></a>00822 <span class="comment"> * vocabulary, and all rows with non-positive count values will be removed</span> |
| <a name="l00823"></a>00823 <span class="comment"> * @param data_table The data table to be normalized</span> |
| <a name="l00824"></a>00824 <span class="comment"> * @param vocab_table The normalized vocabulary table</span> |
| <a name="l00825"></a>00825 <span class="comment"> * @param output_data_table The normalized data table</span> |
| <a name="l00826"></a>00826 <span class="comment"> **/</span> |
| <a name="l00827"></a>00827 CREATE OR REPLACE FUNCTION |
| <a name="l00828"></a>00828 MADLIB_SCHEMA.__lda_util_norm_dataset |
| <a name="l00829"></a>00829 ( |
| <a name="l00830"></a>00830 data_table TEXT, |
| <a name="l00831"></a>00831 norm_vocab_table TEXT, |
| <a name="l00832"></a>00832 output_data_table TEXT |
| <a name="l00833"></a>00833 ) |
| <a name="l00834"></a>00834 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00835"></a>00835 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00836"></a>00836 <span class="stringliteral"> lda.norm_dataset(data_table, norm_vocab_table, output_data_table)</span> |
| <a name="l00837"></a>00837 <span class="stringliteral"> return [[output_data_table,'</span>normalized data table<span class="stringliteral">']]</span> |
| <a name="l00838"></a>00838 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00839"></a>00839 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00840"></a>00840 <span class="comment">/**</span> |
| <a name="l00841"></a>00841 <span class="comment"> * @brief This UDF extracts the list of wordids from the data table and joins</span> |
| <a name="l00842"></a>00842 <span class="comment"> * it with the vocabulary table to get the list of common wordids, next it will</span> |
| <a name="l00843"></a>00843 <span class="comment"> * normalize the vocabulary based on the common wordids and then normalize the</span> |
| <a name="l00844"></a>00844 <span class="comment"> * data table based on the normalized vocabulary.</span> |
| <a name="l00845"></a>00845 <span class="comment"> * @param data_table The data table to be normalized</span> |
| <a name="l00846"></a>00846 <span class="comment"> * @param vocab_table The vocabulary table to be normalized</span> |
| <a name="l00847"></a>00847 <span class="comment"> * @param output_data_table The normalized data table</span> |
| <a name="l00848"></a>00848 <span class="comment"> * @param output_vocab_table The normalized vocabulary table</span> |
| <a name="l00849"></a>00849 <span class="comment"> **/</span> |
| <a name="l00850"></a>00850 CREATE OR REPLACE FUNCTION |
| <a name="l00851"></a>00851 MADLIB_SCHEMA.__lda_util_conorm_data |
| <a name="l00852"></a>00852 ( |
| <a name="l00853"></a>00853 data_table TEXT, |
| <a name="l00854"></a>00854 vocab_table TEXT, |
| <a name="l00855"></a>00855 output_data_table TEXT, |
| <a name="l00856"></a>00856 output_vocab_table TEXT |
| <a name="l00857"></a>00857 ) |
| <a name="l00858"></a>00858 RETURNS SETOF MADLIB_SCHEMA.lda_result AS $$ |
| <a name="l00859"></a>00859 PythonFunctionBodyOnly(`lda', `lda<span class="stringliteral">')</span> |
| <a name="l00860"></a>00860 <span class="stringliteral"> lda.conorm_data(</span> |
| <a name="l00861"></a>00861 <span class="stringliteral"> data_table, vocab_table, output_data_table, output_vocab_table)</span> |
| <a name="l00862"></a>00862 <span class="stringliteral"> return [[output_data_table,'</span>normalized data table<span class="stringliteral">'],</span> |
| <a name="l00863"></a>00863 <span class="stringliteral"> [output_vocab_table,'</span>normalized vocab table<span class="stringliteral">']]</span> |
| <a name="l00864"></a>00864 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT;</span> |
| <a name="l00865"></a>00865 <span class="stringliteral"></span> |
| </pre></div></div> |
| </div> |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="lda_8sql__in.html">lda.sql_in</a> </li> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Functions</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| |
| <li class="footer">Generated on Fri May 10 2013 01:37:13 for MADlib by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.5.1 </li> |
| </ul> |
| </div> |
| |
| |
| </body> |
| </html> |
