| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <title>MADlib: linear.sql_in Source File</title> |
| |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <script src="../mathjax/MathJax.js"> |
| MathJax.Hub.Config({ |
| extensions: ["tex2jax.js", "TeX/AMSmath.js", "TeX/AMSsymbols.js"], |
| jax: ["input/TeX","output/HTML-CSS"], |
| }); |
| </script> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div! --> |
| |
| |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| |
| |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">MADlib |
|  <span id="projectnumber">0.6</span> <span style="font-size:10pt; font-style:italic"><a href="../latest/./linear_8sql__in_source.html"> A newer version is available</a></span> |
| </div> |
| <div id="projectbrief">User Documentation</div> |
| </td> |
| |
| |
| |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| |
| <!-- Generated by Doxygen 1.7.5.1 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="modules.html"><span>Modules</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| initNavTree('linear_8sql__in.html',''); |
| </script> |
| <div id="doc-content"> |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">linear.sql_in</div> </div> |
| </div> |
| <div class="contents"> |
| <a href="linear_8sql__in.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* ----------------------------------------------------------------------- */</span><span class="comment">/**</span> |
| <a name="l00002"></a>00002 <span class="comment"> *</span> |
| <a name="l00003"></a>00003 <span class="comment"> * @file linear.sql_in</span> |
| <a name="l00004"></a>00004 <span class="comment"> *</span> |
| <a name="l00005"></a>00005 <span class="comment"> * @brief SQL functions for linear regression</span> |
| <a name="l00006"></a>00006 <span class="comment"> * @date January 2011</span> |
| <a name="l00007"></a>00007 <span class="comment"> *</span> |
| <a name="l00008"></a>00008 <span class="comment"> * @sa For a brief introduction to linear regression, see the module</span> |
| <a name="l00009"></a>00009 <span class="comment"> * description \ref grp_linreg.</span> |
| <a name="l00010"></a>00010 <span class="comment"> *</span> |
| <a name="l00011"></a>00011 <span class="comment"> */</span><span class="comment">/* ----------------------------------------------------------------------- */</span> |
| <a name="l00012"></a>00012 |
| <a name="l00013"></a>00013 m4_include(`SQLCommon.m4<span class="stringliteral">') --'</span> |
| <a name="l00014"></a>00014 <span class="comment"></span> |
| <a name="l00015"></a>00015 <span class="comment">/**</span> |
| <a name="l00016"></a>00016 <span class="comment">@addtogroup grp_linreg</span> |
| <a name="l00017"></a>00017 <span class="comment"></span> |
| <a name="l00018"></a>00018 <span class="comment">@about</span> |
| <a name="l00019"></a>00019 <span class="comment"></span> |
| <a name="l00020"></a>00020 <span class="comment">Ordinary least-squares (OLS) linear regression refers to a stochastic model in</span> |
| <a name="l00021"></a>00021 <span class="comment">which the conditional mean of the dependent variable (usually denoted \f$ Y \f$)</span> |
| <a name="l00022"></a>00022 <span class="comment">is an affine function of the vector of independent variables (usually denoted</span> |
| <a name="l00023"></a>00023 <span class="comment">\f$ \boldsymbol x \f$). That is,</span> |
| <a name="l00024"></a>00024 <span class="comment">\f[</span> |
| <a name="l00025"></a>00025 <span class="comment"> E[Y \mid \boldsymbol x] = \boldsymbol c^T \boldsymbol x</span> |
| <a name="l00026"></a>00026 <span class="comment">\f]</span> |
| <a name="l00027"></a>00027 <span class="comment">for some unknown vector of coefficients \f$ \boldsymbol c \f$. The assumption is</span> |
| <a name="l00028"></a>00028 <span class="comment">that the residuals are i.i.d. distributed Gaussians. That is, the (conditional)</span> |
| <a name="l00029"></a>00029 <span class="comment">probability density of \f$ Y \f$ is given by</span> |
| <a name="l00030"></a>00030 <span class="comment">\f[</span> |
| <a name="l00031"></a>00031 <span class="comment"> f(y \mid \boldsymbol x)</span> |
| <a name="l00032"></a>00032 <span class="comment"> = \frac{1}{\sqrt{2 \pi \sigma^2}}</span> |
| <a name="l00033"></a>00033 <span class="comment"> \cdot \exp\left(-\frac{1}{2 \sigma^2}</span> |
| <a name="l00034"></a>00034 <span class="comment"> \cdot (y - \boldsymbol x^T \boldsymbol c)^2 \right)</span> |
| <a name="l00035"></a>00035 <span class="comment"> \,.</span> |
| <a name="l00036"></a>00036 <span class="comment">\f]</span> |
| <a name="l00037"></a>00037 <span class="comment">OLS linear regression finds the vector of coefficients \f$ \boldsymbol c \f$</span> |
| <a name="l00038"></a>00038 <span class="comment">that maximizes the likelihood of the observations.</span> |
| <a name="l00039"></a>00039 <span class="comment"></span> |
| <a name="l00040"></a>00040 <span class="comment">Let</span> |
| <a name="l00041"></a>00041 <span class="comment">- \f$ \boldsymbol y \in \mathbf R^n \f$ denote the vector of observed dependent</span> |
| <a name="l00042"></a>00042 <span class="comment"> variables, with \f$ n \f$ rows, containing the observed values of the</span> |
| <a name="l00043"></a>00043 <span class="comment"> dependent variable,</span> |
| <a name="l00044"></a>00044 <span class="comment">- \f$ X \in \mathbf R^{n \times k} \f$ denote the design matrix with \f$ k \f$</span> |
| <a name="l00045"></a>00045 <span class="comment"> columns and \f$ n \f$ rows, containing all observed vectors of independent</span> |
| <a name="l00046"></a>00046 <span class="comment"> variables.</span> |
| <a name="l00047"></a>00047 <span class="comment"> \f$ \boldsymbol x_i \f$ as rows,</span> |
| <a name="l00048"></a>00048 <span class="comment">- \f$ X^T \f$ denote the transpose of \f$ X \f$,</span> |
| <a name="l00049"></a>00049 <span class="comment">- \f$ X^+ \f$ denote the pseudo-inverse of \f$ X \f$.</span> |
| <a name="l00050"></a>00050 <span class="comment"></span> |
| <a name="l00051"></a>00051 <span class="comment">Maximizing the likelihood is equivalent to maximizing the log-likelihood</span> |
| <a name="l00052"></a>00052 <span class="comment">\f$ \sum_{i=1}^n \log f(y_i \mid \boldsymbol x_i) \f$, which simplifies to</span> |
| <a name="l00053"></a>00053 <span class="comment">minimizing the <b>residual sum of squares</b> \f$ RSS \f$ (also called sum of</span> |
| <a name="l00054"></a>00054 <span class="comment">squared residuals or sum of squared errors of prediction),</span> |
| <a name="l00055"></a>00055 <span class="comment">\f[</span> |
| <a name="l00056"></a>00056 <span class="comment"> RSS = \sum_{i=1}^n ( y_i - \boldsymbol c^T \boldsymbol x_i )^2</span> |
| <a name="l00057"></a>00057 <span class="comment"> = (\boldsymbol y - X \boldsymbol c)^T (\boldsymbol y - X \boldsymbol c)</span> |
| <a name="l00058"></a>00058 <span class="comment"> \,.</span> |
| <a name="l00059"></a>00059 <span class="comment">\f]</span> |
| <a name="l00060"></a>00060 <span class="comment">The first-order conditions yield that the \f$ RSS \f$ is minimized at</span> |
| <a name="l00061"></a>00061 <span class="comment">\f[</span> |
| <a name="l00062"></a>00062 <span class="comment"> \boldsymbol c = (X^T X)^+ X^T \boldsymbol y</span> |
| <a name="l00063"></a>00063 <span class="comment"> \,.</span> |
| <a name="l00064"></a>00064 <span class="comment">\f]</span> |
| <a name="l00065"></a>00065 <span class="comment"></span> |
| <a name="l00066"></a>00066 <span class="comment">Computing the <b>total sum of squares</b> \f$ TSS \f$, the <b>explained</span> |
| <a name="l00067"></a>00067 <span class="comment">sum of squares</b> \f$ ESS \f$ (also called the regression sum of</span> |
| <a name="l00068"></a>00068 <span class="comment">squares), and the <b>coefficient of determination</b> \f$ R^2 \f$ is</span> |
| <a name="l00069"></a>00069 <span class="comment">done according to the following formulas:</span> |
| <a name="l00070"></a>00070 <span class="comment">\f{align*}{</span> |
| <a name="l00071"></a>00071 <span class="comment"> ESS & = \boldsymbol y^T X \boldsymbol c</span> |
| <a name="l00072"></a>00072 <span class="comment"> - \frac{ \| y \|_1^2 }{n} \\</span> |
| <a name="l00073"></a>00073 <span class="comment"> TSS & = \sum_{i=1}^n y_i^2</span> |
| <a name="l00074"></a>00074 <span class="comment"> - \frac{ \| y \|_1^2 }{n} \\</span> |
| <a name="l00075"></a>00075 <span class="comment"> R^2 & = \frac{ESS}{TSS}</span> |
| <a name="l00076"></a>00076 <span class="comment">\f}</span> |
| <a name="l00077"></a>00077 <span class="comment">Note: The last equality follows from the definition</span> |
| <a name="l00078"></a>00078 <span class="comment">\f$ R^2 = 1 - \frac{RSS}{TSS} \f$ and the fact that for linear regression</span> |
| <a name="l00079"></a>00079 <span class="comment">\f$ TSS = RSS + ESS \f$. A proof of the latter can be found, e.g., at:</span> |
| <a name="l00080"></a>00080 <span class="comment">http://en.wikipedia.org/wiki/Sum_of_squares</span> |
| <a name="l00081"></a>00081 <span class="comment"></span> |
| <a name="l00082"></a>00082 <span class="comment">We estimate the variance</span> |
| <a name="l00083"></a>00083 <span class="comment">\f$ Var[Y - \boldsymbol c^T \boldsymbol x \mid \boldsymbol x] \f$ as</span> |
| <a name="l00084"></a>00084 <span class="comment">\f[</span> |
| <a name="l00085"></a>00085 <span class="comment"> \sigma^2 = \frac{RSS}{n - k}</span> |
| <a name="l00086"></a>00086 <span class="comment">\f]</span> |
| <a name="l00087"></a>00087 <span class="comment">and compute the t-statistic for coefficient \f$ i \f$ as</span> |
| <a name="l00088"></a>00088 <span class="comment">\f[</span> |
| <a name="l00089"></a>00089 <span class="comment"> t_i = \frac{c_i}{\sqrt{\sigma^2 \cdot \left( (X^T X)^{-1} \right)_{ii} }}</span> |
| <a name="l00090"></a>00090 <span class="comment"> \,.</span> |
| <a name="l00091"></a>00091 <span class="comment">\f]</span> |
| <a name="l00092"></a>00092 <span class="comment"></span> |
| <a name="l00093"></a>00093 <span class="comment">The \f$ p \f$-value for coefficient \f$ i \f$ gives the probability of seeing a</span> |
| <a name="l00094"></a>00094 <span class="comment">value at least as extreme as the one observed, provided that the null hypothesis</span> |
| <a name="l00095"></a>00095 <span class="comment">(\f$ c_i = 0 \f$) is true. Letting \f$ F_\nu \f$ denote the</span> |
| <a name="l00096"></a>00096 <span class="comment">cumulative density function of student-t with \f$ \nu \f$ degrees of freedom,</span> |
| <a name="l00097"></a>00097 <span class="comment">the \f$ p \f$-value for coefficient \f$ i \f$</span> |
| <a name="l00098"></a>00098 <span class="comment">is therefore</span> |
| <a name="l00099"></a>00099 <span class="comment">\f[</span> |
| <a name="l00100"></a>00100 <span class="comment"> p_i = \Pr(|T| \geq |t_i|) = 2 \cdot (1 - F_{n - k}( |t_i| ))</span> |
| <a name="l00101"></a>00101 <span class="comment">\f]</span> |
| <a name="l00102"></a>00102 <span class="comment">where \f$ T \f$ is a student-t distributed random variable with mean 0.</span> |
| <a name="l00103"></a>00103 <span class="comment"></span> |
| <a name="l00104"></a>00104 <span class="comment">The condition number [2] \f$ \kappa(X) = \|X\|_2\cdot\|X^{-1}\|_2\f$ is computed</span> |
| <a name="l00105"></a>00105 <span class="comment">as the product of two spectral norms [3]. The spectral norm of a matrix \f$X\f$</span> |
| <a name="l00106"></a>00106 <span class="comment">is the largest singular value of \f$X\f$ i.e. the square root of the largest</span> |
| <a name="l00107"></a>00107 <span class="comment">eigenvalue of the positive-semidefinite matrix \f$X^{*}X\f$:</span> |
| <a name="l00108"></a>00108 <span class="comment"></span> |
| <a name="l00109"></a>00109 <span class="comment">\f[</span> |
| <a name="l00110"></a>00110 <span class="comment"> \|X\|_2 = \sqrt{\lambda_{\max}\left(X^{*}X\right)}\ ,</span> |
| <a name="l00111"></a>00111 <span class="comment">\f]</span> |
| <a name="l00112"></a>00112 <span class="comment">where \f$X^{*}\f$ is the conjugate transpose of \f$X\f$.</span> |
| <a name="l00113"></a>00113 <span class="comment">The condition number of a linear regression problem</span> |
| <a name="l00114"></a>00114 <span class="comment">is a worst-case measure of how sensitive the</span> |
| <a name="l00115"></a>00115 <span class="comment">result is to small perturbations of the input. A large condition number (say,</span> |
| <a name="l00116"></a>00116 <span class="comment">more than 1000) indicates the presence of significant multicollinearity.</span> |
| <a name="l00117"></a>00117 <span class="comment"></span> |
| <a name="l00118"></a>00118 <span class="comment">@input</span> |
| <a name="l00119"></a>00119 <span class="comment"></span> |
| <a name="l00120"></a>00120 <span class="comment">The training data is expected to be of the following form:</span> |
| <a name="l00121"></a>00121 <span class="comment"><pre>{TABLE|VIEW} <em>sourceName</em> (</span> |
| <a name="l00122"></a>00122 <span class="comment"> ...</span> |
| <a name="l00123"></a>00123 <span class="comment"> <em>dependentVariable</em> FLOAT8,</span> |
| <a name="l00124"></a>00124 <span class="comment"> <em>independentVariables</em> FLOAT8[],</span> |
| <a name="l00125"></a>00125 <span class="comment"> ...</span> |
| <a name="l00126"></a>00126 <span class="comment">)</pre></span> |
| <a name="l00127"></a>00127 <span class="comment"></span> |
| <a name="l00128"></a>00128 <span class="comment">@usage</span> |
| <a name="l00129"></a>00129 <span class="comment"></span> |
| <a name="l00130"></a>00130 <span class="comment"><b>(1) The Simple Interface</b></span> |
| <a name="l00131"></a>00131 <span class="comment"></span> |
| <a name="l00132"></a>00132 <span class="comment">- Get vector of coefficients \f$ \boldsymbol c \f$ and all diagnostic statistics:</span> |
| <a name="l00133"></a>00133 <span class="comment"><pre>SELECT (madlib.\ref linregr(<em>dependentVariable</em>,</span> |
| <a name="l00134"></a>00134 <span class="comment"> <em>independentVariables</em>)).*</span> |
| <a name="l00135"></a>00135 <span class="comment">FROM <em>sourceName</em>;</pre></span> |
| <a name="l00136"></a>00136 <span class="comment"> Output:</span> |
| <a name="l00137"></a>00137 <span class="comment"> <pre></span> |
| <a name="l00138"></a>00138 <span class="comment">coef | r2 | std_err | t_stats | p_values | condition_no</span> |
| <a name="l00139"></a>00139 <span class="comment">-----+----+---------+---------+----------+-------------</span> |
| <a name="l00140"></a>00140 <span class="comment"> ...</span> |
| <a name="l00141"></a>00141 <span class="comment"></pre></span> |
| <a name="l00142"></a>00142 <span class="comment"></span> |
| <a name="l00143"></a>00143 <span class="comment">- Get vector of coefficients \f$ \boldsymbol c \f$:\n</span> |
| <a name="l00144"></a>00144 <span class="comment"> <pre>SELECT (madlib.\ref linregr(<em>dependentVariable</em>,</span> |
| <a name="l00145"></a>00145 <span class="comment"> <em>independentVariables</em>)).coef</span> |
| <a name="l00146"></a>00146 <span class="comment">FROM <em>sourceName</em>;</pre></span> |
| <a name="l00147"></a>00147 <span class="comment"></span> |
| <a name="l00148"></a>00148 <span class="comment">- Get a subset of the output columns, e.g., only the array of coefficients</span> |
| <a name="l00149"></a>00149 <span class="comment"> \f$ \boldsymbol c \f$, the coefficient of determination \f$ R^2 \f$, and</span> |
| <a name="l00150"></a>00150 <span class="comment"> the array of p-values \f$ \boldsymbol p \f$:</span> |
| <a name="l00151"></a>00151 <span class="comment"> <pre>SELECT (lr).coef, (lr).r2, (lr).p_values</span> |
| <a name="l00152"></a>00152 <span class="comment">FROM (</span> |
| <a name="l00153"></a>00153 <span class="comment"> SELECT madlib.\ref linregr(<em>dependentVariable</em>,</span> |
| <a name="l00154"></a>00154 <span class="comment"> <em>independentVariables</em>) AS lr</span> |
| <a name="l00155"></a>00155 <span class="comment"> FROM <em>sourceName</em></span> |
| <a name="l00156"></a>00156 <span class="comment">) AS subq;</pre></span> |
| <a name="l00157"></a>00157 <span class="comment"></span> |
| <a name="l00158"></a>00158 <span class="comment"><b>(2) The Full Interface</b></span> |
| <a name="l00159"></a>00159 <span class="comment"></span> |
| <a name="l00160"></a>00160 <span class="comment">The full interface support the analysis of heteroskedasticity of the linear fit.</span> |
| <a name="l00161"></a>00161 <span class="comment"></span> |
| <a name="l00162"></a>00162 <span class="comment"><pre></span> |
| <a name="l00163"></a>00163 <span class="comment">SELECT madlib.\ref linregr_train (</span> |
| <a name="l00164"></a>00164 <span class="comment"> <em>'source_table'</em>, -- name of input table, VARCHAR</span> |
| <a name="l00165"></a>00165 <span class="comment"> <em>'out_table'</em>, -- name of output table, VARCHAR</span> |
| <a name="l00166"></a>00166 <span class="comment"> <em>'dependent_varname'</em>, -- dependent variable, VARCHAR</span> |
| <a name="l00167"></a>00167 <span class="comment"> <em>'independent_varname'</em>, -- independent variable, VARCHAR</span> |
| <a name="l00168"></a>00168 <span class="comment"> [<em>group_cols</em>, -- names of columns to group by, VARCHAR[].</span> |
| <a name="l00169"></a>00169 <span class="comment"> -- Default value: Null</span> |
| <a name="l00170"></a>00170 <span class="comment"> [<em>heteroskedasticity_option</em>]] -- whether to analyze</span> |
| <a name="l00171"></a>00171 <span class="comment"> -- heteroskedasticity,</span> |
| <a name="l00172"></a>00172 <span class="comment"> -- BOOLEAN. Default value: False</span> |
| <a name="l00173"></a>00173 <span class="comment">);</span> |
| <a name="l00174"></a>00174 <span class="comment"></pre></span> |
| <a name="l00175"></a>00175 <span class="comment"></span> |
| <a name="l00176"></a>00176 <span class="comment">Here the <em>'independent_varname'</em> can be the name of a column, which contains</span> |
| <a name="l00177"></a>00177 <span class="comment">array of numeric values. It can also have a format of string 'array[1, x1, x2, x3]',</span> |
| <a name="l00178"></a>00178 <span class="comment">where <em>x1</em>, <em>x2</em> and <em>x3</em> are all column names.</span> |
| <a name="l00179"></a>00179 <span class="comment"></span> |
| <a name="l00180"></a>00180 <span class="comment">Output is stored in the <em>out_table</em>:</span> |
| <a name="l00181"></a>00181 <span class="comment"><pre></span> |
| <a name="l00182"></a>00182 <span class="comment">[ group_col_1 | group_col_2 | ... |] coef | r2 | std_err | t_stats | p_values | condition_no [|</span> |
| <a name="l00183"></a>00183 <span class="comment">-----------+-------------+-----+------+----+---------+---------+----------+--------------+---</span> |
| <a name="l00184"></a>00184 <span class="comment"></span> |
| <a name="l00185"></a>00185 <span class="comment">bp_stats | bp_p_value ]</span> |
| <a name="l00186"></a>00186 <span class="comment">-------------+---------</span> |
| <a name="l00187"></a>00187 <span class="comment"></pre></span> |
| <a name="l00188"></a>00188 <span class="comment"></span> |
| <a name="l00189"></a>00189 <span class="comment">Where the first part <pre>[ group_col_1 | group_col_2 | ... |]</pre> presents</span> |
| <a name="l00190"></a>00190 <span class="comment">only when <em>group_cols</em> is not Null. The last part <pre>[ bp_stats | ... |</span> |
| <a name="l00191"></a>00191 <span class="comment">corrected_p_values ]</pre> presents only when <em>heteroskedasticity_option</em></span> |
| <a name="l00192"></a>00192 <span class="comment">is <em>True</em>.</span> |
| <a name="l00193"></a>00193 <span class="comment"></span> |
| <a name="l00194"></a>00194 <span class="comment">When <em>group_cols</em> is given, the data is grouped by the given columns and</span> |
| <a name="l00195"></a>00195 <span class="comment">a linear model is fit to each group of data. The output will have additional</span> |
| <a name="l00196"></a>00196 <span class="comment">columns for all combinations of the values of all the <em>group_cols</em>. For</span> |
| <a name="l00197"></a>00197 <span class="comment">each combination of <em>group_cols</em> values, linear regression result is</span> |
| <a name="l00198"></a>00198 <span class="comment">shown.</span> |
| <a name="l00199"></a>00199 <span class="comment"></span> |
| <a name="l00200"></a>00200 <span class="comment">When <em>heteroskedasticity_option</em> is <em>True</em>, the output will have</span> |
| <a name="l00201"></a>00201 <span class="comment">additional columns. The function computes the Breusch–Pagan test [4] statistics</span> |
| <a name="l00202"></a>00202 <span class="comment">and the corresponding \f$p\f$-value.</span> |
| <a name="l00203"></a>00203 <span class="comment"></span> |
| <a name="l00204"></a>00204 <span class="comment">@examp</span> |
| <a name="l00205"></a>00205 <span class="comment"></span> |
| <a name="l00206"></a>00206 <span class="comment">The following example is taken from</span> |
| <a name="l00207"></a>00207 <span class="comment">http://www.stat.columbia.edu/~martin/W2110/SAS_7.pdf.</span> |
| <a name="l00208"></a>00208 <span class="comment"></span> |
| <a name="l00209"></a>00209 <span class="comment">-# Create the sample data set:</span> |
| <a name="l00210"></a>00210 <span class="comment">\verbatim</span> |
| <a name="l00211"></a>00211 <span class="comment">sql> CREATE TABLE houses (id INT, tax INT, bedroom INT, bath FLOAT, price INT,</span> |
| <a name="l00212"></a>00212 <span class="comment"> size INT, lot INT);</span> |
| <a name="l00213"></a>00213 <span class="comment">sql> COPY houses FROM STDIN WITH DELIMITER '|';</span> |
| <a name="l00214"></a>00214 <span class="comment"> 1 | 590 | 2 | 1 | 50000 | 770 | 22100</span> |
| <a name="l00215"></a>00215 <span class="comment"> 2 | 1050 | 3 | 2 | 85000 | 1410 | 12000</span> |
| <a name="l00216"></a>00216 <span class="comment"> 3 | 20 | 3 | 1 | 22500 | 1060 | 3500</span> |
| <a name="l00217"></a>00217 <span class="comment"> 4 | 870 | 2 | 2 | 90000 | 1300 | 17500</span> |
| <a name="l00218"></a>00218 <span class="comment"> 5 | 1320 | 3 | 2 | 133000 | 1500 | 30000</span> |
| <a name="l00219"></a>00219 <span class="comment"> 6 | 1350 | 2 | 1 | 90500 | 820 | 25700</span> |
| <a name="l00220"></a>00220 <span class="comment"> 7 | 2790 | 3 | 2.5 | 260000 | 2130 | 25000</span> |
| <a name="l00221"></a>00221 <span class="comment"> 8 | 680 | 2 | 1 | 142500 | 1170 | 22000</span> |
| <a name="l00222"></a>00222 <span class="comment"> 9 | 1840 | 3 | 2 | 160000 | 1500 | 19000</span> |
| <a name="l00223"></a>00223 <span class="comment"> 10 | 3680 | 4 | 2 | 240000 | 2790 | 20000</span> |
| <a name="l00224"></a>00224 <span class="comment"> 11 | 1660 | 3 | 1 | 87000 | 1030 | 17500</span> |
| <a name="l00225"></a>00225 <span class="comment"> 12 | 1620 | 3 | 2 | 118600 | 1250 | 20000</span> |
| <a name="l00226"></a>00226 <span class="comment"> 13 | 3100 | 3 | 2 | 140000 | 1760 | 38000</span> |
| <a name="l00227"></a>00227 <span class="comment"> 14 | 2070 | 2 | 3 | 148000 | 1550 | 14000</span> |
| <a name="l00228"></a>00228 <span class="comment"> 15 | 650 | 3 | 1.5 | 65000 | 1450 | 12000</span> |
| <a name="l00229"></a>00229 <span class="comment">\.</span> |
| <a name="l00230"></a>00230 <span class="comment">\endverbatim</span> |
| <a name="l00231"></a>00231 <span class="comment">-# You can call the linregr() function for an individual metric:</span> |
| <a name="l00232"></a>00232 <span class="comment">\verbatim</span> |
| <a name="l00233"></a>00233 <span class="comment">sql> SELECT (linregr(price, array[1, bedroom, bath, size])).coef FROM houses;</span> |
| <a name="l00234"></a>00234 <span class="comment"> coef</span> |
| <a name="l00235"></a>00235 <span class="comment">------------------------------------------------------------------------</span> |
| <a name="l00236"></a>00236 <span class="comment"> {27923.4334170641,-35524.7753390234,2269.34393735323,130.793920208133}</span> |
| <a name="l00237"></a>00237 <span class="comment">(1 row)</span> |
| <a name="l00238"></a>00238 <span class="comment"></span> |
| <a name="l00239"></a>00239 <span class="comment">sql> SELECT (linregr(price, array[1, bedroom, bath, size])).r2 FROM houses;</span> |
| <a name="l00240"></a>00240 <span class="comment"> r2</span> |
| <a name="l00241"></a>00241 <span class="comment">-------------------</span> |
| <a name="l00242"></a>00242 <span class="comment"> 0.745374010140315</span> |
| <a name="l00243"></a>00243 <span class="comment">(1 row)</span> |
| <a name="l00244"></a>00244 <span class="comment"></span> |
| <a name="l00245"></a>00245 <span class="comment">sql> SELECT (linregr(price, array[1, bedroom, bath, size])).std_err FROM houses;</span> |
| <a name="l00246"></a>00246 <span class="comment"> std_err</span> |
| <a name="l00247"></a>00247 <span class="comment">----------------------------------------------------------------------</span> |
| <a name="l00248"></a>00248 <span class="comment"> {56306.4821787474,25036.6537279169,22208.6687270562,36.208642285651}</span> |
| <a name="l00249"></a>00249 <span class="comment">(1 row)</span> |
| <a name="l00250"></a>00250 <span class="comment"></span> |
| <a name="l00251"></a>00251 <span class="comment">sql> SELECT (linregr(price, array[1, bedroom, bath, size])).t_stats FROM houses;</span> |
| <a name="l00252"></a>00252 <span class="comment"> t_stats</span> |
| <a name="l00253"></a>00253 <span class="comment">------------------------------------------------------------------------</span> |
| <a name="l00254"></a>00254 <span class="comment"> {0.495918628487924,-1.41891067892239,0.10218279921428,3.6122293450358}</span> |
| <a name="l00255"></a>00255 <span class="comment">(1 row)</span> |
| <a name="l00256"></a>00256 <span class="comment"></span> |
| <a name="l00257"></a>00257 <span class="comment">sql> SELECT (linregr(price, array[1, bedroom, bath, size])).p_values FROM houses;</span> |
| <a name="l00258"></a>00258 <span class="comment"> p_values</span> |
| <a name="l00259"></a>00259 <span class="comment">-----------------------------------------------------------------------------</span> |
| <a name="l00260"></a>00260 <span class="comment"> {0.629711069315512,0.183633155781461,0.920450514073051,0.00408159079312354}</span> |
| <a name="l00261"></a>00261 <span class="comment">(1 row)</span> |
| <a name="l00262"></a>00262 <span class="comment">\endverbatim</span> |
| <a name="l00263"></a>00263 <span class="comment">-# Alternatively you can call the linreg() function for the full record:</span> |
| <a name="l00264"></a>00264 <span class="comment">\verbatim</span> |
| <a name="l00265"></a>00265 <span class="comment">sql> \x on</span> |
| <a name="l00266"></a>00266 <span class="comment">Expanded display is on.</span> |
| <a name="l00267"></a>00267 <span class="comment">sql> SELECT (r).* FROM (SELECT linregr(price, array[1, bedroom, bath, size])</span> |
| <a name="l00268"></a>00268 <span class="comment"> AS r FROM houses) q;</span> |
| <a name="l00269"></a>00269 <span class="comment">-[ RECORD 1 ]+-----------------------------------------------------------------</span> |
| <a name="l00270"></a>00270 <span class="comment">coef | {27923.4334170641,-35524.7753390234,2269.34393735323,130.793920208133}</span> |
| <a name="l00271"></a>00271 <span class="comment">r2 | 0.745374010140315</span> |
| <a name="l00272"></a>00272 <span class="comment">std_err | {56306.4821787474,25036.6537279169,22208.6687270562,36.208642285651}</span> |
| <a name="l00273"></a>00273 <span class="comment">t_stats | {0.495918628487924,-1.41891067892239,0.10218279921428,3.6122293450358}</span> |
| <a name="l00274"></a>00274 <span class="comment">p_values | {0.629711069315512,0.183633155781461,0.920450514073051,0.00408159079312354}</span> |
| <a name="l00275"></a>00275 <span class="comment">condition_no | 9783.018</span> |
| <a name="l00276"></a>00276 <span class="comment"></span> |
| <a name="l00277"></a>00277 <span class="comment">\endverbatim</span> |
| <a name="l00278"></a>00278 <span class="comment"></span> |
| <a name="l00279"></a>00279 <span class="comment">-# You can call linregr_train() function for more functionality</span> |
| <a name="l00280"></a>00280 <span class="comment">\verbatim</span> |
| <a name="l00281"></a>00281 <span class="comment">sql> SELECT madlib.linregr_train('houses', 'result', 'price',</span> |
| <a name="l00282"></a>00282 <span class="comment"> 'array[1, tax, bath, size]',</span> |
| <a name="l00283"></a>00283 <span class="comment"> '{bedroom}'::varchar[], True);</span> |
| <a name="l00284"></a>00284 <span class="comment"></span> |
| <a name="l00285"></a>00285 <span class="comment">sql> SELECT * from result;</span> |
| <a name="l00286"></a>00286 <span class="comment">-[ RECORD 1]---------+-------------------------------------------------------</span> |
| <a name="l00287"></a>00287 <span class="comment">bedroom | 2</span> |
| <a name="l00288"></a>00288 <span class="comment">coef | {-84242.0345, 55.4430, -78966.9754, 225.6119}</span> |
| <a name="l00289"></a>00289 <span class="comment">r2 | 0.9688</span> |
| <a name="l00290"></a>00290 <span class="comment">std_err | {35019.00, 19.57, 23036.81, 49.04}</span> |
| <a name="l00291"></a>00291 <span class="comment">t_stats | {-2.406, 2.833, -3.428, 4.600}</span> |
| <a name="l00292"></a>00292 <span class="comment">p_values | {0.251, 0.216, 0.181, 0.136}</span> |
| <a name="l00293"></a>00293 <span class="comment">condition_no | 10086.1</span> |
| <a name="l00294"></a>00294 <span class="comment">bp_stats | 2.5451</span> |
| <a name="l00295"></a>00295 <span class="comment">bp_p_value | 0.4672</span> |
| <a name="l00296"></a>00296 <span class="comment"></span> |
| <a name="l00297"></a>00297 <span class="comment">-[ RECORD 2]---------+------------------------------------------------------</span> |
| <a name="l00298"></a>00298 <span class="comment">bedroom | 3</span> |
| <a name="l00299"></a>00299 <span class="comment">coef | {-88155.8292502747,27.1966436293179,41404.0293389239,62.6375210724027}</span> |
| <a name="l00300"></a>00300 <span class="comment">r2 | 0.841699901312963</span> |
| <a name="l00301"></a>00301 <span class="comment">std_err | {57867.9999699512,17.82723091538,43643.1321521931,70.8506824870639}</span> |
| <a name="l00302"></a>00302 <span class="comment">t_stats | {-1.52339512850022,1.52556747362568,0.948695185179172,0.884077878626493}</span> |
| <a name="l00303"></a>00303 <span class="comment">p_values | {0.18816143289241,0.187636685729725,0.38634003235866,0.417132778730133}</span> |
| <a name="l00304"></a>00304 <span class="comment">condition_no | 11722.62</span> |
| <a name="l00305"></a>00305 <span class="comment">bp_stats | 6.7538</span> |
| <a name="l00306"></a>00306 <span class="comment">bp_p_value | 0.08017</span> |
| <a name="l00307"></a>00307 <span class="comment"></span> |
| <a name="l00308"></a>00308 <span class="comment">-[ RECORD 3]---------+-------------------------------------------------------</span> |
| <a name="l00309"></a>00309 <span class="comment">bedroom | 4</span> |
| <a name="l00310"></a>00310 <span class="comment">coef | {0.0112536020318378,41.4132554771633,0.0225072040636757,31.3975496688276}</span> |
| <a name="l00311"></a>00311 <span class="comment">r2 | 1</span> |
| <a name="l00312"></a>00312 <span class="comment">std_err | {0,0,0,0}</span> |
| <a name="l00313"></a>00313 <span class="comment">t_stats | {Infinity,Infinity,Infinity,Infinity}</span> |
| <a name="l00314"></a>00314 <span class="comment">p_values | Null</span> |
| <a name="l00315"></a>00315 <span class="comment">condition_no | Null</span> |
| <a name="l00316"></a>00316 <span class="comment">bp_stats | Null</span> |
| <a name="l00317"></a>00317 <span class="comment">bp_p_value | Null</span> |
| <a name="l00318"></a>00318 <span class="comment"></span> |
| <a name="l00319"></a>00319 <span class="comment">\endverbatim</span> |
| <a name="l00320"></a>00320 <span class="comment"></span> |
| <a name="l00321"></a>00321 <span class="comment">@literature</span> |
| <a name="l00322"></a>00322 <span class="comment"></span> |
| <a name="l00323"></a>00323 <span class="comment">[1] Cosma Shalizi: Statistics 36-350: Data Mining, Lecture Notes, 21 October</span> |
| <a name="l00324"></a>00324 <span class="comment"> 2009, http://www.stat.cmu.edu/~cshalizi/350/lectures/17/lecture-17.pdf</span> |
| <a name="l00325"></a>00325 <span class="comment"></span> |
| <a name="l00326"></a>00326 <span class="comment">[2] Wikipedia: Condition Number, http://en.wikipedia.org/wiki/Condition_number.</span> |
| <a name="l00327"></a>00327 <span class="comment"></span> |
| <a name="l00328"></a>00328 <span class="comment">[3] Wikipedia: Spectral Norm,</span> |
| <a name="l00329"></a>00329 <span class="comment"> http://en.wikipedia.org/wiki/Spectral_norm#Spectral_norm</span> |
| <a name="l00330"></a>00330 <span class="comment"></span> |
| <a name="l00331"></a>00331 <span class="comment">[4] Wikipedia: Breusch–Pagan test,</span> |
| <a name="l00332"></a>00332 <span class="comment"> http://en.wikipedia.org/wiki/Breusch%E2%80%93Pagan_test</span> |
| <a name="l00333"></a>00333 <span class="comment"></span> |
| <a name="l00334"></a>00334 <span class="comment">[5] Wikipedia: Heteroscedasticity-consistent standard errors,</span> |
| <a name="l00335"></a>00335 <span class="comment">http://en.wikipedia.org/wiki/Heteroscedasticity-consistent_standard_errors</span> |
| <a name="l00336"></a>00336 <span class="comment"></span> |
| <a name="l00337"></a>00337 <span class="comment">@sa File linear.sql_in documenting the SQL functions.</span> |
| <a name="l00338"></a>00338 <span class="comment"></span> |
| <a name="l00339"></a>00339 <span class="comment">@internal</span> |
| <a name="l00340"></a>00340 <span class="comment">@sa Namespace \ref madlib::modules::regress</span> |
| <a name="l00341"></a>00341 <span class="comment"> documenting the implementation in C++</span> |
| <a name="l00342"></a>00342 <span class="comment">@endinternal</span> |
| <a name="l00343"></a>00343 <span class="comment">*/</span> |
| <a name="l00344"></a>00344 |
| <a name="l00345"></a>00345 --------------------------------------------------------------------------- |
| <a name="l00346"></a>00346 CREATE TYPE MADLIB_SCHEMA.linregr_result AS ( |
| <a name="l00347"></a>00347 coef DOUBLE PRECISION[], |
| <a name="l00348"></a>00348 r2 DOUBLE PRECISION, |
| <a name="l00349"></a>00349 std_err DOUBLE PRECISION[], |
| <a name="l00350"></a>00350 t_stats DOUBLE PRECISION[], |
| <a name="l00351"></a>00351 p_values DOUBLE PRECISION[], |
| <a name="l00352"></a>00352 condition_no DOUBLE PRECISION |
| <a name="l00353"></a>00353 ); |
| <a name="l00354"></a>00354 |
| <a name="l00355"></a>00355 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_transition( |
| <a name="l00356"></a>00356 state MADLIB_SCHEMA.bytea8, |
| <a name="l00357"></a>00357 y DOUBLE PRECISION, |
| <a name="l00358"></a>00358 x DOUBLE PRECISION[]) |
| <a name="l00359"></a>00359 RETURNS MADLIB_SCHEMA.bytea8 |
| <a name="l00360"></a>00360 AS <span class="stringliteral">'MODULE_PATHNAME'</span> |
| <a name="l00361"></a>00361 LANGUAGE C |
| <a name="l00362"></a>00362 IMMUTABLE STRICT; |
| <a name="l00363"></a>00363 |
| <a name="l00364"></a>00364 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_merge_states( |
| <a name="l00365"></a>00365 state1 MADLIB_SCHEMA.bytea8, |
| <a name="l00366"></a>00366 state2 MADLIB_SCHEMA.bytea8) |
| <a name="l00367"></a>00367 RETURNS MADLIB_SCHEMA.bytea8 |
| <a name="l00368"></a>00368 AS <span class="stringliteral">'MODULE_PATHNAME'</span> |
| <a name="l00369"></a>00369 LANGUAGE C |
| <a name="l00370"></a>00370 IMMUTABLE STRICT; |
| <a name="l00371"></a>00371 |
| <a name="l00372"></a>00372 -- Final functions |
| <a name="l00373"></a>00373 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_final( |
| <a name="l00374"></a>00374 state MADLIB_SCHEMA.bytea8) |
| <a name="l00375"></a>00375 RETURNS MADLIB_SCHEMA.linregr_result |
| <a name="l00376"></a>00376 AS <span class="stringliteral">'MODULE_PATHNAME'</span> |
| <a name="l00377"></a>00377 LANGUAGE C IMMUTABLE STRICT; |
| <a name="l00378"></a>00378 |
| <a name="l00379"></a>00379 --------------------------- HETEROSKEDASTICITY ---------------------------------- |
| <a name="l00380"></a>00380 CREATE TYPE MADLIB_SCHEMA.heteroskedasticity_test_result AS ( |
| <a name="l00381"></a>00381 bp_stats DOUBLE PRECISION, |
| <a name="l00382"></a>00382 bp_p_value DOUBLE PRECISION |
| <a name="l00383"></a>00383 ); |
| <a name="l00384"></a>00384 |
| <a name="l00385"></a>00385 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_transition( |
| <a name="l00386"></a>00386 state MADLIB_SCHEMA.bytea8, |
| <a name="l00387"></a>00387 y DOUBLE PRECISION, |
| <a name="l00388"></a>00388 x DOUBLE PRECISION[], |
| <a name="l00389"></a>00389 coef DOUBLE PRECISION[]) |
| <a name="l00390"></a>00390 RETURNS MADLIB_SCHEMA.bytea8 |
| <a name="l00391"></a>00391 AS <span class="stringliteral">'MODULE_PATHNAME'</span> |
| <a name="l00392"></a>00392 LANGUAGE C |
| <a name="l00393"></a>00393 IMMUTABLE STRICT; |
| <a name="l00394"></a>00394 |
| <a name="l00395"></a>00395 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_merge_states( |
| <a name="l00396"></a>00396 state1 MADLIB_SCHEMA.bytea8, |
| <a name="l00397"></a>00397 state2 MADLIB_SCHEMA.bytea8) |
| <a name="l00398"></a>00398 RETURNS MADLIB_SCHEMA.bytea8 |
| <a name="l00399"></a>00399 AS <span class="stringliteral">'MODULE_PATHNAME'</span> |
| <a name="l00400"></a>00400 LANGUAGE C |
| <a name="l00401"></a>00401 IMMUTABLE STRICT; |
| <a name="l00402"></a>00402 |
| <a name="l00403"></a>00403 -- Final functions |
| <a name="l00404"></a>00404 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_final( |
| <a name="l00405"></a>00405 state MADLIB_SCHEMA.bytea8) |
| <a name="l00406"></a>00406 RETURNS MADLIB_SCHEMA.heteroskedasticity_test_result |
| <a name="l00407"></a>00407 AS <span class="stringliteral">'MODULE_PATHNAME'</span> |
| <a name="l00408"></a>00408 LANGUAGE C IMMUTABLE STRICT; |
| <a name="l00409"></a>00409 <span class="comment"></span> |
| <a name="l00410"></a>00410 <span class="comment">/**</span> |
| <a name="l00411"></a>00411 <span class="comment"> * @brief Compute studentized Breuch-Pagan heteroskedasticity test for</span> |
| <a name="l00412"></a>00412 <span class="comment"> * linear regression.</span> |
| <a name="l00413"></a>00413 <span class="comment"> *</span> |
| <a name="l00414"></a>00414 <span class="comment"> * @param dependentVariable Column containing the dependent variable</span> |
| <a name="l00415"></a>00415 <span class="comment"> * @param independentVariables Column containing the array of independent variables</span> |
| <a name="l00416"></a>00416 <span class="comment"> * @param olsCoefficients Column containing the array of the OLS coefficients (as obtained by linregr)</span> |
| <a name="l00417"></a>00417 <span class="comment"> *</span> |
| <a name="l00418"></a>00418 <span class="comment"> * @par</span> |
| <a name="l00419"></a>00419 <span class="comment"> * To include an intercept in the model, set one coordinate in the</span> |
| <a name="l00420"></a>00420 <span class="comment"> * <tt>independentVariables</tt> array to 1.</span> |
| <a name="l00421"></a>00421 <span class="comment"> *</span> |
| <a name="l00422"></a>00422 <span class="comment"> * @return A composite value:</span> |
| <a name="l00423"></a>00423 <span class="comment"> * - <tt>test_statistic FLOAT8[]</tt> - Prob > test_statistc</span> |
| <a name="l00424"></a>00424 <span class="comment"> * - <tt>p_value FLOAT8[]</tt> - Prob > test_statistc</span> |
| <a name="l00425"></a>00425 <span class="comment"> *</span> |
| <a name="l00426"></a>00426 <span class="comment"> * @usage</span> |
| <a name="l00427"></a>00427 <span class="comment"> * <pre> SELECT (heteoskedasticity_test_linregr(<em>dependentVariable</em>,</span> |
| <a name="l00428"></a>00428 <span class="comment"> * <em>independentVariables</em>, coef)).*</span> |
| <a name="l00429"></a>00429 <span class="comment"> * FROM (</span> |
| <a name="l00430"></a>00430 <span class="comment"> * SELECT linregr(<em>dependentVariable</em>, <em>independentVariables</em>).coef</span> |
| <a name="l00431"></a>00431 <span class="comment"> * ) AS ols_coef, <em>sourceName</em> as src;</span> |
| <a name="l00432"></a>00432 <span class="comment"> * </pre></span> |
| <a name="l00433"></a>00433 <span class="comment"> */</span> |
| <a name="l00434"></a>00434 CREATE AGGREGATE MADLIB_SCHEMA.heteroskedasticity_test_linregr( |
| <a name="l00435"></a>00435 <span class="comment">/*+ "dependentVariable" */</span> DOUBLE PRECISION, |
| <a name="l00436"></a>00436 <span class="comment">/*+ "independentVariables" */</span> DOUBLE PRECISION[], |
| <a name="l00437"></a>00437 <span class="comment">/*+ "olsCoefficients" */</span> DOUBLE PRECISION[]) ( |
| <a name="l00438"></a>00438 |
| <a name="l00439"></a>00439 SFUNC=MADLIB_SCHEMA.hetero_linregr_transition, |
| <a name="l00440"></a>00440 STYPE=MADLIB_SCHEMA.bytea8, |
| <a name="l00441"></a>00441 FINALFUNC=MADLIB_SCHEMA.hetero_linregr_final, |
| <a name="l00442"></a>00442 m4_ifdef(`GREENPLUM<span class="stringliteral">',`prefunc=MADLIB_SCHEMA.hetero_linregr_merge_states,'</span>) |
| <a name="l00443"></a>00443 INITCOND='' |
| <a name="l00444"></a>00444 ); |
| <a name="l00445"></a>00445 |
| <a name="l00446"></a>00446 ---------------------------------------------------------------------------<span class="comment"></span> |
| <a name="l00447"></a>00447 <span class="comment">/**</span> |
| <a name="l00448"></a>00448 <span class="comment"> * @brief Compute linear regression coefficients and diagnostic statistics.</span> |
| <a name="l00449"></a>00449 <span class="comment"> *</span> |
| <a name="l00450"></a>00450 <span class="comment"> * @param dependentVariable Column containing the dependent variable</span> |
| <a name="l00451"></a>00451 <span class="comment"> * @param independentVariables Column containing the array of independent variables</span> |
| <a name="l00452"></a>00452 <span class="comment"> *</span> |
| <a name="l00453"></a>00453 <span class="comment"> * @par</span> |
| <a name="l00454"></a>00454 <span class="comment"> * To include an intercept in the model, set one coordinate in the</span> |
| <a name="l00455"></a>00455 <span class="comment"> * <tt>independentVariables</tt> array to 1.</span> |
| <a name="l00456"></a><a class="code" href="linear_8sql__in.html#a85d6d11ff2c93855061515d92d4785c2">00456</a> <span class="comment"> *</span> |
| <a name="l00457"></a>00457 <span class="comment"> * @return A composite value:</span> |
| <a name="l00458"></a>00458 <span class="comment"> * - <tt>coef FLOAT8[]</tt> - Array of coefficients, \f$ \boldsymbol c \f$</span> |
| <a name="l00459"></a>00459 <span class="comment"> * - <tt>r2 FLOAT8</tt> - Coefficient of determination, \f$ R^2 \f$</span> |
| <a name="l00460"></a>00460 <span class="comment"> * - <tt>std_err FLOAT8[]</tt> - Array of standard errors,</span> |
| <a name="l00461"></a>00461 <span class="comment"> * \f$ \mathit{se}(c_1), \dots, \mathit{se}(c_k) \f$</span> |
| <a name="l00462"></a>00462 <span class="comment"> * - <tt>t_stats FLOAT8[]</tt> - Array of t-statistics, \f$ \boldsymbol t \f$</span> |
| <a name="l00463"></a>00463 <span class="comment"> * - <tt>p_values FLOAT8[]</tt> - Array of p-values, \f$ \boldsymbol p \f$</span> |
| <a name="l00464"></a>00464 <span class="comment"> * - <tt>condition_no FLOAT8</tt> - The condition number of matrix</span> |
| <a name="l00465"></a>00465 <span class="comment"> * \f$ X^T X \f$.</span> |
| <a name="l00466"></a>00466 <span class="comment"> *</span> |
| <a name="l00467"></a>00467 <span class="comment"> * @usage</span> |
| <a name="l00468"></a>00468 <span class="comment"> * - Get vector of coefficients \f$ \boldsymbol c \f$ and all diagnostic</span> |
| <a name="l00469"></a>00469 <span class="comment"> * statistics:\n</span> |
| <a name="l00470"></a>00470 <span class="comment"> * <pre>SELECT (linregr(<em>dependentVariable</em>,</span> |
| <a name="l00471"></a>00471 <span class="comment"> * <em>independentVariables</em>)).*</span> |
| <a name="l00472"></a>00472 <span class="comment"> *FROM <em>sourceName</em>;</pre></span> |
| <a name="l00473"></a>00473 <span class="comment"> * - Get vector of coefficients \f$ \boldsymbol c \f$:\n</span> |
| <a name="l00474"></a>00474 <span class="comment"> * <pre>SELECT (linregr(<em>dependentVariable</em>,</span> |
| <a name="l00475"></a>00475 <span class="comment"> * <em>independentVariables</em>)).coef</span> |
| <a name="l00476"></a>00476 <span class="comment"> *FROM <em>sourceName</em>;</pre></span> |
| <a name="l00477"></a>00477 <span class="comment"> * - Get a subset of the output columns, e.g., only the array of coefficients</span> |
| <a name="l00478"></a>00478 <span class="comment"> * \f$ \boldsymbol c \f$, the coefficient of determination \f$ R^2 \f$, and</span> |
| <a name="l00479"></a>00479 <span class="comment"> * the array of p-values \f$ \boldsymbol p \f$:</span> |
| <a name="l00480"></a>00480 <span class="comment"> * <pre>SELECT (lr).coef, (lr).r2, (lr).p_values</span> |
| <a name="l00481"></a>00481 <span class="comment"> *FROM (</span> |
| <a name="l00482"></a>00482 <span class="comment"> * SELECT linregr( <em>dependentVariable</em>,</span> |
| <a name="l00483"></a>00483 <span class="comment"> * <em>independentVariables</em>) AS lr</span> |
| <a name="l00484"></a>00484 <span class="comment"> * FROM <em>sourceName</em></span> |
| <a name="l00485"></a>00485 <span class="comment"> *) AS subq;</pre></span> |
| <a name="l00486"></a>00486 <span class="comment"> */</span> |
| <a name="l00487"></a>00487 |
| <a name="l00488"></a>00488 CREATE AGGREGATE MADLIB_SCHEMA.<a class="code" href="linear_8sql__in.html#a71d8295a18e93619b3331cefabe6e79b" title="Compute linear regression coefficients and diagnostic statistics.">linregr</a>( |
| <a name="l00489"></a>00489 <span class="comment">/*+ "dependentVariable" */</span> DOUBLE PRECISION, |
| <a name="l00490"></a>00490 <span class="comment">/*+ "independentVariables" */</span> DOUBLE PRECISION[]) ( |
| <a name="l00491"></a>00491 |
| <a name="l00492"></a>00492 SFUNC=MADLIB_SCHEMA.linregr_transition, |
| <a name="l00493"></a>00493 STYPE=MADLIB_SCHEMA.bytea8, |
| <a name="l00494"></a>00494 FINALFUNC=MADLIB_SCHEMA.linregr_final, |
| <a name="l00495"></a>00495 m4_ifdef(`__GREENPLUM__',`prefunc=MADLIB_SCHEMA.linregr_merge_states,') |
| <a name="l00496"></a>00496 INITCOND='' |
| <a name="l00497"></a>00497 ); |
| <a name="l00498"></a>00498 --------------------------- INTERNAL ---------------------------------------<span class="comment"></span> |
| <a name="l00499"></a>00499 <span class="comment">/**</span> |
| <a name="l00500"></a>00500 <span class="comment"> * @brief Return heteroskedasticity values for specific linear regression</span> |
| <a name="l00501"></a>00501 <span class="comment"> * coefficients</span> |
| <a name="l00502"></a>00502 <span class="comment">**/</span> |
| <a name="l00503"></a>00503 CREATE FUNCTION MADLIB_SCHEMA.__internal_get_hsk_result( |
| <a name="l00504"></a>00504 source_table VARCHAR -- name of input table |
| <a name="l00505"></a>00505 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00506"></a>00506 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00507"></a>00507 , linregr_coeffs DOUBLE PRECISION[] -- coeffs from linear regression |
| <a name="l00508"></a>00508 ) |
| <a name="l00509"></a>00509 RETURNS MADLIB_SCHEMA.heteroskedasticity_test_result AS $$ |
| <a name="l00510"></a><a class="code" href="linear_8sql__in.html#a71d8295a18e93619b3331cefabe6e79b">00510</a> DECLARE |
| <a name="l00511"></a>00511 hsk_value MADLIB_SCHEMA.heteroskedasticity_test_result; |
| <a name="l00512"></a>00512 BEGIN |
| <a name="l00513"></a>00513 EXECUTE ' |
| <a name="l00514"></a>00514 SELECT (MADLIB_SCHEMA.<a class="code" href="linear_8sql__in.html#a85d6d11ff2c93855061515d92d4785c2" title="Compute studentized Breuch-Pagan heteroskedasticity test for linear regression.">heteroskedasticity_test_linregr</a>(' |
| <a name="l00515"></a>00515 || dependent_varname || ' , ' |
| <a name="l00516"></a>00516 || independent_varname || ' , ' |
| <a name="l00517"></a>00517 || 'ARRAY[' || array_to_string(linregr_coeffs, ',') || '])).* |
| <a name="l00518"></a>00518 FROM ' || source_table |
| <a name="l00519"></a>00519 INTO hsk_value; |
| <a name="l00520"></a>00520 |
| <a name="l00521"></a>00521 RETURN hsk_value; |
| <a name="l00522"></a>00522 END |
| <a name="l00523"></a>00523 $$ LANGUAGE plpgsql VOLATILE;<span class="comment"></span> |
| <a name="l00524"></a>00524 <span class="comment">/**</span> |
| <a name="l00525"></a>00525 <span class="comment"> * @brief Return linear regression output for source data</span> |
| <a name="l00526"></a>00526 <span class="comment"> *</span> |
| <a name="l00527"></a>00527 <span class="comment">**/</span> |
| <a name="l00528"></a>00528 CREATE FUNCTION MADLIB_SCHEMA.__internal_get_linreg_result( |
| <a name="l00529"></a>00529 source_table VARCHAR -- name of input table |
| <a name="l00530"></a>00530 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00531"></a>00531 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00532"></a>00532 ) |
| <a name="l00533"></a>00533 RETURNS MADLIB_SCHEMA.linregr_result AS $$ |
| <a name="l00534"></a>00534 DECLARE |
| <a name="l00535"></a>00535 lin_rst MADLIB_SCHEMA.linregr_result; |
| <a name="l00536"></a>00536 BEGIN |
| <a name="l00537"></a>00537 EXECUTE ' |
| <a name="l00538"></a>00538 SELECT (MADLIB_SCHEMA.<a class="code" href="linear_8sql__in.html#a71d8295a18e93619b3331cefabe6e79b" title="Compute linear regression coefficients and diagnostic statistics.">linregr</a>( ' |
| <a name="l00539"></a>00539 || dependent_varname || ' , ' |
| <a name="l00540"></a>00540 || independent_varname || ') |
| <a name="l00541"></a>00541 ).* |
| <a name="l00542"></a>00542 FROM ' || source_table |
| <a name="l00543"></a>00543 INTO lin_rst; |
| <a name="l00544"></a>00544 RETURN lin_rst; |
| <a name="l00545"></a>00545 END |
| <a name="l00546"></a>00546 $$ LANGUAGE plpgsql VOLATILE; |
| <a name="l00547"></a>00547 |
| <a name="l00548"></a>00548 CREATE FUNCTION MADLIB_SCHEMA.__internal_get_linregr_insert_string( |
| <a name="l00549"></a>00549 lin_rst MADLIB_SCHEMA.linregr_result, |
| <a name="l00550"></a>00550 out_table TEXT |
| <a name="l00551"></a>00551 ) |
| <a name="l00552"></a>00552 RETURNS VARCHAR AS $$ |
| <a name="l00553"></a>00553 DECLARE |
| <a name="l00554"></a>00554 insert_string VARCHAR; |
| <a name="l00555"></a>00555 BEGIN |
| <a name="l00556"></a>00556 insert_string := 'INSERT INTO ' || out_table || ' VALUES ('; |
| <a name="l00557"></a>00557 insert_string := insert_string || |
| <a name="l00558"></a>00558 CASE |
| <a name="l00559"></a>00559 WHEN (lin_rst).coef is NULL |
| <a name="l00560"></a>00560 THEN '''{}<span class="stringliteral">''</span>,<span class="stringliteral">'</span> |
| <a name="l00561"></a>00561 <span class="stringliteral"> ELSE '</span>ARRAY[<span class="stringliteral">' || array_to_string((lin_rst).coef, '</span>,<span class="stringliteral">') || '</span>], <span class="stringliteral">'</span> |
| <a name="l00562"></a>00562 <span class="stringliteral"> END || </span> |
| <a name="l00563"></a>00563 <span class="stringliteral"> CASE </span> |
| <a name="l00564"></a>00564 <span class="stringliteral"> WHEN (lin_rst).r2 is NULL</span> |
| <a name="l00565"></a>00565 <span class="stringliteral"> THEN '</span>0.0,<span class="stringliteral">'</span> |
| <a name="l00566"></a>00566 <span class="stringliteral"> ELSE (lin_rst).r2 || '</span>,<span class="stringliteral">' </span> |
| <a name="l00567"></a>00567 <span class="stringliteral"> END ||</span> |
| <a name="l00568"></a>00568 <span class="stringliteral"> CASE </span> |
| <a name="l00569"></a>00569 <span class="stringliteral"> WHEN (lin_rst).std_err is NULL</span> |
| <a name="l00570"></a>00570 <span class="stringliteral"> THEN '</span><span class="stringliteral">''</span>{}<span class="stringliteral">''</span>,<span class="stringliteral">'</span> |
| <a name="l00571"></a>00571 <span class="stringliteral"> ELSE '</span>ARRAY[<span class="stringliteral">' || array_to_string((lin_rst).std_err, '</span>,<span class="stringliteral">') || '</span>], <span class="stringliteral">'</span> |
| <a name="l00572"></a>00572 <span class="stringliteral"> END ||</span> |
| <a name="l00573"></a>00573 <span class="stringliteral"> CASE </span> |
| <a name="l00574"></a>00574 <span class="stringliteral"> WHEN (lin_rst).t_stats is NULL</span> |
| <a name="l00575"></a>00575 <span class="stringliteral"> THEN '</span><span class="stringliteral">''</span>{}<span class="stringliteral">''</span>,<span class="stringliteral">'</span> |
| <a name="l00576"></a>00576 <span class="stringliteral"> ELSE '</span>ARRAY[<span class="stringliteral">' || array_to_string((lin_rst).t_stats, '</span>,<span class="stringliteral">') || '</span>], <span class="stringliteral">'</span> |
| <a name="l00577"></a>00577 <span class="stringliteral"> END ||</span> |
| <a name="l00578"></a>00578 <span class="stringliteral"> CASE </span> |
| <a name="l00579"></a>00579 <span class="stringliteral"> WHEN (lin_rst).p_values is NULL</span> |
| <a name="l00580"></a>00580 <span class="stringliteral"> THEN '</span><span class="stringliteral">''</span>{}<span class="stringliteral">''</span>,<span class="stringliteral">'</span> |
| <a name="l00581"></a>00581 <span class="stringliteral"> ELSE '</span>ARRAY[<span class="stringliteral">' || array_to_string((lin_rst).p_values, '</span>,<span class="stringliteral">') || '</span>], <span class="stringliteral">'</span> |
| <a name="l00582"></a>00582 <span class="stringliteral"> END ||</span> |
| <a name="l00583"></a>00583 <span class="stringliteral"> CASE </span> |
| <a name="l00584"></a>00584 <span class="stringliteral"> WHEN (lin_rst).condition_no is NULL</span> |
| <a name="l00585"></a>00585 <span class="stringliteral"> THEN '</span>0.0<span class="stringliteral">'</span> |
| <a name="l00586"></a>00586 <span class="stringliteral"> ELSE (lin_rst).condition_no</span> |
| <a name="l00587"></a>00587 <span class="stringliteral"> END;</span> |
| <a name="l00588"></a>00588 <span class="stringliteral"> RETURN insert_string; </span> |
| <a name="l00589"></a>00589 <span class="stringliteral">END;</span> |
| <a name="l00590"></a>00590 <span class="stringliteral">$$ LANGUAGE plpgsql VOLATILE;</span> |
| <a name="l00591"></a>00591 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00592"></a>00592 <span class="comment">/**</span> |
| <a name="l00593"></a>00593 <span class="comment"> * @brief Compute linear regression coefficients and heterskedasticity values</span> |
| <a name="l00594"></a>00594 <span class="comment"> *</span> |
| <a name="l00595"></a>00595 <span class="comment"> **/</span> |
| <a name="l00596"></a>00596 CREATE FUNCTION MADLIB_SCHEMA.__internal_linregr_train_hetero( |
| <a name="l00597"></a>00597 source_table VARCHAR -- name of input table |
| <a name="l00598"></a>00598 , out_table VARCHAR -- name of output table |
| <a name="l00599"></a>00599 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00600"></a>00600 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00601"></a>00601 , heteroskedasticity_option BOOLEAN -- do you want heteroskedasticity output |
| <a name="l00602"></a>00602 ) |
| <a name="l00603"></a>00603 RETURNS VOID AS $$ |
| <a name="l00604"></a>00604 DECLARE |
| <a name="l00605"></a>00605 insert_string VARCHAR; |
| <a name="l00606"></a>00606 lin_rst MADLIB_SCHEMA.linregr_result; |
| <a name="l00607"></a>00607 hsk_value MADLIB_SCHEMA.heteroskedasticity_test_result; |
| <a name="l00608"></a>00608 BEGIN |
| <a name="l00609"></a>00609 IF (source_table IS NULL OR source_table = '<span class="stringliteral">') THEN</span> |
| <a name="l00610"></a>00610 <span class="stringliteral"> RAISE EXCEPTION '</span>Invalid input table name given.<span class="stringliteral">';</span> |
| <a name="l00611"></a>00611 <span class="stringliteral"> END IF;</span> |
| <a name="l00612"></a>00612 <span class="stringliteral"> IF (out_table IS NULL OR out_table = '</span><span class="stringliteral">') THEN</span> |
| <a name="l00613"></a>00613 <span class="stringliteral"> RAISE EXCEPTION '</span>Invalid output table name given.<span class="stringliteral">';</span> |
| <a name="l00614"></a>00614 <span class="stringliteral"> END IF;</span> |
| <a name="l00615"></a>00615 <span class="stringliteral"> IF (dependent_varname IS NULL OR dependent_varname = '</span><span class="stringliteral">') THEN</span> |
| <a name="l00616"></a>00616 <span class="stringliteral"> RAISE EXCEPTION '</span>Invalid dependent variable name given.<span class="stringliteral">';</span> |
| <a name="l00617"></a>00617 <span class="stringliteral"> END IF;</span> |
| <a name="l00618"></a>00618 <span class="stringliteral"> IF (independent_varname IS NULL OR independent_varname = '</span><span class="stringliteral">') THEN</span> |
| <a name="l00619"></a>00619 <span class="stringliteral"> RAISE EXCEPTION '</span>Invalid independent variable name given.<span class="stringliteral">';</span> |
| <a name="l00620"></a>00620 <span class="stringliteral"> END IF;</span> |
| <a name="l00621"></a>00621 <span class="stringliteral"> -- create output table with appropriate column names</span> |
| <a name="l00622"></a>00622 <span class="stringliteral"> EXECUTE '</span>DROP TABLE IF EXISTS <span class="stringliteral">' || out_table;</span> |
| <a name="l00623"></a>00623 <span class="stringliteral"> EXECUTE '</span> |
| <a name="l00624"></a>00624 CREATE TABLE <span class="stringliteral">' || out_table || '</span> ( |
| <a name="l00625"></a>00625 coef DOUBLE PRECISION[], |
| <a name="l00626"></a>00626 r2 DOUBLE PRECISION, |
| <a name="l00627"></a>00627 std_err DOUBLE PRECISION[], |
| <a name="l00628"></a>00628 t_stats DOUBLE PRECISION[], |
| <a name="l00629"></a>00629 p_values DOUBLE PRECISION[], |
| <a name="l00630"></a>00630 condition_no DOUBLE PRECISION)<span class="stringliteral">';</span> |
| <a name="l00631"></a>00631 <span class="stringliteral"> IF heteroskedasticity_option THEN</span> |
| <a name="l00632"></a>00632 <span class="stringliteral"> -- Alter output table to add heteroskedasticity values</span> |
| <a name="l00633"></a>00633 <span class="stringliteral"> EXECUTE '</span> |
| <a name="l00634"></a>00634 ALTER TABLE <span class="stringliteral">' || out_table || '</span> |
| <a name="l00635"></a>00635 ADD COLUMN bp_stats DOUBLE PRECISION, |
| <a name="l00636"></a>00636 ADD COLUMN bp_p_value DOUBLE PRECISION<span class="stringliteral">';</span> |
| <a name="l00637"></a>00637 <span class="stringliteral"> END IF;</span> |
| <a name="l00638"></a>00638 <span class="stringliteral"> -- compute linear regression and heteroskedasticity values (if required)</span> |
| <a name="l00639"></a>00639 <span class="stringliteral"> lin_rst := MADLIB_SCHEMA.__internal_get_linreg_result(</span> |
| <a name="l00640"></a>00640 <span class="stringliteral"> source_table, dependent_varname, independent_varname);</span> |
| <a name="l00641"></a>00641 <span class="stringliteral"> insert_string := MADLIB_SCHEMA.__internal_get_linregr_insert_string(</span> |
| <a name="l00642"></a>00642 <span class="stringliteral"> lin_rst, out_table);</span> |
| <a name="l00643"></a>00643 <span class="stringliteral"> -- Ensure Infinity and NaN are cast properly</span> |
| <a name="l00644"></a>00644 <span class="stringliteral"> insert_string := REGEXP_REPLACE(insert_string, '</span>Infinity<span class="stringliteral">', </span> |
| <a name="l00645"></a>00645 <span class="stringliteral"> '</span><span class="stringliteral">''</span>Infinity<span class="stringliteral">''</span>::<span class="keywordtype">double</span> precision<span class="stringliteral">', '</span>gi<span class="stringliteral">');</span> |
| <a name="l00646"></a>00646 <span class="stringliteral"> insert_string := REGEXP_REPLACE(insert_string, '</span>NaN<span class="stringliteral">', </span> |
| <a name="l00647"></a>00647 <span class="stringliteral"> '</span><span class="stringliteral">''</span>NaN<span class="stringliteral">''</span>::<span class="keywordtype">double</span> precision<span class="stringliteral">', '</span>gi<span class="stringliteral">');</span> |
| <a name="l00648"></a>00648 <span class="stringliteral"> IF heteroskedasticity_option THEN</span> |
| <a name="l00649"></a>00649 <span class="stringliteral"> -- add hsk values in the sql string and execute</span> |
| <a name="l00650"></a>00650 <span class="stringliteral"> hsk_value := MADLIB_SCHEMA.__internal_get_hsk_result(</span> |
| <a name="l00651"></a>00651 <span class="stringliteral"> source_table, dependent_varname,</span> |
| <a name="l00652"></a>00652 <span class="stringliteral"> independent_varname, (lin_rst).coef);</span> |
| <a name="l00653"></a>00653 <span class="stringliteral"> EXECUTE</span> |
| <a name="l00654"></a>00654 <span class="stringliteral"> insert_string || '</span>,<span class="stringliteral">'</span> |
| <a name="l00655"></a>00655 <span class="stringliteral"> || (hsk_value).bp_stats || '</span>,<span class="stringliteral">'</span> |
| <a name="l00656"></a>00656 <span class="stringliteral"> || (hsk_value).bp_p_value || '</span>)<span class="stringliteral">';</span> |
| <a name="l00657"></a>00657 <span class="stringliteral"> ELSE</span> |
| <a name="l00658"></a>00658 <span class="stringliteral"> -- complete the sql string and execute</span> |
| <a name="l00659"></a>00659 <span class="stringliteral"> EXECUTE insert_string || '</span>)<span class="stringliteral">';</span> |
| <a name="l00660"></a>00660 <span class="stringliteral"> END IF;</span> |
| <a name="l00661"></a>00661 <span class="stringliteral">END</span> |
| <a name="l00662"></a>00662 <span class="stringliteral">$$ LANGUAGE plpgsql VOLATILE;</span> |
| <a name="l00663"></a>00663 <span class="stringliteral">---------------------------------------------------------------------------</span><span class="comment"></span> |
| <a name="l00664"></a>00664 <span class="comment">/**</span> |
| <a name="l00665"></a>00665 <span class="comment"> * @brief Compute linear regression coefficients and insert into an output</span> |
| <a name="l00666"></a>00666 <span class="comment"> * table. The function drops the table if it already exists before creating</span> |
| <a name="l00667"></a>00667 <span class="comment"> * the table.</span> |
| <a name="l00668"></a>00668 <span class="comment"> *</span> |
| <a name="l00669"></a>00669 <span class="comment">**/</span> |
| <a name="l00670"></a>00670 CREATE FUNCTION MADLIB_SCHEMA.linregr_train( |
| <a name="l00671"></a>00671 source_table VARCHAR -- name of input table |
| <a name="l00672"></a>00672 , out_table VARCHAR -- name of output table |
| <a name="l00673"></a>00673 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00674"></a>00674 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00675"></a>00675 ) |
| <a name="l00676"></a>00676 RETURNS VOID AS $$ |
| <a name="l00677"></a>00677 BEGIN |
| <a name="l00678"></a>00678 PERFORM MADLIB_SCHEMA.__internal_linregr_train_hetero( |
| <a name="l00679"></a>00679 source_table, out_table, dependent_varname, independent_varname, False); |
| <a name="l00680"></a>00680 -- RAISE NOTICE ' |
| <a name="l00681"></a>00681 -- Finished linear regression |
| <a name="l00682"></a>00682 -- * table : % (%, %) |
| <a name="l00683"></a>00683 -- Output: |
| <a name="l00684"></a>00684 -- * view : SELECT * FROM % ;<span class="stringliteral">', source_table, dependent_varname,</span> |
| <a name="l00685"></a>00685 <span class="stringliteral"> -- independent_varname, out_table;</span> |
| <a name="l00686"></a>00686 <span class="stringliteral">END;</span> |
| <a name="l00687"></a>00687 <span class="stringliteral">$$ LANGUAGE plpgsql VOLATILE;</span><span class="comment"></span> |
| <a name="l00688"></a>00688 <span class="comment">/**</span> |
| <a name="l00689"></a>00689 <span class="comment"> * @brief Compute linear regression coefficients and insert into an output</span> |
| <a name="l00690"></a>00690 <span class="comment"> * table. The function drops the table if it already exists before creating</span> |
| <a name="l00691"></a>00691 <span class="comment"> * the table. Heteroskedasticity values are also added if the option is True.</span> |
| <a name="l00692"></a>00692 <span class="comment">**/</span> |
| <a name="l00693"></a>00693 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_train( |
| <a name="l00694"></a>00694 source_table VARCHAR -- name of input table |
| <a name="l00695"></a>00695 , out_table VARCHAR -- name of output table |
| <a name="l00696"></a>00696 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00697"></a>00697 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00698"></a>00698 , heteroskedasticity_option BOOLEAN -- heteroskedasticity |
| <a name="l00699"></a>00699 ) |
| <a name="l00700"></a>00700 RETURNS VOID AS $$ |
| <a name="l00701"></a>00701 DECLARE |
| <a name="l00702"></a>00702 BEGIN |
| <a name="l00703"></a>00703 PERFORM MADLIB_SCHEMA.__internal_linregr_train_hetero( |
| <a name="l00704"></a>00704 source_table, out_table, dependent_varname, independent_varname, |
| <a name="l00705"></a>00705 heteroskedasticity_option); |
| <a name="l00706"></a>00706 -- RAISE NOTICE ' |
| <a name="l00707"></a>00707 -- Finished linear regression |
| <a name="l00708"></a>00708 -- * table : % (%, %) |
| <a name="l00709"></a>00709 -- Output: |
| <a name="l00710"></a>00710 -- * view : SELECT * FROM % ;<span class="stringliteral">', source_table, dependent_varname,</span> |
| <a name="l00711"></a>00711 <span class="stringliteral"> -- independent_varname, out_table;</span> |
| <a name="l00712"></a>00712 <span class="stringliteral">END;</span> |
| <a name="l00713"></a>00713 <span class="stringliteral">$$ LANGUAGE plpgsql VOLATILE;</span> |
| <a name="l00714"></a>00714 <span class="stringliteral"></span> |
| <a name="l00715"></a>00715 <span class="stringliteral">--------------------- GROUPING ---------------------------------------------</span><span class="comment"></span> |
| <a name="l00716"></a>00716 <span class="comment">/**</span> |
| <a name="l00717"></a>00717 <span class="comment"> * @brief Linear regression training function with grouping support and</span> |
| <a name="l00718"></a>00718 <span class="comment"> * option for heteroskedasticity values.</span> |
| <a name="l00719"></a>00719 <span class="comment"> **/</span> |
| <a name="l00720"></a>00720 CREATE FUNCTION MADLIB_SCHEMA.linregr_train( |
| <a name="l00721"></a>00721 source_table VARCHAR -- name of input table |
| <a name="l00722"></a>00722 , out_table VARCHAR -- name of output table |
| <a name="l00723"></a>00723 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00724"></a>00724 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00725"></a>00725 , group_cols VARCHAR[] -- names of columns to group-by |
| <a name="l00726"></a>00726 , heteroskedasticity_option BOOLEAN -- heteroskedasticity |
| <a name="l00727"></a>00727 ) |
| <a name="l00728"></a>00728 RETURNS VOID AS $$ |
| <a name="l00729"></a>00729 DECLARE |
| <a name="l00730"></a>00730 input_table_name VARCHAR[]; |
| <a name="l00731"></a>00731 actual_table_name VARCHAR; |
| <a name="l00732"></a>00732 schema_name VARCHAR; |
| <a name="l00733"></a>00733 table_creation_string VARCHAR; |
| <a name="l00734"></a>00734 group_string VARCHAR; |
| <a name="l00735"></a>00735 group_array_length INTEGER; |
| <a name="l00736"></a>00736 col_data_type VARCHAR; |
| <a name="l00737"></a>00737 each_group INTEGER; |
| <a name="l00738"></a>00738 linregr_fitting_rst VARCHAR; |
| <a name="l00739"></a>00739 old_msg_level TEXT; |
| <a name="l00740"></a>00740 BEGIN |
| <a name="l00741"></a>00741 |
| <a name="l00742"></a>00742 EXECUTE 'SELECT setting FROM pg_settings WHERE name=<span class="stringliteral">''</span>client_min_messages<span class="charliteral">'''</span> INTO old_msg_level; |
| <a name="l00743"></a>00743 EXECUTE <span class="stringliteral">'SET client_min_messages TO warning'</span>; |
| <a name="l00744"></a>00744 |
| <a name="l00745"></a>00745 -- initial validation |
| <a name="l00746"></a>00746 IF (group_cols IS NULL |
| <a name="l00747"></a>00747 OR array_upper(group_cols, 1) IS NULL |
| <a name="l00748"></a>00748 OR array_upper(group_cols, 1) = 0) |
| <a name="l00749"></a>00749 THEN |
| <a name="l00750"></a>00750 PERFORM MADLIB_SCHEMA.linregr_train( |
| <a name="l00751"></a>00751 source_table, out_table, |
| <a name="l00752"></a>00752 dependent_varname, independent_varname, |
| <a name="l00753"></a>00753 heteroskedasticity_option); |
| <a name="l00754"></a>00754 ELSE |
| <a name="l00755"></a>00755 -- create output table |
| <a name="l00756"></a>00756 EXECUTE 'DROP TABLE IF EXISTS ' || out_table; |
| <a name="l00757"></a>00757 table_creation_string := 'CREATE TABLE ' || out_table || '('; |
| <a name="l00758"></a>00758 group_array_length = array_upper(group_cols, 1); |
| <a name="l00759"></a>00759 |
| <a name="l00760"></a>00760 input_table_name = regexp_split_to_array(source_table, E'\\.'); |
| <a name="l00761"></a>00761 IF array_upper(input_table_name, 1) = 1 THEN |
| <a name="l00762"></a>00762 actual_table_name = input_table_name[1]; |
| <a name="l00763"></a>00763 schema_name := current_schema(); |
| <a name="l00764"></a>00764 ELSIF array_upper(input_table_name, 1) = 2 THEN |
| <a name="l00765"></a>00765 actual_table_name = input_table_name[2]; |
| <a name="l00766"></a>00766 schema_name = input_table_name[1]; |
| <a name="l00767"></a>00767 ELSE |
| <a name="l00768"></a>00768 RAISE EXCEPTION 'Incorrect input source table name provided'; |
| <a name="l00769"></a>00769 END IF; |
| <a name="l00770"></a>00770 |
| <a name="l00771"></a>00771 -- TODO: Check if the column is actually a part of the dataset |
| <a name="l00772"></a>00772 FOR each_group in 1 .. group_array_length |
| <a name="l00773"></a>00773 LOOP |
| <a name="l00774"></a>00774 -- create a <span class="keywordtype">string</span> that makes list of |
| <a name="l00775"></a>00775 EXECUTE 'SELECT data_type FROM information_schema.columns |
| <a name="l00776"></a>00776 WHERE |
| <a name="l00777"></a>00777 table_schema = ''' || schema_name || ''' |
| <a name="l00778"></a>00778 AND table_name = ''' || actual_table_name || ''' |
| <a name="l00779"></a>00779 AND column_name= ''' || group_cols[each_group] || '''' |
| <a name="l00780"></a>00780 INTO col_data_type; |
| <a name="l00781"></a>00781 |
| <a name="l00782"></a>00782 table_creation_string := table_creation_string |
| <a name="l00783"></a>00783 || group_cols[each_group] |
| <a name="l00784"></a>00784 || ' ' || col_data_type || ','; |
| <a name="l00785"></a>00785 END LOOP; |
| <a name="l00786"></a>00786 |
| <a name="l00787"></a>00787 -- finish creating the output table |
| <a name="l00788"></a>00788 EXECUTE table_creation_string || ' |
| <a name="l00789"></a>00789 coef DOUBLE PRECISION[], |
| <a name="l00790"></a>00790 r2 DOUBLE PRECISION, |
| <a name="l00791"></a>00791 std_err DOUBLE PRECISION[], |
| <a name="l00792"></a>00792 t_stats DOUBLE PRECISION[], |
| <a name="l00793"></a>00793 p_values DOUBLE PRECISION[], |
| <a name="l00794"></a>00794 condition_no DOUBLE PRECISION)'; |
| <a name="l00795"></a>00795 IF heteroskedasticity_option THEN |
| <a name="l00796"></a>00796 EXECUTE 'ALTER TABLE ' || out_table || ' |
| <a name="l00797"></a>00797 ADD COLUMN bp_stats DOUBLE PRECISION, |
| <a name="l00798"></a>00798 ADD COLUMN bp_p_value DOUBLE PRECISION'; |
| <a name="l00799"></a>00799 END IF; |
| <a name="l00800"></a>00800 |
| <a name="l00801"></a>00801 group_string := ''; |
| <a name="l00802"></a>00802 FOR each_group in 1 .. (group_array_length-1) |
| <a name="l00803"></a>00803 LOOP |
| <a name="l00804"></a>00804 group_string := group_string || group_cols[each_group] || ','; |
| <a name="l00805"></a>00805 END LOOP; |
| <a name="l00806"></a>00806 group_string := group_string || group_cols[group_array_length]; |
| <a name="l00807"></a>00807 |
| <a name="l00808"></a>00808 IF heteroskedasticity_option THEN |
| <a name="l00809"></a>00809 linregr_fitting_rst := MADLIB_SCHEMA.<a class="code" href="utilities_8sql__in.html#a5b0156044142673183205154f9e63ddc" title="Generate random remporary names for temp table and other names.">__unique_string</a>(); |
| <a name="l00810"></a>00810 EXECUTE ' |
| <a name="l00811"></a>00811 DROP TABLE IF EXISTS '|| linregr_fitting_rst ||'; |
| <a name="l00812"></a>00812 CREATE TEMP TABLE '|| linregr_fitting_rst ||' AS |
| <a name="l00813"></a>00813 SELECT |
| <a name="l00814"></a>00814 '|| group_string ||', |
| <a name="l00815"></a>00815 (MADLIB_SCHEMA.<a class="code" href="linear_8sql__in.html#a71d8295a18e93619b3331cefabe6e79b" title="Compute linear regression coefficients and diagnostic statistics.">linregr</a>('|| dependent_varname ||','|| independent_varname ||')).* |
| <a name="l00816"></a>00816 FROM '|| source_table ||' |
| <a name="l00817"></a>00817 GROUP BY '|| group_string; |
| <a name="l00818"></a>00818 |
| <a name="l00819"></a>00819 EXECUTE ' |
| <a name="l00820"></a>00820 INSERT INTO ' || out_table || ' |
| <a name="l00821"></a>00821 SELECT * |
| <a name="l00822"></a>00822 FROM |
| <a name="l00823"></a>00823 '|| linregr_fitting_rst ||' |
| <a name="l00824"></a>00824 JOIN ( |
| <a name="l00825"></a>00825 SELECT |
| <a name="l00826"></a>00826 '|| group_string ||', |
| <a name="l00827"></a>00827 (MADLIB_SCHEMA.<a class="code" href="linear_8sql__in.html#a85d6d11ff2c93855061515d92d4785c2" title="Compute studentized Breuch-Pagan heteroskedasticity test for linear regression.">heteroskedasticity_test_linregr</a>(' |
| <a name="l00828"></a>00828 || dependent_varname || ',' |
| <a name="l00829"></a>00829 || independent_varname || ', t.coef)).* |
| <a name="l00830"></a>00830 FROM |
| <a name="l00831"></a>00831 '|| source_table ||' AS s |
| <a name="l00832"></a>00832 JOIN |
| <a name="l00833"></a>00833 '|| linregr_fitting_rst ||' AS t |
| <a name="l00834"></a>00834 USING (' || group_string || ') |
| <a name="l00835"></a>00835 GROUP BY ' || group_string ||') z |
| <a name="l00836"></a>00836 USING ('|| group_string ||')'; |
| <a name="l00837"></a>00837 |
| <a name="l00838"></a>00838 EXECUTE 'DROP TABLE IF EXISTS '|| linregr_fitting_rst; |
| <a name="l00839"></a>00839 ELSE |
| <a name="l00840"></a>00840 EXECUTE ' |
| <a name="l00841"></a>00841 INSERT INTO ' || out_table || ' |
| <a name="l00842"></a>00842 SELECT ' || group_string || ', (result).coef, (result).r2, |
| <a name="l00843"></a>00843 (result).std_err, (result).t_stats, |
| <a name="l00844"></a>00844 (result).p_values, (result).condition_no |
| <a name="l00845"></a>00845 FROM ( |
| <a name="l00846"></a>00846 SELECT ' || group_string || |
| <a name="l00847"></a>00847 ', MADLIB_SCHEMA.<a class="code" href="linear_8sql__in.html#a71d8295a18e93619b3331cefabe6e79b" title="Compute linear regression coefficients and diagnostic statistics.">linregr</a>( ' |
| <a name="l00848"></a>00848 || dependent_varname || ' , ' |
| <a name="l00849"></a>00849 || independent_varname || ' ) |
| <a name="l00850"></a>00850 AS result |
| <a name="l00851"></a>00851 FROM ' || source_table || ' |
| <a name="l00852"></a>00852 GROUP BY ' || group_string || |
| <a name="l00853"></a>00853 ') subq'; |
| <a name="l00854"></a>00854 END IF; |
| <a name="l00855"></a>00855 END IF; |
| <a name="l00856"></a>00856 |
| <a name="l00857"></a>00857 EXECUTE 'SET client_min_messages TO '|| old_msg_level; |
| <a name="l00858"></a>00858 END; |
| <a name="l00859"></a>00859 $$ LANGUAGE plpgsql VOLATILE; |
| <a name="l00860"></a>00860 ---------------------------------------------------------------------------<span class="comment"></span> |
| <a name="l00861"></a>00861 <span class="comment">/**</span> |
| <a name="l00862"></a>00862 <span class="comment"> * @brief Linear regression training function with grouping support.</span> |
| <a name="l00863"></a>00863 <span class="comment"> **/</span> |
| <a name="l00864"></a>00864 CREATE FUNCTION MADLIB_SCHEMA.linregr_train( |
| <a name="l00865"></a>00865 source_table VARCHAR -- name of input table |
| <a name="l00866"></a>00866 , out_table VARCHAR -- name of output table |
| <a name="l00867"></a>00867 , dependent_varname VARCHAR -- name of dependent variable |
| <a name="l00868"></a>00868 , independent_varname VARCHAR -- name of independent variable |
| <a name="l00869"></a>00869 , group_cols VARCHAR[] -- names of columns to group-by |
| <a name="l00870"></a>00870 ) |
| <a name="l00871"></a>00871 RETURNS VOID AS $$ |
| <a name="l00872"></a>00872 BEGIN |
| <a name="l00873"></a>00873 PERFORM MADLIB_SCHEMA.linregr_train( source_table, out_table, |
| <a name="l00874"></a>00874 dependent_varname, |
| <a name="l00875"></a>00875 independent_varname, group_cols, FALSE); |
| <a name="l00876"></a>00876 END; |
| <a name="l00877"></a>00877 $$ LANGUAGE plpgsql VOLATILE; |
| <a name="l00878"></a>00878 --------------------------------------------------------------------------- |
| </pre></div></div> |
| </div> |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="linear_8sql__in.html">linear.sql_in</a> </li> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Functions</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| |
| <li class="footer">Generated on Tue Apr 2 2013 14:57:03 for MADlib by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.5.1 </li> |
| </ul> |
| </div> |
| |
| |
| </body> |
| </html> |