| <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
| <html xmlns="http://www.w3.org/1999/xhtml"> |
| <head> |
| <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> |
| <title>MADlib: crf_feature_gen.sql_in Source File</title> |
| |
| <link href="tabs.css" rel="stylesheet" type="text/css"/> |
| <link href="doxygen.css" rel="stylesheet" type="text/css" /> |
| <link href="navtree.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="jquery.js"></script> |
| <script type="text/javascript" src="resize.js"></script> |
| <script type="text/javascript" src="navtree.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(initResizable); |
| </script> |
| <link href="search/search.css" rel="stylesheet" type="text/css"/> |
| <script type="text/javascript" src="search/search.js"></script> |
| <script type="text/javascript"> |
| $(document).ready(function() { searchBox.OnSelectItem(0); }); |
| </script> |
| <script src="../mathjax/MathJax.js"> |
| MathJax.Hub.Config({ |
| extensions: ["tex2jax.js", "TeX/AMSmath.js", "TeX/AMSsymbols.js"], |
| jax: ["input/TeX","output/HTML-CSS"], |
| }); |
| </script> |
| </head> |
| <body> |
| <div id="top"><!-- do not remove this div! --> |
| |
| |
| <div id="titlearea"> |
| <table cellspacing="0" cellpadding="0"> |
| <tbody> |
| <tr style="height: 56px;"> |
| |
| |
| <td style="padding-left: 0.5em;"> |
| <div id="projectname">MADlib |
|  <span id="projectnumber">0.6</span> <span style="font-size:10pt; font-style:italic"><a href="../latest/./crf__feature__gen_8sql__in_source.html"> A newer version is available</a></span> |
| </div> |
| <div id="projectbrief">User Documentation</div> |
| </td> |
| |
| |
| |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| |
| <!-- Generated by Doxygen 1.7.5.1 --> |
| <script type="text/javascript"> |
| var searchBox = new SearchBox("searchBox", "search",false,'Search'); |
| </script> |
| <script type="text/javascript" src="dynsections.js"></script> |
| <div id="navrow1" class="tabs"> |
| <ul class="tablist"> |
| <li><a href="index.html"><span>Main Page</span></a></li> |
| <li><a href="modules.html"><span>Modules</span></a></li> |
| <li class="current"><a href="files.html"><span>Files</span></a></li> |
| <li> |
| <div id="MSearchBox" class="MSearchBoxInactive"> |
| <span class="left"> |
| <img id="MSearchSelect" src="search/mag_sel.png" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| alt=""/> |
| <input type="text" id="MSearchField" value="Search" accesskey="S" |
| onfocus="searchBox.OnSearchFieldFocus(true)" |
| onblur="searchBox.OnSearchFieldFocus(false)" |
| onkeyup="searchBox.OnSearchFieldChange(event)"/> |
| </span><span class="right"> |
| <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a> |
| </span> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div id="navrow2" class="tabs2"> |
| <ul class="tablist"> |
| <li><a href="files.html"><span>File List</span></a></li> |
| <li><a href="globals.html"><span>File Members</span></a></li> |
| </ul> |
| </div> |
| </div> |
| <div id="side-nav" class="ui-resizable side-nav-resizable"> |
| <div id="nav-tree"> |
| <div id="nav-tree-contents"> |
| </div> |
| </div> |
| <div id="splitbar" style="-moz-user-select:none;" |
| class="ui-resizable-handle"> |
| </div> |
| </div> |
| <script type="text/javascript"> |
| initNavTree('crf__feature__gen_8sql__in.html',''); |
| </script> |
| <div id="doc-content"> |
| <div class="header"> |
| <div class="headertitle"> |
| <div class="title">crf_feature_gen.sql_in</div> </div> |
| </div> |
| <div class="contents"> |
| <a href="crf__feature__gen_8sql__in.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">/* ----------------------------------------------------------------------- */</span><span class="comment">/**</span> |
| <a name="l00002"></a>00002 <span class="comment"> *</span> |
| <a name="l00003"></a>00003 <span class="comment"> * @file crf_feature_gen.sql_in </span> |
| <a name="l00004"></a>00004 <span class="comment"> *</span> |
| <a name="l00005"></a>00005 <span class="comment"> * @brief SQL function for POS/NER feature extraction</span> |
| <a name="l00006"></a>00006 <span class="comment"> * @date February 2012</span> |
| <a name="l00007"></a>00007 <span class="comment"> *</span> |
| <a name="l00008"></a>00008 <span class="comment"> * @sa For an introduction to POS/NER feature extraction, see the module</span> |
| <a name="l00009"></a>00009 <span class="comment"> * description \ref grp_crf</span> |
| <a name="l00010"></a>00010 <span class="comment"> */</span><span class="comment">/* ----------------------------------------------------------------------- */</span> |
| <a name="l00011"></a>00011 |
| <a name="l00012"></a>00012 m4_include(`SQLCommon.m4<span class="stringliteral">')</span> |
| <a name="l00013"></a>00013 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00014"></a>00014 <span class="comment">/**</span> |
| <a name="l00015"></a>00015 <span class="comment"> * @brief This function extracts POS/NER features from the training data.</span> |
| <a name="l00016"></a>00016 <span class="comment"> * </span> |
| <a name="l00017"></a>00017 <span class="comment"> * @param segmenttbl Name of table containing all the tokenized training sentences. </span> |
| <a name="l00018"></a>00018 <span class="comment"> * @param regextbl Name of table containing all the regular expressions to capture regex features.</span> |
| <a name="l00019"></a>00019 <span class="comment"> * @param dictionary Name of table containing the dictionary.</span> |
| <a name="l00020"></a>00020 <span class="comment"> * @param featuretbl features generated from the traning dataset</span> |
| <a name="l00021"></a>00021 <span class="comment"> * @param featureset unique featrue set generated from the training dataset </span> |
| <a name="l00022"></a>00022 <span class="comment"> *</span> |
| <a name="l00023"></a>00023 <span class="comment"> */</span> |
| <a name="l00024"></a>00024 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.crf_train_fgen( |
| <a name="l00025"></a>00025 segmenttbl text, |
| <a name="l00026"></a>00026 regextbl text, |
| <a name="l00027"></a>00027 dictionary text, |
| <a name="l00028"></a>00028 featuretbl text, |
| <a name="l00029"></a>00029 featureset text) RETURNS void AS |
| <a name="l00030"></a>00030 $$ |
| <a name="l00031"></a>00031 origClientMinMessages = plpy.execute("SELECT setting AS setting FROM pg_settings WHERE name = \'client_min_messages\';") |
| <a name="l00032"></a>00032 plpy.execute("SET client_min_messages TO warning;") |
| <a name="l00033"></a>00033 |
| <a name="l00034"></a>00034 plpy.execute("SELECT MADLIB_SCHEMA.create_schema_pg_temp();"); |
| <a name="l00035"></a>00035 tmp1_feature = "pg_temp._madlib_tmp1_feature" |
| <a name="l00036"></a>00036 tmp_rtbl = "pg_temp._madlib_tmp_rtbl" |
| <a name="l00037"></a>00037 tmp_dense_mtbl = "pg_temp._madlib_tmp_dense_mtbl" |
| <a name="l00038"></a>00038 dense_mtbl = "pg_temp._madlib_dense_mtbl" |
| <a name="l00039"></a>00039 sparse_rtbl = "pg_temp._madlib_sparse_rtbl" |
| <a name="l00040"></a>00040 sparse_mtbl = "pg_temp._madlib_sparse_mtbl" |
| <a name="l00041"></a>00041 tmp_featureset = "pg_temp._madlib_tmp_featureset" |
| <a name="l00042"></a>00042 tmp_segmenttbl = "pg_temp._madlib_tmp_segmenttbl" |
| <a name="l00043"></a>00043 |
| <a name="l00044"></a>00044 |
| <a name="l00045"></a>00045 plpy.execute("""DROP TABLE IF EXISTS """ + tmp1_feature + """,""" + tmp_rtbl + """,""" + tmp_dense_mtbl + """,""" + dense_mtbl + """,""" + sparse_rtbl + """,""" + sparse_mtbl + """,""" + tmp_featureset + """,""" + tmp_segmenttbl + """;""") |
| <a name="l00046"></a><a class="code" href="crf__feature__gen_8sql__in.html#a80e9192613662ba6dfd3ac90057205ee">00046</a> |
| <a name="l00047"></a>00047 plpy.execute("""CREATE TABLE """ + tmp1_feature + """(start_pos integer,doc_id integer, f_name text, feature integer[]);""") |
| <a name="l00048"></a>00048 plpy.execute("""CREATE TABLE """ + tmp_rtbl + """(start_pos integer,doc_id integer, feature integer[]);""") |
| <a name="l00049"></a>00049 plpy.execute("""CREATE TABLE """ + tmp_dense_mtbl + """(start_pos integer,doc_id integer, feature integer[]);""") |
| <a name="l00050"></a>00050 plpy.execute("""CREATE TABLE """ + dense_mtbl + """(doc_id integer, dense_m integer[]);""") |
| <a name="l00051"></a>00051 plpy.execute("""CREATE TABLE """ + sparse_rtbl + """(doc_id integer,f_size integer, sparse_r integer[]);""") |
| <a name="l00052"></a>00052 plpy.execute("""CREATE TABLE """ + sparse_mtbl + """(sparse_m integer[]);""") |
| <a name="l00053"></a>00053 plpy.execute("""CREATE TABLE """ + tmp_featureset + """(f_name text, feature integer[]);""") |
| <a name="l00054"></a>00054 plpy.execute("""CREATE TABLE """ + tmp_segmenttbl + """(start_pos int,doc_id int,seg_text text,label int,max_pos int);""") |
| <a name="l00055"></a>00055 |
| <a name="l00056"></a>00056 plpy.execute("""SET client_min_messages TO """ + str(origClientMinMessages[0]['setting<span class="stringliteral">']) + """;""")</span> |
| <a name="l00057"></a>00057 <span class="stringliteral"></span> |
| <a name="l00058"></a>00058 <span class="stringliteral"> # replace digits with "DIGIT" keyword</span> |
| <a name="l00059"></a>00059 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_segmenttbl + """ SELECT start_pos,doc_id,seg_text,label,max_pos FROM """ + segmenttbl + """ WHERE </span> |
| <a name="l00060"></a>00060 <span class="stringliteral"> NOT (seg_text ~ E'</span>^[-+]?([0-9]{1,3}[,]?)*[0-9]{1,3}$<span class="stringliteral">' OR seg_text ~ E'</span>^[-+]?[0-9]*[.][0-9]+$<span class="stringliteral">');""")</span> |
| <a name="l00061"></a>00061 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_segmenttbl + """ SELECT start_pos,doc_id,'</span>DIGIT<span class="stringliteral">',label,max_pos FROM """ + segmenttbl + """ WHERE </span> |
| <a name="l00062"></a>00062 <span class="stringliteral"> seg_text ~ E'</span>^[-+]?([0-9]{1,3}[,]?)*[0-9]{1,3}$<span class="stringliteral">' OR seg_text ~E'</span>^[-+]?[0-9]*[.][0-9]+$<span class="stringliteral">';""") </span> |
| <a name="l00063"></a>00063 <span class="stringliteral"></span> |
| <a name="l00064"></a>00064 <span class="stringliteral"> # insert into dictionary table</span> |
| <a name="l00065"></a>00065 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + dictionary + """(token, total)</span> |
| <a name="l00066"></a>00066 <span class="stringliteral"> SELECT seg_text, count(*)</span> |
| <a name="l00067"></a>00067 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """ </span> |
| <a name="l00068"></a>00068 <span class="stringliteral"> GROUP BY seg_text;""")</span> |
| <a name="l00069"></a>00069 <span class="stringliteral"> </span> |
| <a name="l00070"></a>00070 <span class="stringliteral"> # create a temporary table to store all the features</span> |
| <a name="l00071"></a>00071 <span class="stringliteral"> </span> |
| <a name="l00072"></a>00072 <span class="stringliteral"> # extract all the edge features</span> |
| <a name="l00073"></a>00073 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp1_feature + """(start_pos, doc_id, f_name, feature) </span> |
| <a name="l00074"></a>00074 <span class="stringliteral"> SELECT doc2.start_pos, doc2.doc_id, '</span>E.<span class="stringliteral">', ARRAY[doc1.label, doc2.label]</span> |
| <a name="l00075"></a>00075 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """ doc1, """ + tmp_segmenttbl + """ doc2</span> |
| <a name="l00076"></a>00076 <span class="stringliteral"> WHERE doc1.doc_id = doc2.doc_id AND doc1.start_pos+1 = doc2.start_pos;""")</span> |
| <a name="l00077"></a>00077 <span class="stringliteral"> </span> |
| <a name="l00078"></a>00078 <span class="stringliteral"> #extract all the regex features</span> |
| <a name="l00079"></a>00079 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp1_feature + """(start_pos, doc_id, f_name, feature) </span> |
| <a name="l00080"></a>00080 <span class="stringliteral"> SELECT start_pos, doc_id, '</span>R_<span class="stringliteral">' || name, ARRAY[-1, label]</span> |
| <a name="l00081"></a>00081 <span class="stringliteral"> FROM """ + regextbl + """, """ + tmp_segmenttbl + """ </span> |
| <a name="l00082"></a>00082 <span class="stringliteral"> WHERE seg_text ~ pattern;""")</span> |
| <a name="l00083"></a>00083 <span class="stringliteral"> </span> |
| <a name="l00084"></a>00084 <span class="stringliteral"> #extract all the start feature</span> |
| <a name="l00085"></a>00085 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp1_feature + """(start_pos, doc_id, f_name, feature) </span> |
| <a name="l00086"></a>00086 <span class="stringliteral"> SELECT start_pos, doc_id, '</span>S.<span class="stringliteral">', ARRAY[-1, label]</span> |
| <a name="l00087"></a>00087 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """ </span> |
| <a name="l00088"></a>00088 <span class="stringliteral"> WHERE start_pos = 0;""")</span> |
| <a name="l00089"></a>00089 <span class="stringliteral"> </span> |
| <a name="l00090"></a>00090 <span class="stringliteral"> #extract all the end featue</span> |
| <a name="l00091"></a>00091 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp1_feature + """(start_pos, doc_id, f_name, feature) </span> |
| <a name="l00092"></a>00092 <span class="stringliteral"> SELECT start_pos, doc_id, '</span>End.<span class="stringliteral">', ARRAY[-1, label]</span> |
| <a name="l00093"></a>00093 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """ </span> |
| <a name="l00094"></a>00094 <span class="stringliteral"> WHERE start_pos = max_pos;""")</span> |
| <a name="l00095"></a>00095 <span class="stringliteral"></span> |
| <a name="l00096"></a>00096 <span class="stringliteral"> #word feature</span> |
| <a name="l00097"></a>00097 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp1_feature + """(start_pos, doc_id, f_name, feature) </span> |
| <a name="l00098"></a>00098 <span class="stringliteral"> SELECT start_pos, doc_id, '</span>W_<span class="stringliteral">' || seg_text, ARRAY[-1, label]</span> |
| <a name="l00099"></a>00099 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """;""")</span> |
| <a name="l00100"></a>00100 <span class="stringliteral"> </span> |
| <a name="l00101"></a>00101 <span class="stringliteral"> #unknown feature</span> |
| <a name="l00102"></a>00102 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp1_feature + """(start_pos, doc_id, f_name, feature) </span> |
| <a name="l00103"></a>00103 <span class="stringliteral"> SELECT start_pos, doc_id, '</span>U<span class="stringliteral">', ARRAY[-1, label]</span> |
| <a name="l00104"></a>00104 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """ seg, """ + dictionary + """ dic</span> |
| <a name="l00105"></a>00105 <span class="stringliteral"> WHERE seg.seg_text = dic.token AND dic.total <= 1;""")</span> |
| <a name="l00106"></a>00106 <span class="stringliteral"> </span> |
| <a name="l00107"></a>00107 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_featureset + """(f_name, feature) </span> |
| <a name="l00108"></a>00108 <span class="stringliteral"> SELECT DISTINCT f_name, feature</span> |
| <a name="l00109"></a>00109 <span class="stringliteral"> FROM """ + tmp1_feature + """;""")</span> |
| <a name="l00110"></a>00110 <span class="stringliteral"></span> |
| <a name="l00111"></a>00111 <span class="stringliteral"> plpy.execute("""DROP SEQUENCE IF EXISTS seq;</span> |
| <a name="l00112"></a>00112 <span class="stringliteral"> CREATE SEQUENCE seq START 1 INCREMENT 1;""")</span> |
| <a name="l00113"></a>00113 <span class="stringliteral"></span> |
| <a name="l00114"></a>00114 <span class="stringliteral"> #get all distcint features</span> |
| <a name="l00115"></a>00115 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + featureset + """(f_index, f_name, feature) </span> |
| <a name="l00116"></a>00116 <span class="stringliteral"> SELECT nextval('</span>seq<span class="stringliteral">')-1, f_name, feature</span> |
| <a name="l00117"></a>00117 <span class="stringliteral"> FROM """ + tmp_featureset + """;""")</span> |
| <a name="l00118"></a>00118 <span class="stringliteral"> </span> |
| <a name="l00119"></a>00119 <span class="stringliteral"> rv = plpy.execute("""SELECT COUNT(*) AS total_feature FROM """ + featureset + """;""")</span> |
| <a name="l00120"></a>00120 <span class="stringliteral"></span> |
| <a name="l00121"></a>00121 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_rtbl + """(start_pos,doc_id,feature)</span> |
| <a name="l00122"></a>00122 <span class="stringliteral"> SELECT start_pos, doc_id, array_cat(fset.feature, </span> |
| <a name="l00123"></a>00123 <span class="stringliteral"> ARRAY[f_index,start_pos, </span> |
| <a name="l00124"></a>00124 <span class="stringliteral"> CASE WHEN """ + tmp1_feature + """.feature = fset.feature THEN 1</span> |
| <a name="l00125"></a>00125 <span class="stringliteral"> ELSE 0</span> |
| <a name="l00126"></a>00126 <span class="stringliteral"> END] )</span> |
| <a name="l00127"></a>00127 <span class="stringliteral"> FROM """ + tmp1_feature + """, """ + featureset + """ fset</span> |
| <a name="l00128"></a>00128 <span class="stringliteral"> WHERE """ + tmp1_feature + """.f_name = fset.f_name AND fset.f_name <> '</span>E.<span class="stringliteral">';""")</span> |
| <a name="l00129"></a>00129 <span class="stringliteral"></span> |
| <a name="l00130"></a>00130 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + sparse_rtbl + """(doc_id, f_size, sparse_r)</span> |
| <a name="l00131"></a>00131 <span class="stringliteral"> SELECT doc_id,""" + str(rv[0]['</span>total_feature<span class="stringliteral">']) + """, </span> |
| <a name="l00132"></a>00132 <span class="stringliteral"> MADLIB_SCHEMA.array_union(feature::integer[] order by start_pos)</span> |
| <a name="l00133"></a>00133 <span class="stringliteral"> FROM """ + tmp_rtbl + """</span> |
| <a name="l00134"></a>00134 <span class="stringliteral"> GROUP BY doc_id;""")</span> |
| <a name="l00135"></a>00135 <span class="stringliteral"></span> |
| <a name="l00136"></a>00136 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_dense_mtbl + """(start_pos,doc_id,feature)</span> |
| <a name="l00137"></a>00137 <span class="stringliteral"> SELECT start_pos, doc_id, </span> |
| <a name="l00138"></a>00138 <span class="stringliteral"> array_cat(fset.feature, ARRAY[f_index,start_pos,1])</span> |
| <a name="l00139"></a>00139 <span class="stringliteral"> FROM """ + tmp1_feature + """, """ + featureset + """ fset</span> |
| <a name="l00140"></a>00140 <span class="stringliteral"> WHERE start_pos > 0 AND """ + tmp1_feature + """.f_name = fset.f_name AND """ + tmp1_feature + """.feature = fset.feature AND fset.f_name = '</span>E.<span class="stringliteral">';""")</span> |
| <a name="l00141"></a>00141 <span class="stringliteral"></span> |
| <a name="l00142"></a>00142 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + dense_mtbl + """(doc_id, dense_m)</span> |
| <a name="l00143"></a>00143 <span class="stringliteral"> SELECT doc_id, MADLIB_SCHEMA.array_union(feature::integer[] order by start_pos)</span> |
| <a name="l00144"></a>00144 <span class="stringliteral"> FROM """ + tmp_dense_mtbl + """</span> |
| <a name="l00145"></a>00145 <span class="stringliteral"> GROUP BY doc_id;""")</span> |
| <a name="l00146"></a>00146 <span class="stringliteral"></span> |
| <a name="l00147"></a>00147 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + sparse_mtbl + """(sparse_m)</span> |
| <a name="l00148"></a>00148 <span class="stringliteral"> SELECT MADLIB_SCHEMA.array_union(array_cat(ARRAY[f_index],feature))</span> |
| <a name="l00149"></a>00149 <span class="stringliteral"> FROM """ + featureset + """ fset</span> |
| <a name="l00150"></a>00150 <span class="stringliteral"> WHERE f_name = '</span>E.<span class="stringliteral">';""")</span> |
| <a name="l00151"></a>00151 <span class="stringliteral"></span> |
| <a name="l00152"></a>00152 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + featuretbl + """(doc_id, f_size, sparse_r, dense_m, sparse_m)</span> |
| <a name="l00153"></a>00153 <span class="stringliteral"> SELECT """ + sparse_rtbl + """.doc_id, f_size, sparse_r, dense_m, sparse_m </span> |
| <a name="l00154"></a>00154 <span class="stringliteral"> FROM """ + sparse_rtbl + """, """ + dense_mtbl + """, """ + sparse_mtbl + """</span> |
| <a name="l00155"></a>00155 <span class="stringliteral"> WHERE """ + sparse_rtbl + """.doc_id = """ + dense_mtbl + """.doc_id;""")</span> |
| <a name="l00156"></a>00156 <span class="stringliteral"></span> |
| <a name="l00157"></a>00157 <span class="stringliteral">$$ LANGUAGE plpythonu STRICT; </span> |
| <a name="l00158"></a>00158 <span class="stringliteral"></span> |
| <a name="l00159"></a>00159 <span class="stringliteral"></span><span class="comment"></span> |
| <a name="l00160"></a>00160 <span class="comment">/**</span> |
| <a name="l00161"></a>00161 <span class="comment"> * @brief This function extracts POS/NER features from the testing data.</span> |
| <a name="l00162"></a>00162 <span class="comment"> *</span> |
| <a name="l00163"></a>00163 <span class="comment"> * This feature extraction function will produce two factor tables, "m table"</span> |
| <a name="l00164"></a>00164 <span class="comment"> * (\a viterbi_mtbl) and "r table" (\a viterbi_rtbl). The \a viterbi_mtbl</span> |
| <a name="l00165"></a>00165 <span class="comment"> * table and \a viterbi_rtbl table are used to calculate the best label</span> |
| <a name="l00166"></a>00166 <span class="comment"> * sequence for each sentence.</span> |
| <a name="l00167"></a>00167 <span class="comment"> *</span> |
| <a name="l00168"></a>00168 <span class="comment"> * - <em>viterbi_mtbl</em> table</span> |
| <a name="l00169"></a>00169 <span class="comment"> * encodes the edge features which are solely dependent on upon current label and</span> |
| <a name="l00170"></a>00170 <span class="comment"> * previous y value. The m table has three columns which are prev_label, label,</span> |
| <a name="l00171"></a>00171 <span class="comment"> * and value respectively.</span> |
| <a name="l00172"></a>00172 <span class="comment"> * If the number of labels in \f$ n \f$, then the m factor table will \f$ n^2 \f$</span> |
| <a name="l00173"></a>00173 <span class="comment"> * rows. Each row encodes the transition feature weight value from the previous label</span> |
| <a name="l00174"></a>00174 <span class="comment"> * to the current label.</span> |
| <a name="l00175"></a>00175 <span class="comment"> *</span> |
| <a name="l00176"></a>00176 <span class="comment"> * \a startFeature is considered as a special edge feature which is from the</span> |
| <a name="l00177"></a>00177 <span class="comment"> * beginning to the first token. Likewise, \a endFeature can be considered</span> |
| <a name="l00178"></a>00178 <span class="comment"> * as a special edge feature which is from the last token to the very end.</span> |
| <a name="l00179"></a>00179 <span class="comment"> * So m table encodes the edgeFeature, startFeature, and endFeature.</span> |
| <a name="l00180"></a>00180 <span class="comment"> * If the total number of labels in the label space is 45 from 0 to 44,</span> |
| <a name="l00181"></a>00181 <span class="comment"> * then the m factor array is as follows:</span> |
| <a name="l00182"></a>00182 <span class="comment"> * <pre></span> |
| <a name="l00183"></a>00183 <span class="comment"> * 0 1 2 3 4 5...44</span> |
| <a name="l00184"></a>00184 <span class="comment"> * startFeature -1 a a a a a a...a</span> |
| <a name="l00185"></a>00185 <span class="comment"> * edgeFeature 0 a a a a a a...a</span> |
| <a name="l00186"></a>00186 <span class="comment"> * edgeFeature 1 a a a a a a...a</span> |
| <a name="l00187"></a>00187 <span class="comment"> * ...</span> |
| <a name="l00188"></a>00188 <span class="comment"> * edgeFeature 44 a a a a a a...a</span> |
| <a name="l00189"></a>00189 <span class="comment"> * endFeature 45 a a a a a a...a</pre></span> |
| <a name="l00190"></a>00190 <span class="comment"> *</span> |
| <a name="l00191"></a>00191 <span class="comment"> * - viterbi_r table</span> |
| <a name="l00192"></a>00192 <span class="comment"> * is related to specific tokens. It encodes the single state features,</span> |
| <a name="l00193"></a>00193 <span class="comment"> * e.g., wordFeature, RegexFeature for all tokens. The r table is represented</span> |
| <a name="l00194"></a>00194 <span class="comment"> * in the following way.</span> |
| <a name="l00195"></a>00195 <span class="comment"> * <pre></span> |
| <a name="l00196"></a>00196 <span class="comment"> * 0 1 2 3 4...44</span> |
| <a name="l00197"></a>00197 <span class="comment"> * token1 a a a a a...a</span> |
| <a name="l00198"></a>00198 <span class="comment"> * token2 a a a a a...a</pre></span> |
| <a name="l00199"></a>00199 <span class="comment"> *</span> |
| <a name="l00200"></a>00200 <span class="comment"> * @param segmenttbl Name of table containing all the tokenized testing sentences.</span> |
| <a name="l00201"></a>00201 <span class="comment"> * @param dictionary Name of table containing the dictionary.</span> |
| <a name="l00202"></a>00202 <span class="comment"> * @param labeltbl Name of table containing the the label space used in POS or other NLP tasks.</span> |
| <a name="l00203"></a>00203 <span class="comment"> * @param regextbl Name of table containing all the regular expressions to capture regex features.</span> |
| <a name="l00204"></a>00204 <span class="comment"> * @param viterbi_mtbl Name of table to store the m factors.</span> |
| <a name="l00205"></a>00205 <span class="comment"> * @param viterbi_rtbl Name of table to store the r factors.</span> |
| <a name="l00206"></a>00206 <span class="comment"> *</span> |
| <a name="l00207"></a>00207 <span class="comment"> */</span> |
| <a name="l00208"></a>00208 |
| <a name="l00209"></a>00209 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.crf_test_fgen( |
| <a name="l00210"></a>00210 segmenttbl text, |
| <a name="l00211"></a>00211 dictionary text, |
| <a name="l00212"></a>00212 labeltbl text, |
| <a name="l00213"></a>00213 regextbl text, |
| <a name="l00214"></a>00214 featuretbl text, |
| <a name="l00215"></a>00215 viterbi_mtbl text, |
| <a name="l00216"></a>00216 viterbi_rtbl text) RETURNS void AS |
| <a name="l00217"></a>00217 $$ |
| <a name="l00218"></a>00218 # Clear m&r tables |
| <a name="l00219"></a>00219 plpy.execute(""" |
| <a name="l00220"></a>00220 DROP TABLE IF EXISTS {viterbi_mtbl}, {viterbi_rtbl}; |
| <a name="l00221"></a>00221 """.format( |
| <a name="l00222"></a>00222 viterbi_mtbl = viterbi_mtbl, |
| <a name="l00223"></a>00223 viterbi_rtbl = viterbi_rtbl |
| <a name="l00224"></a>00224 )) |
| <a name="l00225"></a>00225 # Create m&r factor table |
| <a name="l00226"></a>00226 plpy.execute(""" |
| <a name="l00227"></a>00227 CREATE TABLE {viterbi_mtbl} (score integer[][]); |
| <a name="l00228"></a>00228 """.format(viterbi_mtbl = viterbi_mtbl)); |
| <a name="l00229"></a>00229 plpy.execute(""" |
| <a name="l00230"></a>00230 CREATE TABLE {viterbi_rtbl} |
| <a name="l00231"></a><a class="code" href="crf__feature__gen_8sql__in.html#a14382e24235202905e522ef513966dee">00231</a> (seg_text text, label integer, score integer); |
| <a name="l00232"></a>00232 """.format(viterbi_rtbl = viterbi_rtbl)) |
| <a name="l00233"></a>00233 # Create index for performance |
| <a name="l00234"></a>00234 plpy.execute(""" |
| <a name="l00235"></a>00235 CREATE INDEX {viterbi_rtbl}_idx ON {viterbi_rtbl} (seg_text) |
| <a name="l00236"></a>00236 """.format(viterbi_rtbl = viterbi_rtbl)) |
| <a name="l00237"></a>00237 |
| <a name="l00238"></a>00238 |
| <a name="l00239"></a>00239 origClientMinMessages = plpy.execute("SELECT setting AS setting FROM pg_settings WHERE name = \'client_min_messages\';") |
| <a name="l00240"></a>00240 plpy.execute("SET client_min_messages TO warning;") |
| <a name="l00241"></a>00241 plpy.execute("SELECT MADLIB_SCHEMA.create_schema_pg_temp();") |
| <a name="l00242"></a>00242 |
| <a name="l00243"></a>00243 prev_labeltbl = "pg_temp._madlib_prev_labeltbl" |
| <a name="l00244"></a>00244 segment_hashtbl = "pg_temp._madlib_segment_hashtbl" |
| <a name="l00245"></a>00245 unknown_segment_hashtbl = "pg_temp._madlib_unknown_segment_hashtbl" |
| <a name="l00246"></a>00246 rtbl = "pg_temp._madlib_rtbl" |
| <a name="l00247"></a>00247 mtbl = "pg_temp._madlib_mtbl" |
| <a name="l00248"></a>00248 tmp_segmenttbl = "pg_temp._madlib_tmp_segmenttbl" |
| <a name="l00249"></a>00249 tmp_dict = "pg_temp._madlib_tmp_dict" |
| <a name="l00250"></a>00250 |
| <a name="l00251"></a>00251 plpy.execute("""DROP TABLE IF EXISTS """ + prev_labeltbl + """,""" + segment_hashtbl + """,""" + unknown_segment_hashtbl + """,""" + rtbl + """,""" + mtbl + """,""" + tmp_segmenttbl + """,""" + tmp_dict + """;""") |
| <a name="l00252"></a>00252 |
| <a name="l00253"></a>00253 plpy.execute("""CREATE TABLE """ + prev_labeltbl + """(id int);""") |
| <a name="l00254"></a>00254 |
| <a name="l00255"></a>00255 # Insert unique tokens into the """ + segment_hashtbl + """ |
| <a name="l00256"></a>00256 plpy.execute("CREATE TABLE """ + segment_hashtbl + """(seg_text text);""") |
| <a name="l00257"></a>00257 |
| <a name="l00258"></a>00258 # create a temp partial dictionary table which stores the words whose occurance |
| <a name="l00259"></a>00259 # is below certain threshold, refer to the CRF Package |
| <a name="l00260"></a>00260 plpy.execute("""CREATE TABLE """ + unknown_segment_hashtbl + """(seg_text text);""") |
| <a name="l00261"></a>00261 |
| <a name="l00262"></a>00262 # Generate a sparse matrix to store the r factors |
| <a name="l00263"></a>00263 plpy.execute("""CREATE TABLE """ + rtbl + """ (seg_text text NOT NULL, label integer, value double precision);""") |
| <a name="l00264"></a>00264 |
| <a name="l00265"></a>00265 # Generate M factor table |
| <a name="l00266"></a>00266 plpy.execute("""CREATE TABLE """ + mtbl + """(prev_label integer, label integer, value double precision);""") |
| <a name="l00267"></a>00267 |
| <a name="l00268"></a>00268 # temp tables to keep segments and dictionary with all digits replaced by the word 'DIGIT<span class="stringliteral">'</span> |
| <a name="l00269"></a>00269 <span class="stringliteral"> plpy.execute("""CREATE TABLE """ + tmp_segmenttbl + """(start_pos int,doc_id int,seg_text text,max_pos int);""")</span> |
| <a name="l00270"></a>00270 <span class="stringliteral"> plpy.execute("""CREATE TABLE """+ tmp_dict + """(token text, total int);""")</span> |
| <a name="l00271"></a>00271 <span class="stringliteral"></span> |
| <a name="l00272"></a>00272 <span class="stringliteral"> plpy.execute("""SET client_min_messages TO """ + str(origClientMinMessages[0]['</span>setting<span class="stringliteral">']) + """;""")</span> |
| <a name="l00273"></a>00273 <span class="stringliteral"></span> |
| <a name="l00274"></a>00274 <span class="stringliteral"> # Calculate the number of labels in the label space</span> |
| <a name="l00275"></a>00275 <span class="stringliteral"> rv = plpy.execute("""SELECT COUNT(*) AS total_label FROM """ + labeltbl + """;""")</span> |
| <a name="l00276"></a>00276 <span class="stringliteral"> nlabel = rv[0]['</span>total_label<span class="stringliteral">']</span> |
| <a name="l00277"></a>00277 <span class="stringliteral"></span> |
| <a name="l00278"></a>00278 <span class="stringliteral"> # replace digits with "DIGIT" keyword</span> |
| <a name="l00279"></a>00279 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_segmenttbl + """ SELECT start_pos,doc_id,seg_text,max_pos FROM """ + segmenttbl + """ WHERE</span> |
| <a name="l00280"></a>00280 <span class="stringliteral"> NOT (seg_text ~ E'</span>^[-+]?([0-9]{1,3}[,]?)*[0-9]{1,3}$<span class="stringliteral">' OR seg_text ~ E'</span>^[-+]?[0-9]*[.][0-9]+$<span class="stringliteral">');""")</span> |
| <a name="l00281"></a>00281 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + tmp_segmenttbl + """ SELECT start_pos,doc_id,'</span>DIGIT<span class="stringliteral">',max_pos FROM """ + segmenttbl + """ WHERE </span> |
| <a name="l00282"></a>00282 <span class="stringliteral"> seg_text ~ E'</span>^[-+]?([0-9]{1,3}[,]?)*[0-9]{1,3}$<span class="stringliteral">' OR seg_text ~E'</span>^[-+]?[0-9]*[.][0-9]+$<span class="stringliteral">';""")</span> |
| <a name="l00283"></a>00283 <span class="stringliteral"></span> |
| <a name="l00284"></a>00284 <span class="stringliteral"> plpy.execute("""INSERT INTO """+ tmp_dict + """ SELECT token,sum(total) FROM """ + dictionary + """ GROUP BY token </span> |
| <a name="l00285"></a>00285 <span class="stringliteral"> HAVING (token NOT LIKE E'</span>^[-+]?([0-9]{1,3}[,]?)*[0-9]{1,3}$<span class="stringliteral">' AND token NOT LIKE E'</span>^[-+]?[0-9]*[.][0-9]+$<span class="stringliteral">');""")</span> |
| <a name="l00286"></a>00286 <span class="stringliteral"> plpy.execute("""INSERT INTO """+ tmp_dict + """ SELECT '</span>DIGIT<span class="stringliteral">',sum(total) FROM """ + dictionary + """ WHERE </span> |
| <a name="l00287"></a>00287 <span class="stringliteral"> (token ~ E'</span>^[-+]?([0-9]{1,3}[,]?)*[0-9]{1,3}$<span class="stringliteral">' OR token ~ E'</span>^[-+]?[0-9]*[.][0-9]+$<span class="stringliteral">') GROUP BY token;""")</span> |
| <a name="l00288"></a>00288 <span class="stringliteral"></span> |
| <a name="l00289"></a>00289 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + segment_hashtbl + """(seg_text)</span> |
| <a name="l00290"></a>00290 <span class="stringliteral"> SELECT DISTINCT seg_text</span> |
| <a name="l00291"></a>00291 <span class="stringliteral"> FROM """ + tmp_segmenttbl + """;""")</span> |
| <a name="l00292"></a>00292 <span class="stringliteral"></span> |
| <a name="l00293"></a>00293 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + unknown_segment_hashtbl + """(seg_text)</span> |
| <a name="l00294"></a>00294 <span class="stringliteral"> ((SELECT DISTINCT seg_text</span> |
| <a name="l00295"></a>00295 <span class="stringliteral"> FROM """ + segment_hashtbl + """)</span> |
| <a name="l00296"></a>00296 <span class="stringliteral"> EXCEPT</span> |
| <a name="l00297"></a>00297 <span class="stringliteral"> (SELECT DISTINCT token</span> |
| <a name="l00298"></a>00298 <span class="stringliteral"> FROM """+ tmp_dict + """ </span> |
| <a name="l00299"></a>00299 <span class="stringliteral"> WHERE total>1));""")</span> |
| <a name="l00300"></a>00300 <span class="stringliteral"></span> |
| <a name="l00301"></a>00301 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + prev_labeltbl + """</span> |
| <a name="l00302"></a>00302 <span class="stringliteral"> SELECT id</span> |
| <a name="l00303"></a>00303 <span class="stringliteral"> FROM """ + labeltbl + """;</span> |
| <a name="l00304"></a>00304 <span class="stringliteral"> INSERT INTO """ + prev_labeltbl + """ VALUES(-1);</span> |
| <a name="l00305"></a>00305 <span class="stringliteral"> INSERT INTO """ + prev_labeltbl + """ VALUES( """ + str(nlabel) + """);""")</span> |
| <a name="l00306"></a>00306 <span class="stringliteral"></span> |
| <a name="l00307"></a>00307 <span class="stringliteral"> # Generate sparse M factor table</span> |
| <a name="l00308"></a>00308 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + mtbl + """(prev_label, label, value)</span> |
| <a name="l00309"></a>00309 <span class="stringliteral"> SELECT prev_label.id, label.id, 0</span> |
| <a name="l00310"></a>00310 <span class="stringliteral"> FROM """ + labeltbl + """ AS label,</span> |
| <a name="l00311"></a>00311 <span class="stringliteral"> """ + prev_labeltbl + """ as prev_label;""")</span> |
| <a name="l00312"></a>00312 <span class="stringliteral"></span> |
| <a name="l00313"></a>00313 <span class="stringliteral"> # EdgeFeature and startFeature, startFeature can be considered as a special edgeFeature</span> |
| <a name="l00314"></a>00314 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + mtbl + """(prev_label, label, value)</span> |
| <a name="l00315"></a>00315 <span class="stringliteral"> SELECT prev_label_id,label_id,weight</span> |
| <a name="l00316"></a>00316 <span class="stringliteral"> FROM """ + featuretbl + """ AS features</span> |
| <a name="l00317"></a>00317 <span class="stringliteral"> WHERE features.prev_label_id<>-1 OR features.name = '</span>S.<span class="stringliteral">';""")</span> |
| <a name="l00318"></a>00318 <span class="stringliteral"></span> |
| <a name="l00319"></a>00319 <span class="stringliteral"> # EndFeature, endFeature can be considered as a special edgeFeature</span> |
| <a name="l00320"></a>00320 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + mtbl + """(prev_label, label, value)</span> |
| <a name="l00321"></a>00321 <span class="stringliteral"> SELECT """ + str(nlabel) + """, label_id, weight</span> |
| <a name="l00322"></a>00322 <span class="stringliteral"> FROM """ + featuretbl + """ AS features</span> |
| <a name="l00323"></a>00323 <span class="stringliteral"> WHERE features.name = '</span>End.<span class="stringliteral">';""")</span> |
| <a name="l00324"></a>00324 <span class="stringliteral"></span> |
| <a name="l00325"></a>00325 <span class="stringliteral">m4_ifdef(`__HAS_ORDERED_AGGREGATES__'</span>, ` |
| <a name="l00326"></a>00326 plpy.execute(<span class="stringliteral">""</span><span class="stringliteral">"INSERT INTO "</span><span class="stringliteral">""</span> + viterbi_mtbl + <span class="stringliteral">""</span><span class="stringliteral">"</span> |
| <a name="l00327"></a>00327 <span class="stringliteral"> SELECT array_agg(weight ORDER BY prev_label,label)</span> |
| <a name="l00328"></a>00328 <span class="stringliteral"> FROM (SELECT prev_label, label, (SUM(value)*1000)::integer AS weight</span> |
| <a name="l00329"></a>00329 <span class="stringliteral"> FROM "</span><span class="stringliteral">""</span> + mtbl + <span class="stringliteral">""</span><span class="stringliteral">"</span> |
| <a name="l00330"></a>00330 <span class="stringliteral"> GROUP BY prev_label,label</span> |
| <a name="l00331"></a>00331 <span class="stringliteral"> ORDER BY prev_label,label) as TEMP_MTBL;"</span><span class="stringliteral">""</span>.format( |
| <a name="l00332"></a>00332 viterbi_mtbl = viterbi_mtbl |
| <a name="l00333"></a>00333 )) |
| <a name="l00334"></a>00334 <span class="stringliteral">', `</span> |
| <a name="l00335"></a>00335 <span class="stringliteral"> plpy.execute("""INSERT INTO """ + viterbi_mtbl + """</span> |
| <a name="l00336"></a>00336 <span class="stringliteral"> SELECT ARRAY(</span> |
| <a name="l00337"></a>00337 <span class="stringliteral"> SELECT</span> |
| <a name="l00338"></a>00338 <span class="stringliteral"> (SUM(value) * 1000)::integer</span> |
| <a name="l00339"></a>00339 <span class="stringliteral"> FROM</span> |
| <a name="l00340"></a>00340 <span class="stringliteral"> """ + mtbl + """</span> |
| <a name="l00341"></a>00341 <span class="stringliteral"> GROUP BY</span> |
| <a name="l00342"></a>00342 <span class="stringliteral"> prev_label, label</span> |
| <a name="l00343"></a>00343 <span class="stringliteral"> ORDER BY</span> |
| <a name="l00344"></a>00344 <span class="stringliteral"> prev_label, label</span> |
| <a name="l00345"></a>00345 <span class="stringliteral"> );""".format(</span> |
| <a name="l00346"></a>00346 <span class="stringliteral"> viterbi_mtbl = viterbi_mtbl</span> |
| <a name="l00347"></a>00347 <span class="stringliteral"> ))</span> |
| <a name="l00348"></a>00348 <span class="stringliteral">'</span>) |
| <a name="l00349"></a>00349 |
| <a name="l00350"></a>00350 plpy.execute(<span class="stringliteral">""</span><span class="stringliteral">"INSERT INTO "</span><span class="stringliteral">""</span> + rtbl + <span class="stringliteral">""</span><span class="stringliteral">"(seg_text, label, value)</span> |
| <a name="l00351"></a>00351 <span class="stringliteral"> SELECT segment_hashtbl.seg_text, labels.id, 0</span> |
| <a name="l00352"></a>00352 <span class="stringliteral"> FROM "</span><span class="stringliteral">""</span> + segment_hashtbl + <span class="stringliteral">""</span><span class="stringliteral">" segment_hashtbl,</span> |
| <a name="l00353"></a>00353 <span class="stringliteral"> "</span><span class="stringliteral">""</span> + labeltbl + <span class="stringliteral">""</span><span class="stringliteral">" AS labels;"</span><span class="stringliteral">""</span>) |
| <a name="l00354"></a>00354 |
| <a name="l00355"></a>00355 <span class="preprocessor"> # RegExFeature</span> |
| <a name="l00356"></a>00356 <span class="preprocessor"></span> plpy.execute(<span class="stringliteral">""</span><span class="stringliteral">"INSERT INTO "</span><span class="stringliteral">""</span> + rtbl + <span class="stringliteral">""</span><span class="stringliteral">"(seg_text, label, value)</span> |
| <a name="l00357"></a>00357 <span class="stringliteral"> SELECT segment_hashtbl.seg_text, features.label_id, features.weight</span> |
| <a name="l00358"></a>00358 <span class="stringliteral"> FROM "</span><span class="stringliteral">""</span> + segment_hashtbl + <span class="stringliteral">""</span><span class="stringliteral">" AS segment_hashtbl,</span> |
| <a name="l00359"></a>00359 <span class="stringliteral"> "</span><span class="stringliteral">""</span> + featuretbl + <span class="stringliteral">""</span><span class="stringliteral">" AS features,</span> |
| <a name="l00360"></a>00360 <span class="stringliteral"> "</span><span class="stringliteral">""</span> + regextbl + <span class="stringliteral">""</span><span class="stringliteral">" AS regex</span> |
| <a name="l00361"></a>00361 <span class="stringliteral"> WHERE segment_hashtbl.seg_text ~ regex.pattern</span> |
| <a name="l00362"></a>00362 <span class="stringliteral"> AND features.name ='R_' || regex.name;"</span><span class="stringliteral">""</span>) |
| <a name="l00363"></a>00363 |
| <a name="l00364"></a>00364 <span class="preprocessor"> # UnknownFeature</span> |
| <a name="l00365"></a>00365 <span class="preprocessor"></span> plpy.execute(<span class="stringliteral">""</span><span class="stringliteral">"INSERT INTO "</span><span class="stringliteral">""</span> + rtbl + <span class="stringliteral">""</span><span class="stringliteral">"(seg_text, label, value)</span> |
| <a name="l00366"></a>00366 <span class="stringliteral"> SELECT segment_hashtbl.seg_text, features.label_id, features.weight</span> |
| <a name="l00367"></a>00367 <span class="stringliteral"> FROM "</span><span class="stringliteral">""</span> + unknown_segment_hashtbl + <span class="stringliteral">""</span><span class="stringliteral">" AS segment_hashtbl,</span> |
| <a name="l00368"></a>00368 <span class="stringliteral"> "</span><span class="stringliteral">""</span> + featuretbl + <span class="stringliteral">""</span><span class="stringliteral">" AS features</span> |
| <a name="l00369"></a>00369 <span class="stringliteral"> WHERE features.name = 'U';"</span><span class="stringliteral">""</span>) |
| <a name="l00370"></a>00370 |
| <a name="l00371"></a>00371 <span class="preprocessor"> # Wordfeature</span> |
| <a name="l00372"></a>00372 <span class="preprocessor"></span> plpy.execute(<span class="stringliteral">""</span><span class="stringliteral">"INSERT INTO "</span><span class="stringliteral">""</span> + rtbl + <span class="stringliteral">""</span><span class="stringliteral">"(seg_text, label, value)</span> |
| <a name="l00373"></a>00373 <span class="stringliteral"> SELECT seg_text, label_id, weight</span> |
| <a name="l00374"></a>00374 <span class="stringliteral"> FROM "</span><span class="stringliteral">""</span> + segment_hashtbl + <span class="stringliteral">""</span><span class="stringliteral">",</span> |
| <a name="l00375"></a>00375 <span class="stringliteral"> "</span><span class="stringliteral">""</span> + featuretbl + <span class="stringliteral">""</span><span class="stringliteral">"</span> |
| <a name="l00376"></a>00376 <span class="stringliteral"> WHERE name = 'W_' || seg_text;"</span><span class="stringliteral">""</span>) |
| <a name="l00377"></a>00377 |
| <a name="l00378"></a>00378 <span class="preprocessor"> # Factor table</span> |
| <a name="l00379"></a>00379 <span class="preprocessor"></span> plpy.execute(<span class="stringliteral">""</span><span class="stringliteral">"INSERT INTO "</span><span class="stringliteral">""</span> + viterbi_rtbl + <span class="stringliteral">""</span><span class="stringliteral">"(seg_text, label, score)</span> |
| <a name="l00380"></a>00380 <span class="stringliteral"> SELECT seg_text,label,(SUM(value)*1000)::integer AS score</span> |
| <a name="l00381"></a>00381 <span class="stringliteral"> FROM "</span><span class="stringliteral">""</span> + rtbl + <span class="stringliteral">""</span><span class="stringliteral">"</span> |
| <a name="l00382"></a>00382 <span class="stringliteral"> GROUP BY seg_text,label;"</span><span class="stringliteral">""</span>) |
| <a name="l00383"></a>00383 |
| <a name="l00384"></a>00384 $$ LANGUAGE plpythonu STRICT; |
| </pre></div></div> |
| </div> |
| <div id="nav-path" class="navpath"> |
| <ul> |
| <li class="navelem"><a class="el" href="crf__feature__gen_8sql__in.html">crf_feature_gen.sql_in</a> </li> |
| <!-- window showing the filter options --> |
| <div id="MSearchSelectWindow" |
| onmouseover="return searchBox.OnSearchSelectShow()" |
| onmouseout="return searchBox.OnSearchSelectHide()" |
| onkeydown="return searchBox.OnSearchSelectKey(event)"> |
| <a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark"> </span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark"> </span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark"> </span>Functions</a></div> |
| |
| <!-- iframe showing the search results (closed by default) --> |
| <div id="MSearchResultsWindow"> |
| <iframe src="javascript:void(0)" frameborder="0" |
| name="MSearchResults" id="MSearchResults"> |
| </iframe> |
| </div> |
| |
| |
| <li class="footer">Generated on Tue Apr 2 2013 14:57:03 for MADlib by |
| <a href="http://www.doxygen.org/index.html"> |
| <img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.7.5.1 </li> |
| </ul> |
| </div> |
| |
| |
| </body> |
| </html> |
