methods/stemmer/src/pg_gp/porter_stemmer.sql_in - madlib - Git at Google

 /* ----------------------------------------------------------------------- *//**
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
  *
  * @file porter_stemmer.sql_in
  *
  * @brief implementation of porter stemmer operations in SQL
  * @date   September 2015
  *
  *
  *//* ----------------------------------------------------------------------- */

 m4_include(`SQLCommon.m4')

 /**
 @addtogroup grp_stemmer

 <div class="toc"><b>Contents</b>
 <ul>
 <li><a href="#notes">Implementation Notes</a></li>
 <li><a href="#list">List of Stemmer Operations</a></li>
 <li><a href="#examples">Examples</a></li>
 <li><a href="#related">Related Topics</a></li>
 </ul>
 </div>

 @brief Provides porter stemmer operations supporting other MADlib modules.

 This module provides a basic stemming operation for text input.
 It is a support module for several machine learning algorithms that
 require a stemmer. Currently, it only supports English words.

 This function is a SQL interface to the implementation of the
 <a href="http://tartarus.org/~martin/PorterStemmer/">Porter Stemming Algorithm</a>.
 The original stemming algorithm is written and maintained by Martin Porter

 @anchor notes
 @par Implementation Notes

 All functions described in this module work with text OR text array.

 Several of the function require TEXT VALUES, and returns NULL for a NULL input.
 See details in description of individual functions.

 @anchor list
 @par Stemmer Operations
 <table class="output">
 <tr><th>stem_token()</th><td> Returns the stem of the token. Returns NULL if input is NULL.</td></tr>

 <tr><th>stem_token_arr()</th><td> Returns the stems in an array of input token array.
 The stem would be NULL for corresponding NULL token.</td></tr>

 @anchor examples
 @examp

 -# Create a table with some words to be stemmed.
 <pre class="example">
 CREATE TABLE token_tbl ( id integer,
                          word text
                        );
 INSERT INTO token_tbl VALUES
  (1, 'kneel'),
  (2, 'kneeled'),
  (3, 'kneeling'),
  (4, 'kneels'),
  (5, 'knees'),
  (6, 'knell'),
  (7, 'knelt'),
  (8, 'knew'),
  (9, 'knick'),
  (10, 'knif'),
  (11, 'knife'),
  (12, 'knight'),
  (13, 'knightly'),
  (14, 'knights'),
  (15, 'knit'),
  (16, 'knits'),
  (17, 'knitted'),
  (18, 'knitting'),
  (19, 'knives'),
  (20, 'knob'),
  (21, 'knobs'),
  (22, 'knock'),
  (23, 'knocked'),
  (24, 'knocker'),
  (25, 'knockers'),
  (26, 'knocking'),
  (27, 'knocks'),
  (28, 'knopp'),
  (29, 'knot'),
  (30, 'knots');
 </pre>

 -# Return the stem words
 <pre class="example">
 SELECT madlib.stem_token(word) FROM token_tbl;
 </pre>
 <pre class="result">
  stem_token
 &nbsp;------------
  kneel
  kneel
  kneel
  kneel
  knee
  knell
  knelt
  knew
  knick
  knif
  knife
  knight
  knight
  knight
  knit
  knit
  knit
  knit
  knive
  knob
  knob
  knock
  knock
  knocker
  knocker
  knock
  knock
  knopp
  knot
  knot
 (30 rows)
 </pre>

 -# The input can be processed as an array
 <pre class="example">
 SELECT madlib.stem_token_arr(array_agg(word order by word)) FROM token_tbl;
 </pre>
 <pre class="result">
   stem_token_arr
 &nbsp;-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  {kneel,kneel,kneel,kneel,knee,knell,knelt,knew,knick,knif,knife,knight,knight,knight,knit,knit,knit,knit,knive,knob,knob,knock,knock,knocker,knocker,knock,knock,knopp,knot,knot}
 (1 row)
 </pre>

 @anchor related
 @par Related Topics

 File porter_stemmer.sql_in for list of functions and usage.
 */

 /**
  * @brief Returns stem of input token. Returns NULL if input token is NULL.
  *
  * @param token Text token
  * @returns stem of token.
  *
  */
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token(token text) RETURNS text
 AS 'MODULE_PATHNAME', 'stem_token'
 LANGUAGE C IMMUTABLE
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');

 /**
  * @brief Returns stems in an array of input token array. Returns NULL element for corresponding input NULL token.
  *
  * @param token_arr Text[] token
  * @returns stems array of input token array.
  *
  */
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token_arr(token_arr text[]) RETURNS text[]
 AS 'MODULE_PATHNAME', 'stem_token_arr'
 LANGUAGE C IMMUTABLE
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');


 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token()
 RETURNS TEXT AS $$
     return """
 ------------------------------------------------------------------
                         SUMMARY
 ------------------------------------------------------------------
 Stemming: Reducing a word to its stem or root form.
 This function is a SQL interface to the implementation of the
 Porter Stemming Algorithm (http://tartarus.org/~martin/PorterStemmer/).

 ------------------------------------------------------------------
                         USAGE
 ------------------------------------------------------------------
 We provide two forms of the stem function:

 1. Text input:
     SELECT {schema_madlib}.stem_token(input_token) from input_table;

 2. Array input:
     SELECT {schema_madlib}.stem_token_arr(input_token_array) from input_table;

 Here the 'input_token' is of type TEXT and input_token_array is of type TEXT[].

 ------------------------------------------------------------------
                         EXAMPLE
 ------------------------------------------------------------------
 CREATE TABLE token_tbl ( id integer,
                          word text
                        );

 INSERT INTO token_tbl VALUES
  (1, 'kneel'), (2, 'kneeled'), (3, 'kneeling'), (4, 'kneels'), (5, 'knees'),
  (6, 'knell'), (7, 'knelt'), (8, 'knew'), (9, 'knick'), (10, 'knif'),
  (11, 'knife'), (12, 'knight'), (13, 'knightly'), (14, 'knights'), (15, 'knit'),
  (16, 'knits'), (17, 'knitted'), (18, 'knitting'), (19, 'knives'), (20, 'knob'),
  (21, 'knobs'), (22, 'knock'), (23, 'knocked'), (24, 'knocker'), (25, 'knockers'),
  (26, 'knocking'), (27, 'knocks'), (28, 'knopp'), (29, 'knot'), (30, 'knots');

  -- text input
 SELECT id, {schema_madlib}.stem_token(word) FROM token_tbl;

  -- array input
 SELECT {schema_madlib}.stem_token_arr(array_agg(word order by id)) FROM token_tbl;
         """.format(schema_madlib='MADLIB_SCHEMA')
 $$ LANGUAGE PLPYTHONU IMMUTABLE
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');

 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token_arr()
 RETURNS TEXT AS $$
     SELECT MADLIB_SCHEMA.stem_token();
 $$ LANGUAGE SQL
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
	/* ----------------------------------------------------------------------- //*
	*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing,
	* software distributed under the License is distributed on an
	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	* KIND, either express or implied. See the License for the
	* specific language governing permissions and limitations
	* under the License.
	*
	* @file porter_stemmer.sql_in
	*
	* @brief implementation of porter stemmer operations in SQL
	* @date September 2015
	*
	*
	// ----------------------------------------------------------------------- */

	m4_include(`SQLCommon.m4')

	/**
	@addtogroup grp_stemmer

	<div class="toc"><b>Contents</b>
	<ul>
	<li><a href="#notes">Implementation Notes</a></li>
	<li><a href="#list">List of Stemmer Operations</a></li>
	<li><a href="#examples">Examples</a></li>
	<li><a href="#related">Related Topics</a></li>
	</ul>
	</div>

	@brief Provides porter stemmer operations supporting other MADlib modules.

	This module provides a basic stemming operation for text input.
	It is a support module for several machine learning algorithms that
	require a stemmer. Currently, it only supports English words.

	This function is a SQL interface to the implementation of the
	<a href="http://tartarus.org/~martin/PorterStemmer/">Porter Stemming Algorithm</a>.
	The original stemming algorithm is written and maintained by Martin Porter

	@anchor notes
	@par Implementation Notes

	All functions described in this module work with text OR text array.

	Several of the function require TEXT VALUES, and returns NULL for a NULL input.
	See details in description of individual functions.

	@anchor list
	@par Stemmer Operations
	<table class="output">
	<tr><th>stem_token()</th><td> Returns the stem of the token. Returns NULL if input is NULL.</td></tr>

	<tr><th>stem_token_arr()</th><td> Returns the stems in an array of input token array.
	The stem would be NULL for corresponding NULL token.</td></tr>

	@anchor examples
	@examp

	-# Create a table with some words to be stemmed.
	<pre class="example">
	CREATE TABLE token_tbl ( id integer,
	word text
	);
	INSERT INTO token_tbl VALUES
	(1, 'kneel'),
	(2, 'kneeled'),
	(3, 'kneeling'),
	(4, 'kneels'),
	(5, 'knees'),
	(6, 'knell'),
	(7, 'knelt'),
	(8, 'knew'),
	(9, 'knick'),
	(10, 'knif'),
	(11, 'knife'),
	(12, 'knight'),
	(13, 'knightly'),
	(14, 'knights'),
	(15, 'knit'),
	(16, 'knits'),
	(17, 'knitted'),
	(18, 'knitting'),
	(19, 'knives'),
	(20, 'knob'),
	(21, 'knobs'),
	(22, 'knock'),
	(23, 'knocked'),
	(24, 'knocker'),
	(25, 'knockers'),
	(26, 'knocking'),
	(27, 'knocks'),
	(28, 'knopp'),
	(29, 'knot'),
	(30, 'knots');
	</pre>

	-# Return the stem words
	<pre class="example">
	SELECT madlib.stem_token(word) FROM token_tbl;
	</pre>
	<pre class="result">
	stem_token
	------------
	kneel
	kneel
	kneel
	kneel
	knee
	knell
	knelt
	knew
	knick
	knif
	knife
	knight
	knight
	knight
	knit
	knit
	knit
	knit
	knive
	knob
	knob
	knock
	knock
	knocker
	knocker
	knock
	knock
	knopp
	knot
	knot
	(30 rows)
	</pre>

	-# The input can be processed as an array
	<pre class="example">
	SELECT madlib.stem_token_arr(array_agg(word order by word)) FROM token_tbl;
	</pre>
	<pre class="result">
	stem_token_arr
	-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
	{kneel,kneel,kneel,kneel,knee,knell,knelt,knew,knick,knif,knife,knight,knight,knight,knit,knit,knit,knit,knive,knob,knob,knock,knock,knocker,knocker,knock,knock,knopp,knot,knot}
	(1 row)
	</pre>

	@anchor related
	@par Related Topics

	File porter_stemmer.sql_in for list of functions and usage.
	*/

	/**
	* @brief Returns stem of input token. Returns NULL if input token is NULL.
	*
	* @param token Text token
	* @returns stem of token.
	*
	*/
	CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token(token text) RETURNS text
	AS 'MODULE_PATHNAME', 'stem_token'
	LANGUAGE C IMMUTABLE
	m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');

	/**
	* @brief Returns stems in an array of input token array. Returns NULL element for corresponding input NULL token.
	*
	* @param token_arr Text[] token
	* @returns stems array of input token array.
	*
	*/
	CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token_arr(token_arr text[]) RETURNS text[]
	AS 'MODULE_PATHNAME', 'stem_token_arr'
	LANGUAGE C IMMUTABLE
	m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');


	CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token()
	RETURNS TEXT AS $$
	return """
	------------------------------------------------------------------
	SUMMARY
	------------------------------------------------------------------
	Stemming: Reducing a word to its stem or root form.
	This function is a SQL interface to the implementation of the
	Porter Stemming Algorithm (http://tartarus.org/~martin/PorterStemmer/).

	------------------------------------------------------------------
	USAGE
	------------------------------------------------------------------
	We provide two forms of the stem function:

	1. Text input:
	SELECT {schema_madlib}.stem_token(input_token) from input_table;

	2. Array input:
	SELECT {schema_madlib}.stem_token_arr(input_token_array) from input_table;

	Here the 'input_token' is of type TEXT and input_token_array is of type TEXT[].

	------------------------------------------------------------------
	EXAMPLE
	------------------------------------------------------------------
	CREATE TABLE token_tbl ( id integer,
	word text
	);

	INSERT INTO token_tbl VALUES
	(1, 'kneel'), (2, 'kneeled'), (3, 'kneeling'), (4, 'kneels'), (5, 'knees'),
	(6, 'knell'), (7, 'knelt'), (8, 'knew'), (9, 'knick'), (10, 'knif'),
	(11, 'knife'), (12, 'knight'), (13, 'knightly'), (14, 'knights'), (15, 'knit'),
	(16, 'knits'), (17, 'knitted'), (18, 'knitting'), (19, 'knives'), (20, 'knob'),
	(21, 'knobs'), (22, 'knock'), (23, 'knocked'), (24, 'knocker'), (25, 'knockers'),
	(26, 'knocking'), (27, 'knocks'), (28, 'knopp'), (29, 'knot'), (30, 'knots');

	-- text input
	SELECT id, {schema_madlib}.stem_token(word) FROM token_tbl;

	-- array input
	SELECT {schema_madlib}.stem_token_arr(array_agg(word order by id)) FROM token_tbl;
	""".format(schema_madlib='MADLIB_SCHEMA')
	$$ LANGUAGE PLPYTHONU IMMUTABLE
	m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');

	CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token_arr()
	RETURNS TEXT AS $$
	SELECT MADLIB_SCHEMA.stem_token();
	$$ LANGUAGE SQL
	m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');