-----------------------------------------------------------------------------
-- Hivemall: Hive scalable Machine Learning Library
-----------------------------------------------------------------------------

-- CREATE DATABASE IF NOT EXISTS hivemall;
-- USE hivemall;
-- set hivevar:hivemall_jar=hdfs:///apps/hivemall/hivemall-with-dependencies.jar;

DROP FUNCTION IF EXISTS hivemall_version;
CREATE FUNCTION hivemall_version as 'hivemall.HivemallVersionUDF' USING JAR '${hivemall_jar}';

---------------------------
-- binary classification --
---------------------------

DROP FUNCTION IF EXISTS train_perceptron;
CREATE FUNCTION train_perceptron as 'hivemall.classifier.PerceptronUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa;
CREATE FUNCTION train_pa as 'hivemall.classifier.PassiveAggressiveUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa1;
CREATE FUNCTION train_pa1 as 'hivemall.classifier.PassiveAggressiveUDTF$PA1' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa2;
CREATE FUNCTION train_pa2 as 'hivemall.classifier.PassiveAggressiveUDTF$PA2' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_cw;
CREATE FUNCTION train_cw as 'hivemall.classifier.ConfidenceWeightedUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_arow;
CREATE FUNCTION train_arow as 'hivemall.classifier.AROWClassifierUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_arowh;
CREATE FUNCTION train_arowh as 'hivemall.classifier.AROWClassifierUDTF$AROWh' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_scw;
CREATE FUNCTION train_scw as 'hivemall.classifier.SoftConfideceWeightedUDTF$SCW1' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_scw2;
CREATE FUNCTION train_scw2 as 'hivemall.classifier.SoftConfideceWeightedUDTF$SCW2' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_adagrad_rda;
CREATE FUNCTION train_adagrad_rda as 'hivemall.classifier.AdaGradRDAUDTF' USING JAR '${hivemall_jar}';

--------------------------------
--  Multiclass classification --
--------------------------------

DROP FUNCTION IF EXISTS train_multiclass_perceptron;
CREATE FUNCTION train_multiclass_perceptron as 'hivemall.classifier.multiclass.MulticlassPerceptronUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_pa;
CREATE FUNCTION train_multiclass_pa as 'hivemall.classifier.multiclass.MulticlassPassiveAggressiveUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_pa1;
CREATE FUNCTION train_multiclass_pa1 as 'hivemall.classifier.multiclass.MulticlassPassiveAggressiveUDTF$PA1' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_pa2;
CREATE FUNCTION train_multiclass_pa2 as 'hivemall.classifier.multiclass.MulticlassPassiveAggressiveUDTF$PA2' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_cw;
CREATE FUNCTION train_multiclass_cw as 'hivemall.classifier.multiclass.MulticlassConfidenceWeightedUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_arow;
CREATE FUNCTION train_multiclass_arow as 'hivemall.classifier.multiclass.MulticlassAROWClassifierUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_arowh;
CREATE FUNCTION train_multiclass_arowh as 'hivemall.classifier.multiclass.MulticlassAROWClassifierUDTF$AROWh' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_scw;
CREATE FUNCTION train_multiclass_scw as 'hivemall.classifier.multiclass.MulticlassSoftConfidenceWeightedUDTF$SCW1' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_multiclass_scw2;
CREATE FUNCTION train_multiclass_scw2 as 'hivemall.classifier.multiclass.MulticlassSoftConfidenceWeightedUDTF$SCW2' USING JAR '${hivemall_jar}';

--------------------------
-- similarity functions --
--------------------------

DROP FUNCTION IF EXISTS cosine_similarity;
CREATE FUNCTION cosine_similarity as 'hivemall.knn.similarity.CosineSimilarityUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS jaccard_similarity;
CREATE FUNCTION jaccard_similarity as 'hivemall.knn.similarity.JaccardIndexUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS angular_similarity;
CREATE FUNCTION angular_similarity as 'hivemall.knn.similarity.AngularSimilarityUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS euclid_similarity;
CREATE FUNCTION euclid_similarity as 'hivemall.knn.similarity.EuclidSimilarity' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS distance2similarity;
CREATE FUNCTION distance2similarity as 'hivemall.knn.similarity.Distance2SimilarityUDF' USING JAR '${hivemall_jar}';

------------------------
-- distance functions --
------------------------

DROP FUNCTION IF EXISTS popcnt;
CREATE FUNCTION popcnt as 'hivemall.knn.distance.PopcountUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS kld;
CREATE FUNCTION kld as 'hivemall.knn.distance.KLDivergenceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS hamming_distance;
CREATE FUNCTION hamming_distance as 'hivemall.knn.distance.HammingDistanceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS euclid_distance;
CREATE FUNCTION euclid_distance as 'hivemall.knn.distance.EuclidDistanceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS cosine_distance;
CREATE FUNCTION cosine_distance as 'hivemall.knn.distance.CosineDistanceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS angular_distance;
CREATE FUNCTION angular_distance as 'hivemall.knn.distance.AngularDistanceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS jaccard_distance;
CREATE FUNCTION jaccard_distance as 'hivemall.knn.distance.JaccardDistanceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS manhattan_distance;
CREATE FUNCTION manhattan_distance as 'hivemall.knn.distance.ManhattanDistanceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS minkowski_distance;
CREATE FUNCTION minkowski_distance as 'hivemall.knn.distance.MinkowskiDistanceUDF' USING JAR '${hivemall_jar}';

-------------------
-- LSH functions --
-------------------

DROP FUNCTION IF EXISTS minhashes;
CREATE FUNCTION minhashes as 'hivemall.knn.lsh.MinHashesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS minhash;
CREATE FUNCTION minhash as 'hivemall.knn.lsh.MinHashUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS bbit_minhash;
CREATE FUNCTION bbit_minhash as 'hivemall.knn.lsh.bBitMinHashUDF' USING JAR '${hivemall_jar}';

----------------------
-- voting functions --
----------------------

DROP FUNCTION IF EXISTS voted_avg;
CREATE FUNCTION voted_avg as 'hivemall.ensemble.bagging.VotedAvgUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS weight_voted_avg;
CREATE FUNCTION weight_voted_avg as 'hivemall.ensemble.bagging.WeightVotedAvgUDAF' USING JAR '${hivemall_jar}';

--------------------
-- misc functions --
--------------------

DROP FUNCTION IF EXISTS max_label;
CREATE FUNCTION max_label as 'hivemall.ensemble.MaxValueLabelUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS maxrow;
CREATE FUNCTION maxrow as 'hivemall.ensemble.MaxRowUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS argmin_kld;
CREATE FUNCTION argmin_kld as 'hivemall.ensemble.ArgminKLDistanceUDAF' USING JAR '${hivemall_jar}';

-----------------------
-- hashing functions --
-----------------------

DROP FUNCTION IF EXISTS mhash;
CREATE FUNCTION mhash as 'hivemall.ftvec.hashing.MurmurHash3UDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS sha1;
CREATE FUNCTION sha1 as 'hivemall.ftvec.hashing.Sha1UDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS array_hash_values;
CREATE FUNCTION array_hash_values as 'hivemall.ftvec.hashing.ArrayHashValuesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS prefixed_hash_values;
CREATE FUNCTION prefixed_hash_values as 'hivemall.ftvec.hashing.ArrayPrefixedHashValuesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS feature_hashing;
CREATE FUNCTION feature_hashing as 'hivemall.ftvec.hashing.FeatureHashingUDF' USING JAR '${hivemall_jar}';

-----------------------
-- pairing functions --
-----------------------

DROP FUNCTION IF EXISTS polynomial_features;
CREATE FUNCTION polynomial_features as 'hivemall.ftvec.pairing.PolynomialFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS powered_features;
CREATE FUNCTION powered_features as 'hivemall.ftvec.pairing.PoweredFeaturesUDF' USING JAR '${hivemall_jar}';

-----------------------
-- scaling functions --
-----------------------

DROP FUNCTION IF EXISTS rescale;
CREATE FUNCTION rescale as 'hivemall.ftvec.scaling.RescaleUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS zscore;
CREATE FUNCTION zscore as 'hivemall.ftvec.scaling.ZScoreUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS l2_normalize;
CREATE FUNCTION l2_normalize as 'hivemall.ftvec.scaling.L2NormalizationUDF' USING JAR '${hivemall_jar}';

--------------------
-- misc functions --
--------------------

DROP FUNCTION IF EXISTS amplify;
CREATE FUNCTION amplify as 'hivemall.ftvec.amplify.AmplifierUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS rand_amplify;
CREATE FUNCTION rand_amplify as 'hivemall.ftvec.amplify.RandomAmplifierUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS add_bias;
CREATE FUNCTION add_bias as 'hivemall.ftvec.AddBiasUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS sort_by_feature;
CREATE FUNCTION sort_by_feature as 'hivemall.ftvec.SortByFeatureUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS extract_feature;
CREATE FUNCTION extract_feature as 'hivemall.ftvec.ExtractFeatureUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS extract_weight;
CREATE FUNCTION extract_weight as 'hivemall.ftvec.ExtractWeightUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS add_feature_index;
CREATE FUNCTION add_feature_index as 'hivemall.ftvec.AddFeatureIndexUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS feature;
CREATE FUNCTION feature as 'hivemall.ftvec.FeatureUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS feature_index;
CREATE FUNCTION feature_index as 'hivemall.ftvec.FeatureIndexUDF' USING JAR '${hivemall_jar}';

----------------------------------
-- feature converting functions --
----------------------------------

DROP FUNCTION IF EXISTS conv2dense;
CREATE FUNCTION conv2dense as 'hivemall.ftvec.conv.ConvertToDenseModelUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS to_dense_features;
CREATE FUNCTION to_dense_features as 'hivemall.ftvec.conv.ToDenseFeaturesUDF' USING JAR '${hivemall_jar}';

-- alias
DROP FUNCTION IF EXISTS to_dense;
CREATE FUNCTION to_dense as 'hivemall.ftvec.conv.ToDenseFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS to_sparse_features;
CREATE FUNCTION to_sparse_features as 'hivemall.ftvec.conv.ToSparseFeaturesUDF' USING JAR '${hivemall_jar}';

-- alias
DROP FUNCTION IF EXISTS to_sparse;
CREATE FUNCTION to_sparse as 'hivemall.ftvec.conv.ToSparseFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS quantify;
CREATE FUNCTION quantify as 'hivemall.ftvec.conv.QuantifyColumnsUDTF' USING JAR '${hivemall_jar}';

--------------------------
-- feature transformers --
--------------------------

DROP FUNCTION IF EXISTS vectorize_features;
CREATE FUNCTION vectorize_features as 'hivemall.ftvec.trans.VectorizeFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS categorical_features;
CREATE FUNCTION categorical_features as 'hivemall.ftvec.trans.CategoricalFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS ffm_features;
CREATE FUNCTION ffm_features as 'hivemall.ftvec.trans.FFMFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS indexed_features;
CREATE FUNCTION indexed_features as 'hivemall.ftvec.trans.IndexedFeatures' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS quantified_features;
CREATE FUNCTION quantified_features as 'hivemall.ftvec.trans.QuantifiedFeaturesUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS quantitative_features;
CREATE FUNCTION quantitative_features as 'hivemall.ftvec.trans.QuantitativeFeaturesUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS binarize_label;
CREATE FUNCTION binarize_label as 'hivemall.ftvec.trans.BinarizeLabelUDTF' USING JAR '${hivemall_jar}';

------------------------------
-- ranking helper functions --
------------------------------

DROP FUNCTION IF EXISTS bpr_sampling;
CREATE FUNCTION bpr_sampling as 'hivemall.ftvec.ranking.BprSamplingUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS item_pairs_sampling;
CREATE FUNCTION item_pairs_sampling as 'hivemall.ftvec.ranking.ItemPairsSamplingUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS populate_not_in;
CREATE FUNCTION populate_not_in as 'hivemall.ftvec.ranking.PopulateNotInUDTF' USING JAR '${hivemall_jar}';

--------------------------
-- ftvec/text functions --
--------------------------

DROP FUNCTION IF EXISTS tf;
CREATE FUNCTION tf as 'hivemall.ftvec.text.TermFrequencyUDAF' USING JAR '${hivemall_jar}';

--------------------------
-- Regression functions --
--------------------------

DROP FUNCTION IF EXISTS logress;
CREATE FUNCTION logress as 'hivemall.regression.LogressUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_logistic_regr;
CREATE FUNCTION train_logistic_regr as 'hivemall.regression.LogressUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa1_regr;
CREATE FUNCTION train_pa1_regr as 'hivemall.regression.PassiveAggressiveRegressionUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa1a_regr;
CREATE FUNCTION train_pa1a_regr as 'hivemall.regression.PassiveAggressiveRegressionUDTF$PA1a' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa2_regr;
CREATE FUNCTION train_pa2_regr as 'hivemall.regression.PassiveAggressiveRegressionUDTF$PA2' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_pa2a_regr;
CREATE FUNCTION train_pa2a_regr as 'hivemall.regression.PassiveAggressiveRegressionUDTF$PA2a' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_arow_regr;
CREATE FUNCTION train_arow_regr as 'hivemall.regression.AROWRegressionUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_arowe_regr;
CREATE FUNCTION train_arowe_regr as 'hivemall.regression.AROWRegressionUDTF$AROWe' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_arowe2_regr;
CREATE FUNCTION train_arowe2_regr as 'hivemall.regression.AROWRegressionUDTF$AROWe2' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_adagrad_regr;
CREATE FUNCTION train_adagrad_regr as 'hivemall.regression.AdaGradUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_adadelta_regr;
CREATE FUNCTION train_adadelta_regr as 'hivemall.regression.AdaDeltaUDTF' USING JAR '${hivemall_jar}';

---------------------
-- array functions --
---------------------

DROP FUNCTION IF EXISTS float_array;
CREATE FUNCTION float_array as 'hivemall.tools.array.AllocFloatArrayUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS array_remove;
CREATE FUNCTION array_remove as 'hivemall.tools.array.ArrayRemoveUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS sort_and_uniq_array;
CREATE FUNCTION sort_and_uniq_array as 'hivemall.tools.array.SortAndUniqArrayUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS subarray_endwith;
CREATE FUNCTION subarray_endwith as 'hivemall.tools.array.SubarrayEndWithUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS subarray_startwith;
CREATE FUNCTION subarray_startwith as 'hivemall.tools.array.SubarrayStartWithUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS array_concat;
CREATE FUNCTION array_concat as 'hivemall.tools.array.ArrayConcatUDF' USING JAR '${hivemall_jar}';

-- alias for backward compatibility
DROP FUNCTION IF EXISTS concat_array;
CREATE FUNCTION concat_array as 'hivemall.tools.array.ArrayConcatUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS subarray;
CREATE FUNCTION subarray as 'hivemall.tools.array.SubarrayUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS array_avg;
CREATE FUNCTION array_avg as 'hivemall.tools.array.ArrayAvgGenericUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS array_sum;
CREATE FUNCTION array_sum as 'hivemall.tools.array.ArraySumUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS to_string_array;
CREATE FUNCTION to_string_array as 'hivemall.tools.array.ToStringArrayUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS array_intersect;
CREATE FUNCTION array_intersect as 'hivemall.tools.array.ArrayIntersectUDF' USING JAR '${hivemall_jar}';

-----------------------------
-- bit operation functions --
-----------------------------

DROP FUNCTION IF EXISTS bits_collect;
CREATE FUNCTION bits_collect as 'hivemall.tools.bits.BitsCollectUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS to_bits;
CREATE FUNCTION to_bits as 'hivemall.tools.bits.ToBitsUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS unbits;
CREATE FUNCTION unbits as 'hivemall.tools.bits.UnBitsUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS bits_or;
CREATE FUNCTION bits_or as 'hivemall.tools.bits.BitsORUDF' USING JAR '${hivemall_jar}';

---------------------------
-- compression functions --
---------------------------

DROP FUNCTION IF EXISTS inflate;
CREATE FUNCTION inflate as 'hivemall.tools.compress.InflateUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS deflate;
CREATE FUNCTION deflate as 'hivemall.tools.compress.DeflateUDF' USING JAR '${hivemall_jar}';

---------------------
-- map functions --
---------------------

DROP FUNCTION IF EXISTS map_get_sum;
CREATE FUNCTION map_get_sum as 'hivemall.tools.map.MapGetSumUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS map_tail_n;
CREATE FUNCTION map_tail_n as 'hivemall.tools.map.MapTailNUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS to_map;
CREATE FUNCTION to_map as 'hivemall.tools.map.UDAFToMap' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS to_ordered_map;
CREATE FUNCTION to_ordered_map as 'hivemall.tools.map.UDAFToOrderedMap' USING JAR '${hivemall_jar}';

---------------------
-- list functions --
---------------------

DROP FUNCTION IF EXISTS to_ordered_list;
CREATE FUNCTION to_ordered_list as 'hivemall.tools.list.UDAFToOrderedList' USING JAR '${hivemall_jar}';

---------------------
-- Math functions --
---------------------

DROP FUNCTION IF EXISTS sigmoid;
CREATE FUNCTION sigmoid as 'hivemall.tools.math.SigmoidGenericUDF' USING JAR '${hivemall_jar}';

----------------------
-- mapred functions --
----------------------

DROP FUNCTION IF EXISTS taskid;
CREATE FUNCTION taskid as 'hivemall.tools.mapred.TaskIdUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS jobid;
CREATE FUNCTION jobid as 'hivemall.tools.mapred.JobIdUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS rowid;
CREATE FUNCTION rowid as 'hivemall.tools.mapred.RowIdUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS distcache_gets;
CREATE FUNCTION distcache_gets as 'hivemall.tools.mapred.DistributedCacheLookupUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS jobconf_gets;
CREATE FUNCTION jobconf_gets as 'hivemall.tools.mapred.JobConfGetsUDF' USING JAR '${hivemall_jar}';

--------------------
-- misc functions --
--------------------

DROP FUNCTION IF EXISTS generate_series;
CREATE FUNCTION generate_series as 'hivemall.tools.GenerateSeriesUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS convert_label;
CREATE FUNCTION convert_label as 'hivemall.tools.ConvertLabelUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS x_rank;
CREATE FUNCTION x_rank as 'hivemall.tools.RankSequenceUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS each_top_k;
CREATE FUNCTION each_top_k as 'hivemall.tools.EachTopKUDTF' USING JAR '${hivemall_jar}';

-------------------------------
-- Text processing functions --
-------------------------------

DROP FUNCTION IF EXISTS tokenize;
CREATE FUNCTION tokenize as 'hivemall.tools.text.TokenizeUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS is_stopword;
CREATE FUNCTION is_stopword as 'hivemall.tools.text.StopwordUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS singularize;
CREATE FUNCTION singularize as 'hivemall.tools.text.SingularizeUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS split_words;
CREATE FUNCTION split_words as 'hivemall.tools.text.SplitWordsUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS normalize_unicode;
CREATE FUNCTION normalize_unicode as 'hivemall.tools.text.NormalizeUnicodeUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS base91;
CREATE FUNCTION base91 as 'hivemall.tools.text.Base91UDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS unbase91;
CREATE FUNCTION unbase91 as 'hivemall.tools.text.Unbase91UDF' USING JAR '${hivemall_jar}';

---------------------------------
-- Dataset generator functions --
---------------------------------

DROP FUNCTION IF EXISTS lr_datagen;
CREATE FUNCTION lr_datagen as 'hivemall.dataset.LogisticRegressionDataGeneratorUDTF' USING JAR '${hivemall_jar}';

--------------------------
-- Evaluating functions --
--------------------------

DROP FUNCTION IF EXISTS f1score;
CREATE FUNCTION f1score as 'hivemall.evaluation.FMeasureUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS mae;
CREATE FUNCTION mae as 'hivemall.evaluation.MeanAbsoluteErrorUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS mse;
CREATE FUNCTION mse as 'hivemall.evaluation.MeanSquaredErrorUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS rmse;
CREATE FUNCTION rmse as 'hivemall.evaluation.RootMeanSquaredErrorUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS r2;
CREATE FUNCTION r2 as 'hivemall.evaluation.R2UDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS ndcg;
CREATE FUNCTION ndcg as 'hivemall.evaluation.NDCGUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS logloss;
CREATE FUNCTION logloss as 'hivemall.evaluation.LogarithmicLossUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS auc;
CREATE FUNCTION auc as 'hivemall.evaluation.AUCUDAF' USING JAR '${hivemall_jar}';

--------------------------
-- Matrix Factorization --
--------------------------

DROP FUNCTION IF EXISTS mf_predict;
CREATE FUNCTION mf_predict as 'hivemall.mf.MFPredictionUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_mf_sgd;
CREATE FUNCTION train_mf_sgd as 'hivemall.mf.MatrixFactorizationSGDUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_mf_adagrad;
CREATE FUNCTION train_mf_adagrad as 'hivemall.mf.MatrixFactorizationAdaGradUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_bprmf;
CREATE FUNCTION train_bprmf as 'hivemall.mf.BPRMatrixFactorizationUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS bprmf_predict;
CREATE FUNCTION bprmf_predict as 'hivemall.mf.BPRMFPredictionUDF' USING JAR '${hivemall_jar}';

---------------------------
-- Factorization Machine --
---------------------------

DROP FUNCTION IF EXISTS fm_predict;
CREATE FUNCTION fm_predict as 'hivemall.fm.FMPredictGenericUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_fm;
CREATE FUNCTION train_fm as 'hivemall.fm.FactorizationMachineUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_ffm;
CREATE FUNCTION train_ffm as 'hivemall.fm.FieldAwareFactorizationMachineUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS ffm_predict;
CREATE FUNCTION ffm_predict as 'hivemall.fm.FFMPredictUDF' USING JAR '${hivemall_jar}';

---------------------------
-- Geo-Spatial functions --
---------------------------

DROP FUNCTION IF EXISTS tile;
CREATE FUNCTION tile as 'hivemall.geospatial.TileUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS map_url;
CREATE FUNCTION map_url as 'hivemall.geospatial.MapURLUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS lat2tiley;
CREATE FUNCTION lat2tiley as 'hivemall.geospatial.Lat2TileYUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS lon2tilex;
CREATE FUNCTION lon2tilex as 'hivemall.geospatial.Lon2TileXUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS tilex2lon;
CREATE FUNCTION tilex2lon as 'hivemall.geospatial.TileX2LonUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS tiley2lat;
CREATE FUNCTION tiley2lat as 'hivemall.geospatial.TileY2LatUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS haversine_distance;
CREATE FUNCTION haversine_distance as 'hivemall.geospatial.HaversineDistanceUDF' USING JAR '${hivemall_jar}';

----------------------------
-- Smile related features --
----------------------------

DROP FUNCTION IF EXISTS train_randomforest_classifier;
CREATE FUNCTION train_randomforest_classifier as 'hivemall.smile.classification.RandomForestClassifierUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_randomforest_regressor;
CREATE FUNCTION train_randomforest_regressor as 'hivemall.smile.regression.RandomForestRegressionUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS train_randomforest_regr;
CREATE FUNCTION train_randomforest_regr as 'hivemall.smile.regression.RandomForestRegressionUDTF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS tree_predict;
CREATE FUNCTION tree_predict as 'hivemall.smile.tools.TreePredictUDF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS rf_ensemble;
CREATE FUNCTION rf_ensemble as 'hivemall.smile.tools.RandomForestEnsembleUDAF' USING JAR '${hivemall_jar}';

DROP FUNCTION IF EXISTS guess_attribute_types;
CREATE FUNCTION guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF' USING JAR '${hivemall_jar}';