src/test/scripts/functions/transform/tokenize/TokenizeUniHashWide.dml

#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

F1 = read($1, data_type="frame", format="csv", sep=",");

# Example Preprocessing:
F1[,4] = map(F1[,4], "x -> x.toLowerCase()");

# Example to estimate max tokens based on string length
# max_token = as.integer(max(as.matrix(map(F1[,4], "x -> x.length()"))));
# print(max_length);

max_token = 2000;

# Example spec:
# jspec = "{\"algo\": \"whitespace\",\"out\": \"count\",\"id_col\": [2],\"tokenize_col\": 3}";

jspec = read($2, data_type="scalar", value_type="string");

# If slicing is performed, then only the remaining columns are considered for tokenizer spec:
F2 = tokenize(target=F1[,2:4], spec=jspec, max_tokens=max_token);
write(F2, $3, format="csv");

# Followup spec to transform tokens of ids, leave hash alone
jspec2 = "{\"ids\": true, \"recode\": [1,2]}";

# Afterward, you can transform it into a matrix:
[X, M] = transformencode(target=F2, spec=jspec2);