scripts/builtin/dmv.dml

#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

# The dmv-function is used to find disguised missing values utilising syntactical pattern recognition.
#
# INPUT:
# ---------------------------------------------------------------------------------------------
# X          Input Frame
# threshold  Threshold value in interval [0, 1] for dominant pattern per column (e.g., 0.8 means
#            that 80% of the entries per column must adhere this pattern to be dominant)
# replace    The string disguised missing values are replaced with
# ---------------------------------------------------------------------------------------------
#
# OUTPUT:
# -------------------------------------------------------------------------------------------------
# Y     Frame X including detected disguised missing values
# -------------------------------------------------------------------------------------------------

f_dmv = function(Frame[String] X, Double threshold=0.8, String replace="NA") return (Frame[String] Y) {
  if( threshold < 0 | threshold > 1 )
    stop("Stopping due to invalid input, threshold required in interval [0, 1] found " + threshold)

  Y = map(X, "UtilFunctions.syntacticalPatternDiscovery(" + threshold + "," + replace + ")")
}