scripts/builtin/WoE.dml

#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

# function Weight of evidence / information gain
#
# INPUT:
# --------------------------------------------------
# X       ---
# Y       ---
# mask    ---
# --------------------------------------------------
#
# OUTPUT:
# ------------------------------------------------
# F              Weighted X matrix where the entropy mask is applied
# entropyMatrix  A entropy matrix to apply to data
# ------------------------------------------------

m_WoE = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] mask)
return (Matrix[Double] F, Matrix[Double] entropyMatrix) {
  
  tempX = replace(target=X, pattern=NaN, replacement=1)
  entropyMatrix = matrix(0, rows=ncol(tempX), cols = max((tempX*mask)))
  if(sum(mask) > 0)
  {
    for(i in 1:ncol(mask))
    {
      if(as.scalar(mask[1, i]) == 1)
      {
        L = tempX[, i]
        entropy = getEntropy(L, Y)
        entropyMatrix[i, 1:ncol(entropy)] = entropy
      }
    
    }
  }
  F = WoEApply(X, Y, entropyMatrix)
}



getEntropy = function(Matrix[Double] eX, Matrix[Double] eY)
return(Matrix[Double] entropyMatrix)
{

  tab = table(eX, eY)
  # print("tab \n"+toString(tab))
  entropyMatrix = matrix(0, rows=1, cols=nrow(tab))
  catTotal = rowSums(tab)
  for(i in 1:nrow(tab))
  {
    # print("catProb: " +catProb)
    entropy =  (tab[i,]/catTotal[i])
    catEntropy = sum(-entropy * log(entropy, 2))
    catEntropy = ifelse(is.na(catEntropy), 0, catEntropy)
    # print("cat entropy: "+catEntropy)
    entropyMatrix[1, i] = catEntropy
  }  
}