scripts/builtin/lmCG.dml

#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License") you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

# The lmCG function solves linear regression using the conjugate gradient algorithm
#
# INPUT:
# ---------------------------------------------------------------
# X        Matrix of feature vectors.
# y        1-column matrix of response values.
# icpt     Intercept presence, shifting and rescaling the columns of X
# reg      Regularization constant (lambda) for L2-regularization. set to nonzero
#          for highly dependant/sparse/numerous features
# tol      Tolerance (epsilon) conjugate gradient procedure terminates early if L2
#          norm of the beta-residual is less than tolerance * its initial norm
# maxi     Maximum number of conjugate gradient iterations. 0 = no maximum
# verbose  If TRUE print messages are activated
# ---------------------------------------------------------------
#
# OUTPUT:
# ---------------------------------------------------------------
# B     The model fit beta that can be used as input in lmPredict
# ---------------------------------------------------------------

m_lmCG = function(Matrix[Double] X, Matrix[Double] y, Integer icpt = 0,
    Double reg = 1e-7, Double tol = 1e-7, Integer maxi = 0, Boolean verbose = TRUE)
    return (Matrix[Double] B) {
  intercept_status = icpt
  regularization = reg
  tolerance = tol
  max_iteration = maxi

  n = nrow(X)
  m = ncol(X)

  # Introduce the intercept, shift and rescale the columns of X if needed
  # add the intercept column
  if(intercept_status == 1 | intercept_status == 2){
    ones_n = matrix(1, rows = n, cols = 1)
    X = cbind(X, ones_n)
    m_ext = ncol(X)
    scale_lambda = matrix(1, rows = m_ext, cols = 1)
    scale_lambda [m_ext, 1] = 0
  }
  else{
    scale_lambda = matrix(1, rows = m, cols = 1)
    m_ext = m
  }


  # scale-&-shift X columns to mean 0, variance 1
  # Important assumption: X [, m_ext] = ones_n
  if(intercept_status == 2){                           
    avg_X_cols = t(colSums(X)) / n
    var_X_cols = (t(colSums(X ^ 2)) - n * (avg_X_cols ^ 2)) / (n - 1)
    is_unsafe = (var_X_cols <= 0)
    scale_X = 1.0 / sqrt(var_X_cols * (1 - is_unsafe) + is_unsafe)
    scale_X [m_ext, 1] = 1
    shift_X = - avg_X_cols * scale_X
    shift_X [m_ext, 1] = 0
  }else{
    scale_X = matrix(1, rows = m_ext, cols = 1)
    shift_X = matrix(0, rows = m_ext, cols = 1)
  }

  # Henceforth, if intercept_status == 2, we use "X %*% (SHIFT/SCALE TRANSFORM)"
  # instead of "X".  However, in order to preserve the sparsity of X,
  # we apply the transform associatively to some other part of the expression
  # in which it occurs.  To avoid materializing a large matrix, we rewrite it:
  #
  # ssX_A  = (SHIFT/SCALE TRANSFORM) %*% A    --- is rewritten as:
  # ssX_A  = diag (scale_X) %*% A
  # ssX_A [m_ext, ] = ssX_A [m_ext, ] + t(shift_X) %*% A
  #
  # tssX_A = t(SHIFT/SCALE TRANSFORM) %*% A   --- is rewritten as:
  # tssX_A = diag (scale_X) %*% A + shift_X %*% A [m_ext, ]

  lambda = scale_lambda * regularization
  beta_unscaled = matrix(0, rows = m_ext, cols = 1)

  if(max_iteration == 0){
    max_iteration = m_ext
  }
  i = 0

  # BEGIN THE CONJUGATE GRADIENT ALGORITHM
  if(verbose) print("Running the CG algorithm...")

  r = - t(X) %*% y

  if(intercept_status == 2){
    r = scale_X * r + shift_X %*% r [m_ext, ]
  }

  p = - r
  norm_r2 = sum(r ^ 2)
  norm_r2_initial = norm_r2
  norm_r2_target = norm_r2_initial * tolerance ^ 2
  if(verbose){
    print("||r|| initial value = " + sqrt(norm_r2_initial) +
      ", target value = " + sqrt(norm_r2_target))
  }

  while(i < max_iteration & norm_r2 > norm_r2_target){
    if(intercept_status == 2){
      ssX_p = scale_X * p
      ssX_p [m_ext, ] = ssX_p [m_ext, ] + t(shift_X) %*% p
    }else{
      ssX_p = p
    }

    q = t(X) %*% (X %*% ssX_p)

    if(intercept_status == 2) {
      q = scale_X * q + shift_X %*% q [m_ext, ]
    }

    q += lambda * p
    a = norm_r2 / sum(p * q)
    beta_unscaled += a * p
    r += a * q
    old_norm_r2 = norm_r2
    norm_r2 = sum(r ^ 2)
    p = -r + (norm_r2 / old_norm_r2) * p
    i = i + 1
    if(verbose){
      print("Iteration " + i + ":  ||r|| / ||r init|| = "
        + sqrt(norm_r2 / norm_r2_initial))
    }
  }

  if(verbose & i >= max_iteration){
    print("Warning: the maximum number of iterations has been reached.")
  }
  
  # END THE CONJUGATE GRADIENT ALGORITHM
  if(intercept_status == 2){
    beta = scale_X * beta_unscaled
    beta[m_ext, ] = beta[m_ext, ] + t(shift_X) %*% beta_unscaled
  }else{
    beta = beta_unscaled
  }

  # Set output variable
  B = beta
}