diff --git a/scripts/nn/examples/Example-MNIST_2NN_ReLu_Softmax.dml b/scripts/nn/examples/Example-MNIST_2NN_Leaky_ReLu_Softmax.dml
similarity index 71%
rename from scripts/nn/examples/Example-MNIST_2NN_ReLu_Softmax.dml
rename to scripts/nn/examples/Example-MNIST_2NN_Leaky_ReLu_Softmax.dml
index 7b225dc..86a3308 100644
--- a/scripts/nn/examples/Example-MNIST_2NN_ReLu_Softmax.dml
+++ b/scripts/nn/examples/Example-MNIST_2NN_Leaky_ReLu_Softmax.dml
@@ -20,17 +20,32 @@
 #-------------------------------------------------------------
 
 /*
+ * This Example trains a feed forward neural network with one input layer, two hidden affine layers (200 neurons) with
+ * leaky relu activations and one affine output layer with a softmax activation
+ *
+ * The reason for this example is to test the performance differences between single threaded, with a parameter server
+ * for parallelization and finally federated across multiple SystemDS instances
+ *
+ * Inputs:
+ *  - train: The file containing the training data
+ *  - test: the file containing the test data
+ *
  * The MNIST Data can be downloaded as follows:
  * mkdir -p data/mnist/
  * cd data/mnist/
  * curl -O https://pjreddie.com/media/files/mnist_train.csv
  * curl -O https://pjreddie.com/media/files/mnist_test.csv
+ *
+ * Sample Invocation
+ *
+ * systemds "<path to systemds repo>/systemds/scripts/nn/examples/Example-MNIST_2NN_Leaky_ReLu_Softmax.dml"
+ *       -nvargs train=<path to data>/mnist_data/mnist_train.csv test=<path to data>/mnist_data/mnist_test.csv
  */
 
 source("nn/examples/mnist_2NN.dml") as mnist_2NN
 
 # Read training data
-data = read("mnist_data/mnist_train.csv", format="csv")
+data = read($train, format="csv")
 n = nrow(data)
 
 # Extract images and labels
@@ -48,11 +63,11 @@
 y_val = labels[1:5000,]
 
 # Train
-epochs = 5
+epochs = 1
 [W_1, b_1, W_2, b_2, W_3, b_3] = mnist_2NN::train(X, y, X_val, y_val, epochs)
 
 # Read test data
-data = read("mnist_data/mnist_test.csv", format="csv")
+data = read($test, format="csv")
 n = nrow(data)
 
 # Extract images and labels
@@ -67,4 +82,4 @@
 probs = mnist_2NN::predict(X_test, W_1, b_1, W_2, b_2, W_3, b_3)
 [loss, accuracy] = mnist_2NN::eval(probs, y_test)
 
-print("Test Accuracy: " + accuracy)
\ No newline at end of file
+print("Test Accuracy: " + accuracy)
diff --git a/scripts/nn/examples/mnist_2NN.dml b/scripts/nn/examples/mnist_2NN.dml
index 3287867..1307e91 100644
--- a/scripts/nn/examples/mnist_2NN.dml
+++ b/scripts/nn/examples/mnist_2NN.dml
@@ -20,24 +20,23 @@
 #-------------------------------------------------------------
 
 /*
- * MNIST 2NN Relu Example
+ * MNIST 2NN Leaky Relu Example
  */
 
 # Imports
 source("nn/layers/affine.dml") as affine
 source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss
-source("nn/layers/relu.dml") as relu
+source("nn/layers/leaky_relu.dml") as leaky_relu
 source("nn/layers/softmax.dml") as softmax
 source("nn/optim/sgd_nesterov.dml") as sgd_nesterov
 
-train = function(matrix[double] X, matrix[double] Y,
-                 matrix[double] X_val, matrix[double] Y_val,
-                 int epochs)
-    return (matrix[double] W_1, matrix[double] b_1,
-            matrix[double] W_2, matrix[double] b_2,
-            matrix[double] W_3, matrix[double] b_3) {
+train = function(matrix[double] X, matrix[double] Y, matrix[double] X_val, 
+                 matrix[double] Y_val, int epochs)
+  return (matrix[double] W_1, matrix[double] b_1, matrix[double] W_2, 
+	        matrix[double] b_2, matrix[double] W_3, matrix[double] b_3)
+{
   /*
-   * Trains a 2NN relu softmax classifier.
+   * Trains a 2 hidden layer leaky relu softmax classifier.
    *
    * The input matrix, X, has N examples, each with D features.
    * The targets, Y, have K classes, and are one-hot encoded.
@@ -53,12 +52,13 @@
    *  - W: Weights (parameters) matrix, of shape (D, M, 3).
    *  - b: Biases vector, of shape (1, M, 3).
    */
+
   N = nrow(X)  # num examples
   D = ncol(X)  # num features
   K = ncol(Y)  # num classes
 
   # Create the network:
-  # input -> 200 neuron affine -> relu -> 200 neuron affine -> relu -> K neurons affine -> softmax
+  # input -> 200 neuron affine -> leaky_relu -> 200 neuron affine -> leaky_relu -> K neurons affine -> softmax
   [W_1, b_1] = affine::init(D, 200)
   [W_2, b_2] = affine::init(200, 200)
   [W_3, b_3] = affine::init(200, K)
@@ -87,12 +87,12 @@
       y_batch = Y[beg:end,]
 
       # Compute forward pass
-      ## input D -> 200 neuron affine -> relu -> 200 neuron affine -> relu -> K neurons affine -> softmax
+      ## input D -> 200 neuron affine -> leaky_relu -> 200 neuron affine -> leaky_relu -> K neurons affine -> softmax
       out_1 = affine::forward(X_batch, W_1, b_1)
-      out_1_relu = relu::forward(out_1)
-      out_2 = affine::forward(out_1_relu, W_2, b_2)
-      out_2_relu = relu::forward(out_2)
-      out_3 = affine::forward(out_2_relu, W_3, b_3)
+      out_1_leaky_relu = leaky_relu::forward(out_1)
+      out_2 = affine::forward(out_1_leaky_relu, W_2, b_2)
+      out_2_leaky_relu = leaky_relu::forward(out_2)
+      out_3 = affine::forward(out_2_leaky_relu, W_3, b_3)
       probs = softmax::forward(out_3)
 
       # Compute loss & accuracy for training & validation data
@@ -102,16 +102,16 @@
       loss_val = cross_entropy_loss::forward(probs_val, Y_val)
       accuracy_val = mean(rowIndexMax(probs_val) == rowIndexMax(Y_val))
       print("Epoch: " + e + ", Iter: " + i + ", Train Loss: " + loss + ", Train Accuracy: " +
-            accuracy + ", Val Loss: " + loss_val + ", Val Accuracy: " + accuracy_val)
+      accuracy + ", Val Loss: " + loss_val + ", Val Accuracy: " + accuracy_val)
 
       # Compute backward pass
       ## loss:
       dprobs = cross_entropy_loss::backward(probs, y_batch)
       dout_3 = softmax::backward(dprobs, out_3)
-      [dout_2_relu, dW_3, db_3] = affine::backward(dout_3, out_2_relu, W_3, b_3)
-      dout_2 = relu::backward(dout_2_relu, out_2)
-      [dout_1_relu, dW_2, db_2] = affine::backward(dout_2, out_1_relu, W_2, b_2)
-      dout_1 = relu::backward(dout_1_relu, out_1)
+      [dout_2_leaky_relu, dW_3, db_3] = affine::backward(dout_3, out_2_leaky_relu, W_3, b_3)
+      dout_2 = leaky_relu::backward(dout_2_leaky_relu, out_2)
+      [dout_1_leaky_relu, dW_2, db_2] = affine::backward(dout_2, out_1_leaky_relu, W_2, b_2)
+      dout_1 = leaky_relu::backward(dout_1_leaky_relu, out_1)
       [dX_batch, dW_1, db_1] = affine::backward(dout_1, X_batch, W_1, b_1)
 
       # Optimize with SGD
@@ -122,6 +122,7 @@
       [W_1, vW_1] = sgd_nesterov::update(W_1, dW_1, lr, mu, vW_1)
       [b_1, vb_1] = sgd_nesterov::update(b_1, db_1, lr, mu, vb_1)
     }
+
     # Anneal momentum towards 0.999
     mu = mu + (0.999 - mu)/(1+epochs-e)
     # Decay learning rate
@@ -130,9 +131,9 @@
 }
 
 predict = function(matrix[double] X,
-                    matrix[double] W_1, matrix[double] b_1,
-                    matrix[double] W_2, matrix[double] b_2,
-                    matrix[double] W_3, matrix[double] b_3)
+                   matrix[double] W_1, matrix[double] b_1,
+                   matrix[double] W_2, matrix[double] b_2,
+                   matrix[double] W_3, matrix[double] b_3)
     return (matrix[double] probs) {
   /*
    * Computes the class probability predictions of a softmax classifier.
@@ -148,10 +149,10 @@
    *  - probs: Class probabilities, of shape (N, K).
    */
   # Compute forward pass
-  ## input -> 200 neuron affine -> relu -> 200 neuron affine -> relu -> K neurons affine -> softmax
-  out_1_relu = relu::forward(affine::forward(X, W_1, b_1))
-  out_2_relu = relu::forward(affine::forward(out_1_relu, W_2, b_2))
-  probs = softmax::forward(affine::forward(out_2_relu, W_3, b_3))
+  ## input -> 200 neuron affine -> leaky_relu -> 200 neuron affine -> leaky_relu -> K neurons affine -> softmax
+  out_1_leaky_relu = leaky_relu::forward(affine::forward(X, W_1, b_1))
+  out_2_leaky_relu = leaky_relu::forward(affine::forward(out_1_leaky_relu, W_2, b_2))
+  probs = softmax::forward(affine::forward(out_2_leaky_relu, W_3, b_3))
 }
 
 eval = function(matrix[double] probs, matrix[double] Y)
diff --git a/scripts/nn/layers/leaky_relu.dml b/scripts/nn/layers/leaky_relu.dml
new file mode 100644
index 0000000..82fe3a1
--- /dev/null
+++ b/scripts/nn/layers/leaky_relu.dml
@@ -0,0 +1,54 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+/*
+ * Leaky Rectified Linear Unit (ReLU) nonlinearity layer.
+ */
+
+forward = function(matrix[double] X)
+    return (matrix[double] out) {
+  /*
+   * Computes the forward pass for a Leaky ReLU nonlinearity layer.
+   *
+   * Inputs:
+   *  - X: Inputs, of shape (any, any).
+   *
+   * Outputs:
+   *  - out: Outputs, of same shape as `X`.
+   */
+  # compute ifelse(X > 0, X, 0.01 * X)
+  out = max(X, 0.01 * X)
+}
+
+backward = function(matrix[double] dout, matrix[double] X)
+    return (matrix[double] dX) {
+  /*
+   * Computes the backward pass for a Leaky ReLU nonlinearity layer.
+   *
+   * Inputs:
+   *  - dout: Gradient wrt `out` from upstream, of same shape as `X`.
+   *  - X: Previous input data matrix, of shape (any, any).
+   *
+   * Outputs:
+   *  - dX: Gradient wrt `X`, of same shape as `X`.
+   */
+  dX = ifelse(X > 0, dout, 0.01 * dout)
+}
diff --git a/scripts/nn/test/grad_check.dml b/scripts/nn/test/grad_check.dml
index 56585c0..c409707 100644
--- a/scripts/nn/test/grad_check.dml
+++ b/scripts/nn/test/grad_check.dml
@@ -46,6 +46,7 @@
 source("scripts/nn/layers/avg_pool2d_builtin.dml") as avg_pool2d_builtin
 source("scripts/nn/layers/upsample2d.dml") as upsample2d
 source("scripts/nn/layers/relu.dml") as relu
+source("scripts/nn/layers/leaky_relu.dml") as leaky_relu
 source("scripts/nn/layers/rnn.dml") as rnn
 source("scripts/nn/layers/scale_shift1d.dml") as scale_shift1d
 source("scripts/nn/layers/scale_shift2d.dml") as scale_shift2d
@@ -1827,6 +1828,50 @@
   }
 }
 
+leaky_relu = function() {
+  /*
+  * Gradient check for the ReLU nonlinearity layer.
+  *
+  * NOTE: This could result in a false-negative in which the test
+  * fails due to a kink being crossed in the nonlinearity.  This
+  * occurs when the tests, f(x-h) and f(x+h), end up on opposite
+  * sides of the zero threshold of max(0, fx).  For now, just run
+  * the tests again.  In the future, we can explicitly check for
+  * this and rerun the test automatically.
+  */
+  print("Grad checking the Leaky ReLU nonlinearity layer with L2 loss.")
+
+  # Generate data
+  N = 3 # num examples
+  M = 10 # num neurons
+  X = rand(rows=N, cols=M, min=-5, max=5)
+  y = rand(rows=N, cols=M)
+  # Compute analytical gradients of loss wrt parameters
+  out = leaky_relu::forward(X)
+  dout = l2_loss::backward(out, y)
+  dX = leaky_relu::backward(dout, X)
+
+  # Grad check
+  h = 1e-5
+  for (i in 1:nrow(X)) {
+    for (j in 1:ncol(X)) {
+      # Compute numerical derivative
+      old = as.scalar(X[i,j])
+      X[i,j] = old - h
+      outmh = leaky_relu::forward(X)
+      lossmh = l2_loss::forward(outmh, y)
+      X[i,j] = old + h
+      outph = leaky_relu::forward(X)
+      lossph = l2_loss::forward(outph, y)
+      X[i,j] = old  # reset
+      dX_num = (lossph-lossmh) / (2*h)  # numerical derivative
+
+      # Check error
+      rel_error = test_util::check_rel_grad_error(as.scalar(dX[i,j]), dX_num, lossph, lossmh)
+    }
+  }
+}
+
 rnn = function() {
   /*
    * Gradient check for the simple RNN layer.
diff --git a/scripts/nn/test/run_tests.dml b/scripts/nn/test/run_tests.dml
index 36f1583..4c37bf4 100644
--- a/scripts/nn/test/run_tests.dml
+++ b/scripts/nn/test/run_tests.dml
@@ -60,6 +60,7 @@
 grad_check::avg_pool2d_builtin()
 grad_check::upsample2d()
 grad_check::relu()
+grad_check::leaky_relu()
 grad_check::rnn()
 grad_check::scale_shift1d()
 grad_check::scale_shift2d()