[MINOR] Fix append / binary-other test issues (runtime, output)
This patch aims to fix two test packages that repeatedly created issues
when ran through github actions. Both had the characteristic of
including multiple 250s+ tests and the new output buffering likely
caused timeouts of not receiving any feedback from the tests. We now
reduced the data sized, disabled output buffering, and slightly improved
the performance of result comparisons. Together these changes improved
the total runtime of these packages by >2x, where most time is spent in
the R baseline computation.
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java
index 5958087..d5aea1c 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -45,6 +45,7 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.StringTokenizer;
@@ -885,7 +886,7 @@
String namesecond = name1;
boolean flag = true;
- /** to ensure that always the matrix with more nnz is iterated */
+ // to ensure that always the matrix with more nnz is iterated
if (m1.size() > m2.size()) {
first = m1;
second = m2;
@@ -896,28 +897,23 @@
int countErrorWithinTolerance = 0;
int countIdentical = 0;
- double minerr = -1;
- double maxerr = 0;
+ double minerr = Double.MAX_VALUE;
+ double maxerr = -Double.MAX_VALUE;
- for (CellIndex index : first.keySet()) {
- Double v1 = first.get(index);
- Double v2 = second.get(index);
- if (v1 == null)
- v1 = 0.0;
- if (v2 == null)
- v2 = 0.0;
- if (Math.abs(v1 - v2) < minerr || minerr == -1)
- minerr = Math.abs(v1 - v2);
- if (Math.abs(v1 - v2) > maxerr)
- maxerr = Math.abs(v1 - v2);
+ for (Entry<CellIndex, Double> e : first.entrySet()) {
+ Double v1 = e.getValue() == null ? 0.0 : e.getValue();
+ Double v2 = second.get(e.getKey());
+ v2 = v2 == null ? 0.0 : v2;
+ minerr = Math.min(minerr, Math.abs(v1 - v2));
+ maxerr = Math.max(maxerr, Math.abs(v1 - v2));
- if (!compareCellValue(first.get(index), second.get(index), 0, ignoreNaN)) {
- if (!compareCellValue(first.get(index), second.get(index), tolerance, ignoreNaN)) {
+ if (!compareCellValue(v1, v2, 0, ignoreNaN)) {
+ if (!compareCellValue(v1, v2, tolerance, ignoreNaN)) {
countErrorWithinTolerance++;
if(!flag)
- System.out.println(index+": "+first.get(index)+" <--> "+second.get(index));
+ System.out.println(e.getKey()+": "+v1+" <--> "+v2);
else
- System.out.println(index+": "+second.get(index)+" <--> "+first.get(index));
+ System.out.println(e.getKey()+": "+v2+" <--> "+v1);
}
} else {
countIdentical++;
@@ -962,7 +958,7 @@
case BOOLEAN: return ((Boolean)in1).compareTo((Boolean)in2);
case INT64: return ((Long)in1).compareTo((Long)in2);
case FP64:
- return (Math.abs((Double)in1-(Double)in2) < tolerance)?0:
+ return (Math.abs((Double)in1-(Double)in2) < tolerance)?0:
((Double)in1).compareTo((Double)in2);
default: throw new RuntimeException("Unsupported value type: "+vt);
}
diff --git a/src/test/java/org/apache/sysds/test/functions/append/AppendChainTest.java b/src/test/java/org/apache/sysds/test/functions/append/AppendChainTest.java
index d1b5f38..b85de58 100644
--- a/src/test/java/org/apache/sysds/test/functions/append/AppendChainTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/append/AppendChainTest.java
@@ -41,7 +41,7 @@
private final static int min=1;
private final static int max=100;
- private final static int rows = 1692;
+ private final static int rows = 492;
private final static int cols1 = 1059;
private final static int cols2a = 1;
private final static int cols3a = 1;
@@ -137,6 +137,7 @@
int expectedCompiled = platform==ExecMode.SINGLE_NODE ?
0 : 8; //3x(rblk+chkpt), append, write
+ setOutputBuffering(false);
runTest(true, false, null, expectedCompiled);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/append/AppendMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/append/AppendMatrixTest.java
index e20954c..9710262 100644
--- a/src/test/java/org/apache/sysds/test/functions/append/AppendMatrixTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/append/AppendMatrixTest.java
@@ -45,7 +45,7 @@
private final static int min=1;
private final static int max=100;
- private final static int rows = 1692;
+ private final static int rows = 492;
//usecase a: inblock single
private final static int cols1a = 375;
private final static int cols2a = 92;
@@ -179,6 +179,7 @@
int expectedCompiled = platform==ExecMode.SINGLE_NODE ?
0 : 6; //2x(rblk+chkpt), append, write
+ setOutputBuffering(false);
runTest(true, false, null, expectedCompiled);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/append/AppendVectorTest.java b/src/test/java/org/apache/sysds/test/functions/append/AppendVectorTest.java
index 0bf3f23..014b73f 100644
--- a/src/test/java/org/apache/sysds/test/functions/append/AppendVectorTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/append/AppendVectorTest.java
@@ -107,6 +107,7 @@
boolean exceptionExpected = false;
int numExpectedJobs = (platform == ExecMode.SINGLE_NODE) ? 0 : 6;
+ setOutputBuffering(false);
runTest(true, exceptionExpected, null, numExpectedJobs);
Assert.assertEquals("Wrong number of executed Spark jobs.",
numExpectedJobs, Statistics.getNoOfExecutedSPInst());
diff --git a/src/test/java/org/apache/sysds/test/functions/append/RBindCBindMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/append/RBindCBindMatrixTest.java
index d161412..de5bd68 100644
--- a/src/test/java/org/apache/sysds/test/functions/append/RBindCBindMatrixTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/append/RBindCBindMatrixTest.java
@@ -128,12 +128,12 @@
rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " "+ expectedDir();
double[][] A = getRandomMatrix(rows1, cols, min, max, sparsity, 823);
- writeInputMatrixWithMTD("A", A, true);
- double[][] B= getRandomMatrix(rows2, cols, min, max, sparsity, 923);
- writeInputMatrixWithMTD("B", B, true);
-
- //execute dml and r scripts
- runTest(true, false, null, -1);
+ writeInputMatrixWithMTD("A", A, true);
+ double[][] B= getRandomMatrix(rows2, cols, min, max, sparsity, 923);
+ writeInputMatrixWithMTD("B", B, true);
+
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare results
diff --git a/src/test/java/org/apache/sysds/test/functions/append/StringAppendTest.java b/src/test/java/org/apache/sysds/test/functions/append/StringAppendTest.java
index edacfad..ae2a602 100644
--- a/src/test/java/org/apache/sysds/test/functions/append/StringAppendTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/append/StringAppendTest.java
@@ -105,6 +105,7 @@
programArgs = new String[]{
"-args", Integer.toString(iters), output("C") };
+ setOutputBuffering(false);
runTest(exceptionExpected ? DMLRuntimeException.class : null);
}
catch(Exception ex) {
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
index 9cedfe0..c2efbbc 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullDistributedMatrixMultiplicationTest.java
@@ -45,7 +45,7 @@
private final static int rowsA = 1501;
private final static int colsA = 1103;
private final static int rowsB = 1103;
- private final static int colsB = 923;
+ private final static int colsB = 323;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.1;
@@ -192,6 +192,7 @@
double[][] B = getRandomMatrix(rowsB, colsB, 0, 1, sparsityB, 9873);
writeInputMatrixWithMTD("B", B, true);
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullIntegerDivisionTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
index cbfb317..2c57e75 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
@@ -47,7 +47,7 @@
private final static double eps = 1e-10;
private final static int rows = 1100;
- private final static int cols = 900;
+ private final static int cols = 300;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.1;
@@ -264,7 +264,8 @@
}
boolean exceptionExpected = false;
runTest(true, exceptionExpected, null, -1);
-
+
+ setOutputBuffering(false);
runRScript(true);
//compare matrices
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalMatrixTest.java
index 2580c87..8ed852c 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalMatrixTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalMatrixTest.java
@@ -45,8 +45,8 @@
private final static String TEST_CLASS_DIR = TEST_DIR + FullLogicalMatrixTest.class.getSimpleName() + "/";
private final static double eps = 1e-10;
- private final static int rows1 = 1383;
- private final static int cols1 = 1432;
+ private final static int rows1 = 1183;
+ private final static int cols1 = 1032;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.01;
@@ -421,6 +421,7 @@
writeInputMatrixWithMTD("B", B, true);
//run tests
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarLeftTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarLeftTest.java
index e01967c..53ca27a 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarLeftTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarLeftTest.java
@@ -271,6 +271,7 @@
writeInputMatrixWithMTD("A", A, true);
//run tests
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarRightTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarRightTest.java
index 33bc263..b2fb7c5 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarRightTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullLogicalScalarRightTest.java
@@ -269,6 +269,7 @@
writeInputMatrixWithMTD("A", A, true);
//run tests
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
index 052f5dd..997e43d 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTest.java
@@ -38,10 +38,10 @@
private final static String TEST_CLASS_DIR = TEST_DIR + FullMatrixMultiplicationTest.class.getSimpleName() + "/";
private final static double eps = 1e-10;
- private final static int rowsA = 1501;
- private final static int colsA = 1703;
- private final static int rowsB = 1703;
- private final static int colsB = 1107;
+ private final static int rowsA = 1101;
+ private final static int colsA = 1203;
+ private final static int rowsB = 1203;
+ private final static int colsB = 1007;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.1;
@@ -307,8 +307,8 @@
double[][] B = getRandomMatrix(rowsB, colsB, 0, 1, sparsityB, 3);
writeInputMatrix("B", B, true);
- boolean exceptionExpected = false;
- runTest(true, exceptionExpected, null, -1);
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare matrices
@@ -363,8 +363,8 @@
double[][] B = getRandomMatrix(rows2, cols2, 0, 1, sparsity1, 3);
writeInputMatrix("B", B, true);
- boolean exceptionExpected = false;
- runTest(true, exceptionExpected, null, -1);
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare matrices
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelf2Test.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelf2Test.java
index beb4b8a..7bb2991 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelf2Test.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelf2Test.java
@@ -162,6 +162,7 @@
writeInputMatrix("A", A, true);
//run dml and R scripts
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
index 9c497f2..b1e2358 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationTransposeSelfTest.java
@@ -41,11 +41,11 @@
private final static double eps = 1e-10;
//for CP
- private final static int rows1 = 3500;
- private final static int cols1 = 1500;
- //for MR
- private final static int rows2 = 7000;//7000;
- private final static int cols2 = 750;//750;
+ private final static int rows1 = 1100;
+ private final static int cols1 = 300;
+ //for Spark
+ private final static int rows2 = 2500;
+ private final static int cols2 = 750;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.1;
@@ -177,8 +177,8 @@
double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
writeInputMatrix("A", A, true);
- boolean exceptionExpected = false;
- runTest(true, exceptionExpected, null, -1);
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare matrices
@@ -248,8 +248,8 @@
double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 7);
writeInputMatrix("A", A, true);
- boolean exceptionExpected = false;
- runTest(true, exceptionExpected, null, -1);
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare matrices
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
index b17ac20..c6e6934 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMatrixMultiplicationUltraSparseTest.java
@@ -40,8 +40,8 @@
private final static double eps = 1e-10;
private final static int rowsA = 1501;
- private final static int colsA = 1703;
- private final static int rowsB = 1703;
+ private final static int colsA = 1603;
+ private final static int rowsB = 1603;
private final static int colsB = 1107;
private final static double sparsity1 = 0.7;
@@ -172,8 +172,8 @@
double[][] B = getRandomMatrix(rowsB, colsB, 0, 1, sparsityRight, 3);
writeInputMatrix("B", B, true);
- boolean exceptionExpected = false;
- runTest(true, exceptionExpected, null, -1);
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare matrices
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
index b593471..a9adc7a 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullMinMaxComparisonTest.java
@@ -258,6 +258,7 @@
HDFSTool.writeMetaDataFile(input("B.mtd"), ValueType.FP64, mc2, FileFormat.TEXT);
//run test
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullPowerTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullPowerTest.java
index c1a76ea..dbce924 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullPowerTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/FullPowerTest.java
@@ -47,7 +47,7 @@
private final static double eps = 1e-10;
private final static int rows = 1100;
- private final static int cols = 900;
+ private final static int cols = 300;
private final static double sparsity1 = 0.7;
private final static double sparsity2 = 0.1;
@@ -193,9 +193,8 @@
double[][] B = getRandomMatrix(1, 1, min, max, 1.0, 3);
writeScalarInputMatrixWithMTD( "B", B, true );
}
- boolean exceptionExpected = false;
- runTest(true, exceptionExpected, null, -1);
-
+ setOutputBuffering(false);
+ runTest(true, false, null, -1);
runRScript(true);
//compare matrices
diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/MatrixMultShortLhsTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/MatrixMultShortLhsTest.java
index 92d0d82..38c7645 100644
--- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/MatrixMultShortLhsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_other/MatrixMultShortLhsTest.java
@@ -36,9 +36,9 @@
private final static double eps = 1e-10;
private final static int rowsA = 10;
- private final static int colsA = 2023;
- private final static int rowsB = 2023;
- private final static int colsB = 1997;
+ private final static int colsA = 1523;
+ private final static int rowsB = 1523;
+ private final static int colsB = 1397;
private final static double sparsity1 = 0.9;
private final static double sparsity2 = 0.1;
@@ -91,6 +91,7 @@
writeInputMatrixWithMTD("B", B, true);
//run tests
+ setOutputBuffering(false);
runTest(true, false, null, -1);
runRScript(true);
