[SYSTEMDS-412] Fix lineage-based reuse for update-inplace indexing
This patch disabled lineage-based reuse for update-inplace left indexing
operations as reuse would create incorrect results due to later in-place
updates the change the cached data object.
Furthermore, this patch also aims to make the codegen tests for robust
wrt the surefire github action integration (less explain output).
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
index a1e2b92..11a4e93 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/context/SparkExecutionContext.java
@@ -171,18 +171,14 @@
_spctx = null;
}
- public void close()
- {
+ public void close() {
synchronized( SparkExecutionContext.class ) {
- if( _spctx != null )
- {
+ if( _spctx != null ) {
//stop the spark context if existing
_spctx.stop();
-
//make sure stopped context is never used again
_spctx = null;
}
-
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
index 48a512a..22964ba 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCacheConfig.java
@@ -25,6 +25,7 @@
import org.apache.sysds.runtime.instructions.Instruction;
import org.apache.sysds.runtime.instructions.cp.ComputationCPInstruction;
import org.apache.sysds.runtime.instructions.cp.ListIndexingCPInstruction;
+import org.apache.sysds.runtime.instructions.cp.MatrixIndexingCPInstruction;
import java.util.Comparator;
@@ -151,7 +152,9 @@
&& !(inst instanceof ListIndexingCPInstruction);
boolean rightop = (ArrayUtils.contains(REUSE_OPCODES, inst.getOpcode())
|| (inst.getOpcode().equals("append") && isVectorAppend(inst, ec)));
- return insttype && rightop;
+ boolean updateInplace = (inst instanceof MatrixIndexingCPInstruction)
+ && ec.getMatrixObject(((ComputationCPInstruction)inst).input1).getUpdateType().isInPlace();
+ return insttype && rightop && !updateInplace;
}
private static boolean isVectorAppend(Instruction inst, ExecutionContext ec) {
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java
index bd369c0..b1d184c 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/CellwiseTmplTest.java
@@ -477,7 +477,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S") };
+ programArgs = new String[]{"-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java
index c48db84..457285c 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/DAGCellwiseTmplTest.java
@@ -127,7 +127,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "hops", "-stats",
+ programArgs = new String[]{"hops", "-stats",
"-args", String.valueOf(cols), output("S") };
fullRScriptName = HOME + testname + ".R";
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java
index 60309b4..08c631c 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/MiscPatternTest.java
@@ -126,7 +126,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args", output("S") };
+ programArgs = new String[]{"recompile_runtime", "-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java
index d452313..d0b1c5a 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/MultiAggTmplTest.java
@@ -174,7 +174,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S") };
+ programArgs = new String[]{"-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java
index b4cdc96..d64f7ed 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/OuterProdTmplTest.java
@@ -265,7 +265,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S"), input("A")};
+ programArgs = new String[]{"-stats", "-args", output("S"), input("A")};
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java
index 10a816c..695ee51 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java
@@ -799,7 +799,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("S") };
+ programArgs = new String[]{"-stats", "-args", output("S") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java
index e76b1ab..c80508d 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowConv2DOperationsTest.java
@@ -84,7 +84,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "recompile_runtime", "-stats", "-args",
+ programArgs = new String[]{"recompile_runtime", "-stats", "-args",
String.valueOf(imgSize), String.valueOf(numImg), String.valueOf(numChannels),
String.valueOf(numFilters), String.valueOf(filterSize), String.valueOf(stride),
String.valueOf(pad), output("B"), sparseVal1, sparseVal2 };
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java
index cbbc9c3..c33a471 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowVectorComparisonTest.java
@@ -132,7 +132,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args", output("R") };
+ programArgs = new String[]{"-stats", "-args", output("R") };
fullRScriptName = HOME + testname + ".R";
rCmd = getRCmd(inputDir(), expectedDir());
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java
index b9a2bb7..74460e4 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/SparseSideInputTest.java
@@ -153,7 +153,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "-stats", "-args",
+ programArgs = new String[]{"-stats", "-args",
input("X"), input("Y"), output("R") };
fullRScriptName = HOME + testname + ".R";
diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java
index 663dcad..f674f88 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java
@@ -110,7 +110,7 @@
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain", "hops", "-stats",
+ programArgs = new String[]{"hops", "-stats",
"-args", input("X"), output("R") };
fullRScriptName = HOME + testname + ".R";
