[MINOR] Cleanups (lineage, java warnings, tasks, formatting, seeds)
diff --git a/dev/Tasks.txt b/dev/Tasks.txt
index 9f9bbec..da1436b 100644
--- a/dev/Tasks.txt
+++ b/dev/Tasks.txt
@@ -357,8 +357,11 @@
* Break append instruction to cbind and rbind
SYSTEMDS-510 IO formats
- * 511 Add protobuf support to write and read FrameBlocks to HDFS OKSYSTEMDS-520 Lineage Tracing, Reuse and Integration III
+ * 511 Add protobuf support to write and read FrameBlocks to HDFS OK
+
+SYSTEMDS-520 Lineage Tracing, Reuse and Integration III
* 521 New lineage option exposing cache policies OK
SYSTEMDS-610 Cleaning Pipelines
- * 611 Initial Brute force execution OK
\ No newline at end of file
+ * 611 Initial Brute force execution OK
+
diff --git a/scripts/builtin/executePipeline.dml b/scripts/builtin/executePipeline.dml
index 9ef42ff..dbbbe44 100644
--- a/scripts/builtin/executePipeline.dml
+++ b/scripts/builtin/executePipeline.dml
@@ -19,29 +19,25 @@
#
#-------------------------------------------------------------
-s_executePipeline = function(Frame[String] pipeline, Matrix[Double] X, List[Unknown] hyperParameters, Boolean verbose)
-return (Matrix[Double] X)
+s_executePipeline = function(Frame[String] pipeline, Matrix[Double] X,
+ List[Unknown] hyperParameters, Boolean verbose)
+ return (Matrix[Double] X)
{
for(i in 1: ncol(pipeline)) {
hp = matrixToList(X, as.matrix(hyperParameters[i]))
X = eval(as.scalar(pipeline[1,i]), hp)
}
-
}
-
-
+
# This function will convert the matrix row-vector into list
matrixToList = function(Matrix[Double] X, Matrix[Double] p)
-return (List[Unknown] l)
+ return (List[Unknown] l)
{
l = list()
l = append(l, X)
- if(sum(p) != -1)
- {
+ if(sum(p) != -1) {
for(i in 1:ncol(p))
l = append(l, as.scalar(p[1,i]))
}
l = append(l, FALSE) #verbose parameter value
}
-
-
diff --git a/scripts/staging/pipelines/enumerator.dml b/scripts/staging/pipelines/enumerator.dml
index 4303b52..d70bf9a 100644
--- a/scripts/staging/pipelines/enumerator.dml
+++ b/scripts/staging/pipelines/enumerator.dml
@@ -258,8 +258,7 @@
psum = sum(as.matrix(paraList[i]))
else
psum = 0.0
- }
-
+ }
}
# This function will compute the top k pipelines from the results of all executions
@@ -323,7 +322,6 @@
fclassify = function(Matrix[Double] X)
return (Double accuracy)
{
-
if(min(X[,1]) < 1)
stop("Y should contain value greater than zero")
@@ -344,7 +342,6 @@
nTest = nrow(testSet)
dTest = ncol(testSet)
-
train_X = trainSet[, 2:dTrain]
train_Y = trainSet[, 1]
@@ -353,12 +350,9 @@
betas = multiLogReg(X=train_X, Y=train_Y, icpt=2, tol=1e-9, reg=1.2, maxi=100, maxii=0, verbose=FALSE)
[prob, yhat, accuracy] = multiLogRegPredict(test_X, betas, test_Y, FALSE)
-
}
-
-
##########################################
## Call the function Enumerator
#########################################
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java
index 6c79457..3335aae 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedResponse.java
@@ -101,7 +101,7 @@
*/
public void setCheckedConstraints(Map<PrivacyLevel,LongAdder> checkedConstraints){
if ( checkedConstraints != null && !checkedConstraints.isEmpty() ){
- this.checkedConstraints = new EnumMap<PrivacyLevel, LongAdder>(PrivacyLevel.class);
+ this.checkedConstraints = new EnumMap<>(PrivacyLevel.class);
this.checkedConstraints.putAll(checkedConstraints);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
index 3c688f5..1e2e6ea 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java
@@ -92,7 +92,7 @@
}
}
- private void conditionalAddCheckedConstraints(FederatedRequest request, FederatedResponse response){
+ private static void conditionalAddCheckedConstraints(FederatedRequest request, FederatedResponse response){
if ( request.checkPrivacy() )
response.setCheckedConstraints(PrivacyMonitor.getCheckedConstraints());
}
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java
index 7f53038..be58d79 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageCache.java
@@ -371,8 +371,8 @@
private static LineageCacheEntry getIntern(LineageItem key) {
// This method is called only when entry is present either in cache or in local FS.
- if (_cache.containsKey(key) && _cache.get(key).getCacheStatus() != LineageCacheStatus.SPILLED) {
- LineageCacheEntry e = _cache.get(key);
+ LineageCacheEntry e = _cache.get(key);
+ if (e != null && e.getCacheStatus() != LineageCacheStatus.SPILLED) {
// Maintain order for eviction
LineageCacheEviction.getEntry(e);
if (DMLScript.STATISTICS)
diff --git a/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java b/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
index ec86a5e..2a16baa 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageItem.java
@@ -77,6 +77,9 @@
_opcode = opcode;
_data = data;
_inputs = inputs;
+ // materialize hash on construction
+ // (constant time operation if input hashes constructed)
+ _hash = hashCode();
}
public LineageItem[] getInputs() {
diff --git a/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java b/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java
index 9f11300..7bb5687 100644
--- a/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java
+++ b/src/main/java/org/apache/sysds/runtime/privacy/CheckedConstraintsLog.java
@@ -30,12 +30,12 @@
* Class counting the checked privacy constraints and the loaded privacy constraints.
*/
public class CheckedConstraintsLog {
- private static Map<PrivacyLevel,LongAdder> loadedConstraintsTotal = new EnumMap<PrivacyLevel,LongAdder>(PrivacyLevel.class);
+ private static Map<PrivacyLevel,LongAdder> loadedConstraintsTotal = new EnumMap<>(PrivacyLevel.class);
static {
for ( PrivacyLevel level : PrivacyLevel.values() )
loadedConstraintsTotal.put(level, new LongAdder());
}
- private static Map<PrivacyLevel,LongAdder> checkedConstraintsTotal = new EnumMap<PrivacyLevel,LongAdder>(PrivacyLevel.class);
+ private static Map<PrivacyLevel,LongAdder> checkedConstraintsTotal = new EnumMap<>(PrivacyLevel.class);
private static BiFunction<LongAdder, LongAdder, LongAdder> mergeLongAdders = (v1, v2) -> {
v1.add(v2.longValue() );
return v1;
diff --git a/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java b/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java
index 779f464..0eeeb4e 100644
--- a/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java
+++ b/src/main/java/org/apache/sysds/runtime/privacy/PrivacyMonitor.java
@@ -34,7 +34,7 @@
private static EnumMap<PrivacyLevel,LongAdder> checkedConstraints;
static {
- checkedConstraints = new EnumMap<PrivacyLevel,LongAdder>(PrivacyLevel.class);
+ checkedConstraints = new EnumMap<>(PrivacyLevel.class);
for ( PrivacyLevel level : PrivacyLevel.values() ){
checkedConstraints.put(level, new LongAdder());
}
diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java
index b5b21a5..37c4014 100644
--- a/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/builtin/BuiltinMulticlassSVMTest.java
@@ -113,8 +113,8 @@
Integer.toString(run),
expectedDir());
- double[][] X = getRandomMatrix(rows, colsX, 0, 1, sparsity, -1);
- double[][] Y = getRandomMatrix(rows, 1, 0, 10, 1, -1);
+ double[][] X = getRandomMatrix(rows, colsX, 0, 1, sparsity, 2);
+ double[][] Y = getRandomMatrix(rows, 1, 0, 10, 1, 3);
Y = TestUtils.round(Y);
writeInputMatrixWithMTD("X", X, true);
diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java
index c938b28..cd2ee71 100644
--- a/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameDropInvalidLengthTest.java
@@ -186,14 +186,15 @@
}
}
- private ArrayList<Integer> getBadIndexes(int length) {
- ArrayList<Integer> list = new ArrayList();
+ private static ArrayList<Integer> getBadIndexes(int length) {
+ ArrayList<Integer> list = new ArrayList<>();
for(int i =0; i<length; i++)
{
int r = ThreadLocalRandom.current().nextInt(0, rows);
list.add(r);
}
- return (ArrayList) list.stream().distinct().collect(Collectors.toList());
+ return (ArrayList<Integer>) list.stream()
+ .distinct().collect(Collectors.toList());
}
public static void initFrameDataString(FrameBlock frame1, double[][] data, Types.ValueType[] lschema) {
diff --git a/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java b/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java
index ae0a5f9..1a76019 100644
--- a/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/privacy/CheckedConstraintsLogTest.java
@@ -79,7 +79,7 @@
&& CheckedConstraintsLog.getCheckedConstraints().get(PrivacyLevel.PrivateAggregation).longValue() == 150);
}
- private EnumMap<PrivacyLevel,LongAdder> getMap(PrivacyLevel level, long value){
+ private static EnumMap<PrivacyLevel,LongAdder> getMap(PrivacyLevel level, long value){
EnumMap<PrivacyLevel,LongAdder> checked = new EnumMap<>(PrivacyLevel.class);
LongAdder valueAdder = new LongAdder();
valueAdder.add(value);