[MINOR] Enable multi-threaded I/O and operations for new Python API
diff --git a/src/main/java/org/apache/sysds/api/PythonDMLScript.java b/src/main/java/org/apache/sysds/api/PythonDMLScript.java
index a3e268b..62ae738 100644
--- a/src/main/java/org/apache/sysds/api/PythonDMLScript.java
+++ b/src/main/java/org/apache/sysds/api/PythonDMLScript.java
@@ -22,6 +22,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysds.api.jmlc.Connection;
+import org.apache.sysds.conf.CompilerConfig;
import py4j.GatewayServer;
import py4j.GatewayServerListener;
@@ -56,7 +57,17 @@
}
private PythonDMLScript() {
- _connection = new Connection();
+ // we enable multi-threaded I/O and operations for a single JMLC
+ // connection because the calling Python process is unlikely to run
+ // multi-threaded streams of operations on the same shared context
+ _connection = new Connection(
+ CompilerConfig.ConfigType.PARALLEL_CP_READ_TEXTFORMATS,
+ CompilerConfig.ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS,
+ CompilerConfig.ConfigType.PARALLEL_CP_READ_BINARYFORMATS,
+ CompilerConfig.ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS,
+ CompilerConfig.ConfigType.PARALLEL_CP_MATRIX_OPERATIONS,
+ CompilerConfig.ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR,
+ CompilerConfig.ConfigType.ALLOW_DYN_RECOMPILATION);
}
public Connection getConnection() {
@@ -108,5 +119,4 @@
public void serverStopped() {
System.out.println("GatewayServer Stopped");
}
-
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java
index 358d62e..fdc3602 100644
--- a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java
+++ b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java
@@ -31,24 +31,24 @@
* NOTE: parallel execution of multiple DML scripts (in the same JVM) with different configurations
* would require changes/extensions of this class.
*/
-public class ConfigurationManager
+public class ConfigurationManager
{
- /** Global cached job conf for read-only operations */
+ /** Global cached job conf for read-only operations */
private static JobConf _rJob = null;
-
+
/** Global DML configuration (read or defaults) */
private static DMLConfig _dmlconf = null;
-
+
/** Local DML configuration for thread-local config updates */
private static ThreadLocalDMLConfig _ldmlconf = new ThreadLocalDMLConfig();
-
- /** Global compiler configuration (defaults) */
- private static CompilerConfig _cconf = null;
-
- /** Local compiler configuration for thead-local config updates */
- private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig();
-
- //global static initialization
+
+ /** Global compiler configuration (defaults) */
+ private static CompilerConfig _cconf = null;
+
+ /** Local compiler configuration for thead-local config updates */
+ private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig();
+
+ //global static initialization
static {
_rJob = new JobConf();
@@ -59,13 +59,13 @@
}
- /**
- * Returns a cached JobConf object, intended for global use by all operations
- * with read-only access to job conf. This prevents to read the hadoop conf files
- * over and over again from classpath. However,
- *
- * @return the cached JobConf
- */
+ /**
+ * Returns a cached JobConf object, intended for global use by all operations
+ * with read-only access to job conf. This prevents to read the hadoop conf files
+ * over and over again from classpath. However,
+ *
+ * @return the cached JobConf
+ */
public static JobConf getCachedJobConf() {
return _rJob;
}
