[systemds] branch master updated: [MINOR] Enable multi-threaded I/O and operations for new Python API

[mboehm7]

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 5febadc  [MINOR] Enable multi-threaded I/O and operations for new 
Python API
5febadc is described below

commit 5febadc75be76c4365f20e01364610f283881c1e
Author: Matthias Boehm <mboe...@gmail.com>
AuthorDate: Thu Aug 27 21:55:33 2020 +0200

    [MINOR] Enable multi-threaded I/O and operations for new Python API
---
 .../java/org/apache/sysds/api/PythonDMLScript.java | 16 +++++++--
 .../apache/sysds/conf/ConfigurationManager.java    | 38 +++++++++++-----------
 2 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/src/main/java/org/apache/sysds/api/PythonDMLScript.java 
b/src/main/java/org/apache/sysds/api/PythonDMLScript.java
index a3e268b..62ae738 100644
--- a/src/main/java/org/apache/sysds/api/PythonDMLScript.java
+++ b/src/main/java/org/apache/sysds/api/PythonDMLScript.java
@@ -22,6 +22,7 @@ package org.apache.sysds.api;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.sysds.api.jmlc.Connection;
+import org.apache.sysds.conf.CompilerConfig;
 
 import py4j.GatewayServer;
 import py4j.GatewayServerListener;
@@ -56,7 +57,17 @@ public class PythonDMLScript {
        }
 
        private PythonDMLScript() {
-               _connection = new Connection();
+               // we enable multi-threaded I/O and operations for a single JMLC
+               // connection because the calling Python process is unlikely to 
run
+               // multi-threaded streams of operations on the same shared 
context
+               _connection = new Connection(
+                       CompilerConfig.ConfigType.PARALLEL_CP_READ_TEXTFORMATS,
+                       CompilerConfig.ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS,
+                       
CompilerConfig.ConfigType.PARALLEL_CP_READ_BINARYFORMATS,
+                       
CompilerConfig.ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS,
+                       CompilerConfig.ConfigType.PARALLEL_CP_MATRIX_OPERATIONS,
+                       
CompilerConfig.ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR,
+                       CompilerConfig.ConfigType.ALLOW_DYN_RECOMPILATION);
        }
 
        public Connection getConnection() {
@@ -108,5 +119,4 @@ class DMLGateWayListener implements GatewayServerListener {
        public void serverStopped() {
                System.out.println("GatewayServer Stopped");
        }
-
-}
\ No newline at end of file
+}
diff --git a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java 
b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java
index 358d62e..fdc3602 100644
--- a/src/main/java/org/apache/sysds/conf/ConfigurationManager.java
+++ b/src/main/java/org/apache/sysds/conf/ConfigurationManager.java
@@ -31,24 +31,24 @@ import org.apache.sysds.conf.CompilerConfig.ConfigType;
  * NOTE: parallel execution of multiple DML scripts (in the same JVM) with 
different configurations  
  *       would require changes/extensions of this class. 
  */
-public class ConfigurationManager 
+public class ConfigurationManager
 {
-       /** Global cached job conf for read-only operations     */
+       /** Global cached job conf for read-only operations */
        private static JobConf _rJob = null; 
-       
+
        /** Global DML configuration (read or defaults) */
        private static DMLConfig _dmlconf = null; 
-       
+
        /** Local DML configuration for thread-local config updates */
        private static ThreadLocalDMLConfig _ldmlconf = new 
ThreadLocalDMLConfig();
-       
-    /** Global compiler configuration (defaults) */
-    private static CompilerConfig _cconf = null;
-       
-    /** Local compiler configuration for thead-local config updates */
-    private static ThreadLocalCompilerConfig _lcconf = new 
ThreadLocalCompilerConfig();
-    
-    //global static initialization
+
+       /** Global compiler configuration (defaults) */
+       private static CompilerConfig _cconf = null;
+
+       /** Local compiler configuration for thead-local config updates */
+       private static ThreadLocalCompilerConfig _lcconf = new 
ThreadLocalCompilerConfig();
+
+       //global static initialization
        static {
                _rJob = new JobConf();
                
@@ -59,13 +59,13 @@ public class ConfigurationManager
        }
        
        
-    /**
-     * Returns a cached JobConf object, intended for global use by all 
operations 
-     * with read-only access to job conf. This prevents to read the hadoop 
conf files
-     * over and over again from classpath. However, 
-     * 
-     * @return the cached JobConf
-     */
+       /**
+        * Returns a cached JobConf object, intended for global use by all 
operations 
+        * with read-only access to job conf. This prevents to read the hadoop 
conf files
+        * over and over again from classpath. However, 
+        * 
+        * @return the cached JobConf
+        */
        public static JobConf getCachedJobConf() {
                return _rJob;
        }