This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 1e7b6d416c5553ae09fe99d7472443dd02c79fad Author: baunsgaard <[email protected]> AuthorDate: Fri Sep 25 19:04:21 2020 +0200 [SYSTEMDS-2675+2676] Python Strings and printing Add strings to python interface to enable calling SystemDS without enforcing an transfer and parsing into python. --- .../python/systemds/context/systemds_context.py | 5 +- .../python/systemds/operator/operation_node.py | 9 +++- src/main/python/systemds/script_building/dag.py | 1 + .../save_log_reg_mnist_sysds.py} | 54 +++++++--------------- .../tests/matrix/{test_write.py => test_print.py} | 19 +++----- src/main/python/tests/matrix/test_write.py | 7 +++ 6 files changed, 43 insertions(+), 52 deletions(-) diff --git a/src/main/python/systemds/context/systemds_context.py b/src/main/python/systemds/context/systemds_context.py index 017d56c..d80fdfa 100644 --- a/src/main/python/systemds/context/systemds_context.py +++ b/src/main/python/systemds/context/systemds_context.py @@ -37,7 +37,7 @@ from py4j.protocol import Py4JNetworkError from systemds.utils.consts import VALID_INPUT_TYPES from systemds.utils.helpers import get_module_dir from systemds.operator import OperationNode - +from systemds.script_building import OutputType class SystemDSContext(object): """A context with a connection to a java instance with which SystemDS operations are executed. @@ -276,3 +276,6 @@ class SystemDSContext(object): def read(self, path: os.PathLike, **kwargs: Dict[str, VALID_INPUT_TYPES]): return OperationNode(self, 'read', [f'"{path}"'], named_input_nodes=kwargs) + + def scalar(self, v: Dict[str, VALID_INPUT_TYPES]): + return OperationNode(self, v, output_type=OutputType.SCALAR) \ No newline at end of file diff --git a/src/main/python/systemds/operator/operation_node.py b/src/main/python/systemds/operator/operation_node.py index aba4c63..ebe5804 100644 --- a/src/main/python/systemds/operator/operation_node.py +++ b/src/main/python/systemds/operator/operation_node.py @@ -161,6 +161,8 @@ class OperationNode(DAGNode): return f'{output}={self.operation}({inputs_comma_sep});' elif self.output_type == OutputType.NONE: return f'{self.operation}({inputs_comma_sep});' + elif self.output_type == OutputType.SCALAR: + return f'{var_name}={self.operation};' else: return f'{var_name}={self.operation}({inputs_comma_sep});' @@ -339,12 +341,17 @@ class OperationNode(DAGNode): return OperationNode(self.sds_context, 'moment', unnamed_inputs, output_type=OutputType.DOUBLE) def write(self, destination: str, format:str = "binary", **kwargs: Dict[str, VALID_INPUT_TYPES]) -> 'OperationNode': - unnamed_inputs = [self, f'"{destination}"'] named_parameters = {"format":f'"{format}"'} named_parameters.update(kwargs) return OperationNode(self.sds_context, 'write', unnamed_inputs, named_parameters, output_type= OutputType.NONE) + def to_string(self, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> 'OperationNode': + return OperationNode(self.sds_context, 'toString', [self], kwargs, output_type= OutputType.DOUBLE) + + def print(self, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> 'OperationNode': + return OperationNode(self.sds_context, 'print', [self], kwargs, output_type= OutputType.NONE) + def rev(self) -> 'OperationNode': """ Reverses the rows in a matrix diff --git a/src/main/python/systemds/script_building/dag.py b/src/main/python/systemds/script_building/dag.py index 69988f4..9e027e6 100644 --- a/src/main/python/systemds/script_building/dag.py +++ b/src/main/python/systemds/script_building/dag.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: class OutputType(Enum): MATRIX = auto() DOUBLE = auto() + SCALAR = auto() LIST = auto() NONE = auto() diff --git a/src/main/python/tests/matrix/test_write.py b/src/main/python/tests/manual_tests/save_log_reg_mnist_sysds.py similarity index 55% copy from src/main/python/tests/matrix/test_write.py copy to src/main/python/tests/manual_tests/save_log_reg_mnist_sysds.py index 14b0b8f..cbe22a0 100644 --- a/src/main/python/tests/matrix/test_write.py +++ b/src/main/python/tests/manual_tests/save_log_reg_mnist_sysds.py @@ -19,43 +19,21 @@ # # ------------------------------------------------------------- -import math -import os -import random -import shutil -import sys -import unittest - -import numpy as np -import scipy.stats as st from systemds.context import SystemDSContext from systemds.matrix import Matrix - - -class TestWrite(unittest.TestCase): - - sds: SystemDSContext = None - temp_dir: str = "tests/matrix/temp_write/" - - @classmethod - def setUpClass(cls): - cls.sds = SystemDSContext() - - @classmethod - def tearDownClass(cls): - cls.sds.close() - - def tearDown(self): - shutil.rmtree(self.temp_dir, ignore_errors=True) - - def test_write_01(self): - original = np.ones([10, 10]) - X = Matrix(self.sds, original) - X.write(self.temp_dir + "01").compute() - NX = self.sds.read(self.temp_dir + "01") - res = NX.compute() - self.assertTrue(np.allclose(original, res)) - - -if __name__ == "__main__": - unittest.main(exit=False) +from systemds.examples.tutorials.mnist import DataManager + +d = DataManager() + +base_path = "systemds/examples/tutorials/mnist/" +with SystemDSContext() as sds: + # Train Data + X = Matrix(sds, d.get_train_data().reshape((60000, 28*28))) + X.write(base_path + "train_data").compute() + Y = Matrix(sds, d.get_train_labels()) + 1.0 + Y.write(base_path + "train_labels").compute() + Xt = Matrix(sds, d.get_test_data().reshape((10000, 28*28))) + Xt.write(base_path + "test_data").compute() + Yt = Matrix(sds, d.get_test_labels()) + 1.0 + Yt.write(base_path + "test_labels").compute() + diff --git a/src/main/python/tests/matrix/test_write.py b/src/main/python/tests/matrix/test_print.py similarity index 75% copy from src/main/python/tests/matrix/test_write.py copy to src/main/python/tests/matrix/test_print.py index 14b0b8f..ee928df 100644 --- a/src/main/python/tests/matrix/test_write.py +++ b/src/main/python/tests/matrix/test_print.py @@ -32,10 +32,9 @@ from systemds.context import SystemDSContext from systemds.matrix import Matrix -class TestWrite(unittest.TestCase): +class TestPrint(unittest.TestCase): sds: SystemDSContext = None - temp_dir: str = "tests/matrix/temp_write/" @classmethod def setUpClass(cls): @@ -45,17 +44,13 @@ class TestWrite(unittest.TestCase): def tearDownClass(cls): cls.sds.close() - def tearDown(self): - shutil.rmtree(self.temp_dir, ignore_errors=True) - - def test_write_01(self): - original = np.ones([10, 10]) - X = Matrix(self.sds, original) - X.write(self.temp_dir + "01").compute() - NX = self.sds.read(self.temp_dir + "01") - res = NX.compute() - self.assertTrue(np.allclose(original, res)) + def test_print_01(self): + Matrix(self.sds, np.array([1])).to_string().print().compute() + self.assertEquals('1.000',self.sds.get_stdout()[0]) + def test_print_02(self): + self.sds.scalar(1).print().compute() + self.assertEquals('1', self.sds.get_stdout()[0]) if __name__ == "__main__": unittest.main(exit=False) diff --git a/src/main/python/tests/matrix/test_write.py b/src/main/python/tests/matrix/test_write.py index 14b0b8f..a8f0d52 100644 --- a/src/main/python/tests/matrix/test_write.py +++ b/src/main/python/tests/matrix/test_write.py @@ -56,6 +56,13 @@ class TestWrite(unittest.TestCase): res = NX.compute() self.assertTrue(np.allclose(original, res)) + def test_write_02(self): + original = np.array([[1,2,3,4,5]]) + X = Matrix(self.sds, original) + X.write(self.temp_dir + "02").compute() + NX = self.sds.read(self.temp_dir + "02") + res = NX.compute() + self.assertTrue(np.allclose(original, res)) if __name__ == "__main__": unittest.main(exit=False)
