This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 36c8c79599125bd7ef8a629e1c3dcd663bea6f9a Author: baunsgaard <[email protected]> AuthorDate: Thu Apr 29 23:43:16 2021 +0200 [MINOR] Python auto Generator Python generator now defaults to Matrix return if there is only one output. In general we need to rework the generator a little to fit the new design, but for this we can use the new source function that mange types more consistent with the api. Also changed is lmPredict that now takes a y dataset as an argument. This is because lmPredict took a default matrix argument, that did not make sense in the generator parser, later on we need to support it in python. --- scripts/builtin/lmPredict.dml | 2 +- src/main/python/generator/generator.py | 24 ++- .../resources/template_python_script_imports | 2 +- src/main/python/systemds/operator/__init__.py | 2 +- .../python/systemds/operator/algorithm/__init__.py | 3 +- .../systemds/operator/algorithm/builtin/abstain.py | 6 +- .../systemds/operator/algorithm/builtin/als.py | 4 +- .../systemds/operator/algorithm/builtin/alsCG.py | 4 +- .../systemds/operator/algorithm/builtin/alsDS.py | 4 +- .../operator/algorithm/builtin/alsPredict.py | 6 +- .../operator/algorithm/builtin/alsTopkPredict.py | 4 +- .../systemds/operator/algorithm/builtin/arima.py | 6 +- .../systemds/operator/algorithm/builtin/bandit.py | 4 +- .../systemds/operator/algorithm/builtin/bivar.py | 4 +- .../operator/algorithm/builtin/components.py | 6 +- .../operator/algorithm/builtin/confusionMatrix.py | 4 +- .../systemds/operator/algorithm/builtin/cor.py | 6 +- .../systemds/operator/algorithm/builtin/cox.py | 4 +- .../systemds/operator/algorithm/builtin/cspline.py | 4 +- .../operator/algorithm/builtin/csplineDS.py | 4 +- .../systemds/operator/algorithm/builtin/cvlm.py | 4 +- .../systemds/operator/algorithm/builtin/dbscan.py | 6 +- .../operator/algorithm/builtin/decisionTree.py | 6 +- .../operator/algorithm/builtin/discoverFD.py | 6 +- .../systemds/operator/algorithm/builtin/dist.py | 6 +- .../algorithm/builtin/gaussianClassifier.py | 4 +- .../operator/algorithm/builtin/getAccuracy.py | 6 +- .../systemds/operator/algorithm/builtin/glm.py | 6 +- .../systemds/operator/algorithm/builtin/gmm.py | 4 +- .../operator/algorithm/builtin/gmmPredict.py | 4 +- .../systemds/operator/algorithm/builtin/gnmf.py | 4 +- .../operator/algorithm/builtin/gridSearch.py | 4 +- .../operator/algorithm/builtin/hyperband.py | 4 +- .../operator/algorithm/builtin/img_brightness.py | 6 +- .../operator/algorithm/builtin/img_crop.py | 6 +- .../operator/algorithm/builtin/img_mirror.py | 6 +- .../operator/algorithm/builtin/imputeByFD.py | 6 +- .../operator/algorithm/builtin/imputeByMean.py | 6 +- .../operator/algorithm/builtin/imputeByMedian.py | 6 +- .../operator/algorithm/builtin/imputeByMode.py | 6 +- .../operator/algorithm/builtin/intersect.py | 6 +- .../systemds/operator/algorithm/builtin/km.py | 4 +- .../systemds/operator/algorithm/builtin/kmeans.py | 4 +- .../operator/algorithm/builtin/kmeansPredict.py | 6 +- .../systemds/operator/algorithm/builtin/knnbf.py | 6 +- .../systemds/operator/algorithm/builtin/l2svm.py | 6 +- .../operator/algorithm/builtin/l2svmPredict.py | 4 +- .../systemds/operator/algorithm/builtin/lasso.py | 6 +- .../systemds/operator/algorithm/builtin/lm.py | 6 +- .../systemds/operator/algorithm/builtin/lmCG.py | 6 +- .../systemds/operator/algorithm/builtin/lmDS.py | 6 +- .../algorithm/builtin/{dbscan.py => lmPredict.py} | 12 +- .../operator/algorithm/builtin/logSumExp.py | 6 +- .../systemds/operator/algorithm/builtin/msvm.py | 6 +- .../operator/algorithm/builtin/msvmPredict.py | 4 +- .../operator/algorithm/builtin/multiLogReg.py | 6 +- .../algorithm/builtin/multiLogRegPredict.py | 4 +- .../systemds/operator/algorithm/builtin/na_locf.py | 6 +- .../operator/algorithm/builtin/naivebayes.py | 4 +- .../operator/algorithm/builtin/normalize.py | 6 +- .../systemds/operator/algorithm/builtin/outlier.py | 6 +- .../operator/algorithm/builtin/outlierByArima.py | 6 +- .../operator/algorithm/builtin/outlierByIQR.py | 6 +- .../operator/algorithm/builtin/outlierBySd.py | 6 +- .../systemds/operator/algorithm/builtin/pca.py | 4 +- .../systemds/operator/algorithm/builtin/pnmf.py | 4 +- .../systemds/operator/algorithm/builtin/ppca.py | 4 +- .../operator/algorithm/builtin/randomForest.py | 4 +- .../systemds/operator/algorithm/builtin/scale.py | 4 +- .../operator/algorithm/builtin/scaleApply.py | 6 +- .../operator/algorithm/builtin/sherlock.py | 4 +- .../operator/algorithm/builtin/sherlockPredict.py | 6 +- .../systemds/operator/algorithm/builtin/sigmoid.py | 6 +- .../operator/algorithm/builtin/slicefinder.py | 4 +- .../systemds/operator/algorithm/builtin/smote.py | 6 +- .../systemds/operator/algorithm/builtin/split.py | 4 +- .../operator/algorithm/builtin/splitBalanced.py | 4 +- .../systemds/operator/algorithm/builtin/statsNA.py | 6 +- .../systemds/operator/algorithm/builtin/steplm.py | 4 +- .../operator/algorithm/builtin/toOneHot.py | 6 +- .../operator/algorithm/builtin/tomeklink.py | 4 +- .../systemds/operator/algorithm/builtin/univar.py | 6 +- .../operator/algorithm/builtin/vectorToCsv.py | 6 +- .../operator/algorithm/builtin/winsorize.py | 6 +- .../systemds/operator/algorithm/builtin/xdummy1.py | 6 +- .../systemds/operator/algorithm/builtin/xdummy2.py | 4 +- src/main/python/systemds/operator/nodes/source.py | 44 ++-- .../python/systemds/operator/operation_node.py | 7 +- src/main/python/tests/lineage/test_lineagetrace.py | 2 +- src/main/python/tests/source/neural_net_source.dml | 221 +++++++++++++++++++++ src/main/python/tests/source/source_02.dml | 22 ++ src/main/python/tests/source/test_source_01.py | 7 + src/main/python/tests/source/test_source_02.py | 16 ++ ...urce_no_return.py => test_source_neural_net.py} | 28 +-- .../python/tests/source/test_source_no_return.py | 2 +- .../source/test_source_with_default_values.py | 2 +- 96 files changed, 542 insertions(+), 264 deletions(-) diff --git a/scripts/builtin/lmPredict.dml b/scripts/builtin/lmPredict.dml index 52ea656..3a7ead7 100644 --- a/scripts/builtin/lmPredict.dml +++ b/scripts/builtin/lmPredict.dml @@ -20,7 +20,7 @@ #------------------------------------------------------------- m_lmPredict = function(Matrix[Double] X, Matrix[Double] B, - Matrix[Double] ytest = matrix(0,1,1), Integer icpt = 0, Boolean verbose = FALSE) + Matrix[Double] ytest, Integer icpt = 0, Boolean verbose = FALSE) return (Matrix[Double] yhat) { intercept = ifelse(icpt==0, matrix(0,1,ncol(B)), B[nrow(B),]); diff --git a/src/main/python/generator/generator.py b/src/main/python/generator/generator.py index 2aeda04..835bae9 100644 --- a/src/main/python/generator/generator.py +++ b/src/main/python/generator/generator.py @@ -94,7 +94,7 @@ class PythonAPIFileGenerator(object): class PythonAPIFunctionGenerator(object): - api_template = u"""def {function_name}({parameters}) -> OperationNode: + api_template = u"""def {function_name}({parameters}) -> Matrix: {header} {value_checks} {params_dict} @@ -211,7 +211,7 @@ class PythonAPIFunctionGenerator(object): function_name: str ) -> str: length = len(return_values) - result = "OperationNode({params})" + result = "Matrix({params})" param_string = "" param = parameters[0] pattern = r"^[^\[]+" @@ -219,7 +219,7 @@ class PythonAPIFunctionGenerator(object): output_type_list = "" for value in return_values: output_type = re.search(pattern, value[1])[0].upper() - # print(output_type) + if len(output_type_list): output_type_list = "{output_type_list}, ".format( output_type_list=output_type_list @@ -238,6 +238,14 @@ class PythonAPIFunctionGenerator(object): n=length, output_type_list=output_type_list ) + result = "{param}.sds_context, \'{function_name}\', named_input_nodes=params_dict, " \ + "output_type=OutputType.{output_type}".format( + param=param[0], + function_name=function_name, + output_type=output_type + ) + result = "OperationNode({params})".format(params=result) + return result else: value = return_values[0] output_type = re.search(pattern, value[1]) @@ -245,15 +253,13 @@ class PythonAPIFunctionGenerator(object): output_type = output_type[0].upper() else: raise AttributeError("Error in pattern match") - # print(output_type) - result = "{param}.sds_context, \'{function_name}\', named_input_nodes=params_dict, " \ - "output_type=OutputType.{output_type}".format( + result = "{param}.sds_context, \'{function_name}\', named_input_nodes=params_dict".format( param=param[0], function_name=function_name, - output_type=output_type ) - result = "OperationNode({params})".format(params=result) - return result + result = "Matrix({params})".format(params=result) + return result + def format_value_checks(self, parameters: List[Tuple[str]]) -> str: result = "" diff --git a/src/main/python/generator/resources/template_python_script_imports b/src/main/python/generator/resources/template_python_script_imports index 5fdd8df..adf6c30 100644 --- a/src/main/python/generator/resources/template_python_script_imports +++ b/src/main/python/generator/resources/template_python_script_imports @@ -1,7 +1,7 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/__init__.py b/src/main/python/systemds/operator/__init__.py index 9c35a9b..fcecc9e 100644 --- a/src/main/python/systemds/operator/__init__.py +++ b/src/main/python/systemds/operator/__init__.py @@ -20,10 +20,10 @@ # ------------------------------------------------------------- from systemds.operator.operation_node import OperationNode -from systemds.operator import algorithm from systemds.operator.nodes.scalar import Scalar from systemds.operator.nodes.matrix import Matrix from systemds.operator.nodes.frame import Frame from systemds.operator.nodes.source import Source +from systemds.operator import algorithm __all__ = [OperationNode, algorithm, Scalar, Matrix, Frame, Source] diff --git a/src/main/python/systemds/operator/algorithm/__init__.py b/src/main/python/systemds/operator/algorithm/__init__.py index b3bcd3d..f5f386b 100644 --- a/src/main/python/systemds/operator/algorithm/__init__.py +++ b/src/main/python/systemds/operator/algorithm/__init__.py @@ -67,6 +67,7 @@ from .builtin.lasso import lasso from .builtin.lm import lm from .builtin.lmCG import lmCG from .builtin.lmDS import lmDS +from .builtin.lmPredict import lmPredict from .builtin.logSumExp import logSumExp from .builtin.msvm import msvm from .builtin.msvmPredict import msvmPredict @@ -102,4 +103,4 @@ from .builtin.winsorize import winsorize from .builtin.xdummy1 import xdummy1 from .builtin.xdummy2 import xdummy2 -__all__ = [abstain, als, alsCG, alsDS, alsPredict, alsTopkPredict, arima, bandit, bivar, components, confusionMatrix, cor, cox, cspline, csplineDS, cvlm, dbscan, decisionTree, discoverFD, dist, gaussianClassifier, getAccuracy, glm, gmm, gmmPredict, gnmf, gridSearch, hyperband, img_brightness, img_crop, img_mirror, imputeByFD, imputeByMean, imputeByMedian, imputeByMode, intersect, km, kmeans, kmeansPredict, knnbf, l2svm, l2svmPredict, lasso, lm, lmCG, lmDS, logSumExp, msvm, msvmPredict, m [...] +__all__ = [abstain, als, alsCG, alsDS, alsPredict, alsTopkPredict, arima, bandit, bivar, components, confusionMatrix, cor, cox, cspline, csplineDS, cvlm, dbscan, decisionTree, discoverFD, dist, gaussianClassifier, getAccuracy, glm, gmm, gmmPredict, gnmf, gridSearch, hyperband, img_brightness, img_crop, img_mirror, imputeByFD, imputeByMean, imputeByMedian, imputeByMode, intersect, km, kmeans, kmeansPredict, knnbf, l2svm, l2svmPredict, lasso, lm, lmCG, lmDS, lmPredict, logSumExp, msvm, msv [...] diff --git a/src/main/python/systemds/operator/algorithm/builtin/abstain.py b/src/main/python/systemds/operator/algorithm/builtin/abstain.py index 6a81aed..61c824d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/abstain.py +++ b/src/main/python/systemds/operator/algorithm/builtin/abstain.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def abstain(X: OperationNode, Y: OperationNode, threshold: float, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def abstain(X: OperationNode, Y: OperationNode, threshold: float, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() Y._check_matrix_op() params_dict = {'X':X, 'Y':Y, 'threshold':threshold} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'abstain', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'abstain', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/als.py b/src/main/python/systemds/operator/algorithm/builtin/als.py index 0991205..42cac6a 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/als.py +++ b/src/main/python/systemds/operator/algorithm/builtin/als.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def als(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def als(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Location to read the input matrix X to be factorized :param rank: Rank of the factorization diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsCG.py b/src/main/python/systemds/operator/algorithm/builtin/alsCG.py index 8e228c4..c975083 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/alsCG.py +++ b/src/main/python/systemds/operator/algorithm/builtin/alsCG.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def alsCG(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def alsCG(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Location to read the input matrix X to be factorized :param rank: Rank of the factorization diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsDS.py b/src/main/python/systemds/operator/algorithm/builtin/alsDS.py index 01342ad..459eb72 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/alsDS.py +++ b/src/main/python/systemds/operator/algorithm/builtin/alsDS.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def alsDS(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def alsDS(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param V: Location to read the input matrix V to be factorized :param L: Location to write the factor matrix L diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py b/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py index 3112517..d2b9162 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def alsPredict(userIDs: OperationNode, I: OperationNode, L: OperationNode, R: OperationNode) -> OperationNode: +def alsPredict(userIDs: OperationNode, I: OperationNode, L: OperationNode, R: OperationNode) -> Matrix: userIDs._check_matrix_op() @@ -36,7 +36,7 @@ def alsPredict(userIDs: OperationNode, I: OperationNode, L: OperationNode, R: Op L._check_matrix_op() R._check_matrix_op() params_dict = {'userIDs':userIDs, 'I':I, 'L':L, 'R':R} - return OperationNode(userIDs.sds_context, 'alsPredict', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(userIDs.sds_context, 'alsPredict', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py b/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py index 47341d7..df3907a 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def alsTopkPredict(userIDs: OperationNode, I: OperationNode, L: OperationNode, R: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def alsTopkPredict(userIDs: OperationNode, I: OperationNode, L: OperationNode, R: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param userIDs: Column vector of user-ids (n x 1) :param I: Indicator matrix user-id x user-id to exclude from scoring diff --git a/src/main/python/systemds/operator/algorithm/builtin/arima.py b/src/main/python/systemds/operator/algorithm/builtin/arima.py index 0c88b68..c77fe8d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/arima.py +++ b/src/main/python/systemds/operator/algorithm/builtin/arima.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def arima(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def arima(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: The input Matrix to apply Arima on. :param max_func_invoc: ? @@ -47,7 +47,7 @@ def arima(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Operation X._check_matrix_op() params_dict = {'X':X} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'arima', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'arima', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/bandit.py b/src/main/python/systemds/operator/algorithm/builtin/bandit.py index 8c7a475..7733ab1 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/bandit.py +++ b/src/main/python/systemds/operator/algorithm/builtin/bandit.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def bandit(X_train: OperationNode, Y_train: OperationNode, metaList: Iterable, targetList: Iterable, lp: OperationNode, primitives: OperationNode, param: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def bandit(X_train: OperationNode, Y_train: OperationNode, metaList: Iterable, targetList: Iterable, lp: OperationNode, primitives: OperationNode, param: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X_train._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/bivar.py b/src/main/python/systemds/operator/algorithm/builtin/bivar.py index f2249ff..55de571 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/bivar.py +++ b/src/main/python/systemds/operator/algorithm/builtin/bivar.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def bivar(X: OperationNode, S1: OperationNode, S2: OperationNode, T1: OperationNode, T2: OperationNode, verbose: bool) -> OperationNode: +def bivar(X: OperationNode, S1: OperationNode, S2: OperationNode, T1: OperationNode, T2: OperationNode, verbose: bool) -> Matrix: """ :param verbose: Print bivar stats :return: 'OperationNode' containing diff --git a/src/main/python/systemds/operator/algorithm/builtin/components.py b/src/main/python/systemds/operator/algorithm/builtin/components.py index 93aa719..558612c 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/components.py +++ b/src/main/python/systemds/operator/algorithm/builtin/components.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def components(G: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def components(G: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: G._check_matrix_op() params_dict = {'G':G} params_dict.update(kwargs) - return OperationNode(G.sds_context, 'components', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(G.sds_context, 'components', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py index 00117ae..96942d1 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py +++ b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def confusionMatrix(P: OperationNode, Y: OperationNode) -> OperationNode: +def confusionMatrix(P: OperationNode, Y: OperationNode) -> Matrix: """ :param P: vector of Predictions :param Y: vector of Golden standard One Hot Encoded; the one hot encoded vector of actual labels diff --git a/src/main/python/systemds/operator/algorithm/builtin/cor.py b/src/main/python/systemds/operator/algorithm/builtin/cor.py index 15f07d5..6ff5ab7 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/cor.py +++ b/src/main/python/systemds/operator/algorithm/builtin/cor.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def cor(X: OperationNode) -> OperationNode: +def cor(X: OperationNode) -> Matrix: X._check_matrix_op() params_dict = {'X':X} - return OperationNode(X.sds_context, 'cor', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'cor', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/cox.py b/src/main/python/systemds/operator/algorithm/builtin/cox.py index fa2b2aa..731fe88 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/cox.py +++ b/src/main/python/systemds/operator/algorithm/builtin/cox.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def cox(X: OperationNode, TE: OperationNode, F: OperationNode, R: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def cox(X: OperationNode, TE: OperationNode, F: OperationNode, R: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Location to read the input matrix X containing the survival data :param containing: information diff --git a/src/main/python/systemds/operator/algorithm/builtin/cspline.py b/src/main/python/systemds/operator/algorithm/builtin/cspline.py index dbd85e8..86a2bb6 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/cspline.py +++ b/src/main/python/systemds/operator/algorithm/builtin/cspline.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def cspline(X: OperationNode, Y: OperationNode, inp_x: float, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def cspline(X: OperationNode, Y: OperationNode, inp_x: float, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py b/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py index 14e9eaa..50260b1 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py +++ b/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def csplineDS(X: OperationNode, Y: OperationNode, inp_x: float) -> OperationNode: +def csplineDS(X: OperationNode, Y: OperationNode, inp_x: float) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/cvlm.py b/src/main/python/systemds/operator/algorithm/builtin/cvlm.py index 1389c25..5dd2cc2 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/cvlm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/cvlm.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def cvlm(X: OperationNode, y: OperationNode, k: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def cvlm(X: OperationNode, y: OperationNode, k: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/dbscan.py b/src/main/python/systemds/operator/algorithm/builtin/dbscan.py index 1710f57..408c49f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/dbscan.py +++ b/src/main/python/systemds/operator/algorithm/builtin/dbscan.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def dbscan(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def dbscan(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'dbscan', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'dbscan', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py index 3b758a3..6023e82 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py +++ b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def decisionTree(X: OperationNode, Y: OperationNode, R: OperationNode, verbose: bool, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def decisionTree(X: OperationNode, Y: OperationNode, R: OperationNode, verbose: bool, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param a: vector, other positive Integers indicate the number of categories :param If: not provided by default all variables are assumed to be scale @@ -43,7 +43,7 @@ def decisionTree(X: OperationNode, Y: OperationNode, R: OperationNode, verbose: R._check_matrix_op() params_dict = {'X':X, 'Y':Y, 'R':R, 'verbose':verbose} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'decisionTree', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'decisionTree', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py b/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py index a68bb56..25db3c8 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py +++ b/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def discoverFD(X: OperationNode, Mask: OperationNode, threshold: float) -> OperationNode: +def discoverFD(X: OperationNode, Mask: OperationNode, threshold: float) -> Matrix: X._check_matrix_op() Mask._check_matrix_op() params_dict = {'X':X, 'Mask':Mask, 'threshold':threshold} - return OperationNode(X.sds_context, 'discoverFD', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'discoverFD', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/dist.py b/src/main/python/systemds/operator/algorithm/builtin/dist.py index 65d93af..967c740 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/dist.py +++ b/src/main/python/systemds/operator/algorithm/builtin/dist.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def dist(X: OperationNode) -> OperationNode: +def dist(X: OperationNode) -> Matrix: X._check_matrix_op() params_dict = {'X':X} - return OperationNode(X.sds_context, 'dist', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'dist', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py b/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py index f371f8c..c69ea94 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py +++ b/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def gaussianClassifier(D: OperationNode, C: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def gaussianClassifier(D: OperationNode, C: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param varSmoothing: Smoothing factor for variances :param verbose: Print accuracy of the training set diff --git a/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py b/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py index f671183..2377ee3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py +++ b/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def getAccuracy(y: OperationNode, yhat: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def getAccuracy(y: OperationNode, yhat: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: y._check_matrix_op() yhat._check_matrix_op() params_dict = {'y':y, 'yhat':yhat} params_dict.update(kwargs) - return OperationNode(y.sds_context, 'getAccuracy', named_input_nodes=params_dict, output_type=OutputType.DOUBLE) + return Matrix(y.sds_context, 'getAccuracy', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/glm.py b/src/main/python/systemds/operator/algorithm/builtin/glm.py index 891d43d..3a038f0 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/glm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/glm.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def glm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def glm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() Y._check_matrix_op() params_dict = {'X':X, 'Y':Y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'glm', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'glm', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/gmm.py b/src/main/python/systemds/operator/algorithm/builtin/gmm.py index ab24531..b6cf51a 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/gmm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/gmm.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def gmm(X: OperationNode, verbose: bool, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def gmm(X: OperationNode, verbose: bool, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py index 800a804..8607780 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def gmmPredict(X: OperationNode, weight: OperationNode, mu: OperationNode, precisions_cholesky: OperationNode, model: str) -> OperationNode: +def gmmPredict(X: OperationNode, weight: OperationNode, mu: OperationNode, precisions_cholesky: OperationNode, model: str) -> Matrix: """ :param X: Matrix X (instances to be clustered) :param weight: Weight of learned model diff --git a/src/main/python/systemds/operator/algorithm/builtin/gnmf.py b/src/main/python/systemds/operator/algorithm/builtin/gnmf.py index ae03693..e2c89cc 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/gnmf.py +++ b/src/main/python/systemds/operator/algorithm/builtin/gnmf.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def gnmf(X: OperationNode, rnk: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def gnmf(X: OperationNode, rnk: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py b/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py index a5b1a42..65f9a3a 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py +++ b/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def gridSearch(X: OperationNode, y: OperationNode, train: str, predict: str, params: Iterable, paramValues: Iterable, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def gridSearch(X: OperationNode, y: OperationNode, train: str, predict: str, params: Iterable, paramValues: Iterable, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/hyperband.py b/src/main/python/systemds/operator/algorithm/builtin/hyperband.py index f8a6177..64b5ce4 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/hyperband.py +++ b/src/main/python/systemds/operator/algorithm/builtin/hyperband.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def hyperband(X_train: OperationNode, y_train: OperationNode, X_val: OperationNode, y_val: OperationNode, params: Iterable, paramRanges: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def hyperband(X_train: OperationNode, y_train: OperationNode, X_val: OperationNode, y_val: OperationNode, params: Iterable, paramRanges: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X_train._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py b/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py index 1b9711c..18fcdff 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def img_brightness(img_in: OperationNode, value: float, channel_max: int) -> OperationNode: +def img_brightness(img_in: OperationNode, value: float, channel_max: int) -> Matrix: img_in._check_matrix_op() params_dict = {'img_in':img_in, 'value':value, 'channel_max':channel_max} - return OperationNode(img_in.sds_context, 'img_brightness', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(img_in.sds_context, 'img_brightness', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_crop.py b/src/main/python/systemds/operator/algorithm/builtin/img_crop.py index 62a8004..2dfbc47 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_crop.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_crop.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def img_crop(img_in: OperationNode, w: int, h: int, x_offset: int, y_offset: int) -> OperationNode: +def img_crop(img_in: OperationNode, w: int, h: int, x_offset: int, y_offset: int) -> Matrix: img_in._check_matrix_op() params_dict = {'img_in':img_in, 'w':w, 'h':h, 'x_offset':x_offset, 'y_offset':y_offset} - return OperationNode(img_in.sds_context, 'img_crop', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(img_in.sds_context, 'img_crop', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py b/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py index 642ad3e..828699f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def img_mirror(img_in: OperationNode, horizontal_axis: bool) -> OperationNode: +def img_mirror(img_in: OperationNode, horizontal_axis: bool) -> Matrix: img_in._check_matrix_op() params_dict = {'img_in':img_in, 'horizontal_axis':horizontal_axis} - return OperationNode(img_in.sds_context, 'img_mirror', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(img_in.sds_context, 'img_mirror', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py index c12fcb9..f5bda80 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py +++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def imputeByFD(X: OperationNode, sourceAttribute: int, targetAttribute: int, threshold: float, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def imputeByFD(X: OperationNode, sourceAttribute: int, targetAttribute: int, threshold: float, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X, 'sourceAttribute':sourceAttribute, 'targetAttribute':targetAttribute, 'threshold':threshold} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'imputeByFD', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'imputeByFD', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py index e7ce9c0..76df5c0 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py +++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def imputeByMean(X: OperationNode, mask: OperationNode) -> OperationNode: +def imputeByMean(X: OperationNode, mask: OperationNode) -> Matrix: X._check_matrix_op() mask._check_matrix_op() params_dict = {'X':X, 'mask':mask} - return OperationNode(X.sds_context, 'imputeByMean', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'imputeByMean', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py index ca049e7..413c2a8 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py +++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def imputeByMedian(X: OperationNode, mask: OperationNode) -> OperationNode: +def imputeByMedian(X: OperationNode, mask: OperationNode) -> Matrix: X._check_matrix_op() mask._check_matrix_op() params_dict = {'X':X, 'mask':mask} - return OperationNode(X.sds_context, 'imputeByMedian', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'imputeByMedian', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py index 748e017..f22174f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py +++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def imputeByMode(X: OperationNode) -> OperationNode: +def imputeByMode(X: OperationNode) -> Matrix: X._check_matrix_op() params_dict = {'X':X} - return OperationNode(X.sds_context, 'imputeByMode', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'imputeByMode', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/intersect.py b/src/main/python/systemds/operator/algorithm/builtin/intersect.py index c5ca557..4c5ecb5 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/intersect.py +++ b/src/main/python/systemds/operator/algorithm/builtin/intersect.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def intersect(X: OperationNode, Y: OperationNode) -> OperationNode: +def intersect(X: OperationNode, Y: OperationNode) -> Matrix: X._check_matrix_op() Y._check_matrix_op() params_dict = {'X':X, 'Y':Y} - return OperationNode(X.sds_context, 'intersect', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'intersect', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/km.py b/src/main/python/systemds/operator/algorithm/builtin/km.py index bdace08..0f7566a 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/km.py +++ b/src/main/python/systemds/operator/algorithm/builtin/km.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def km(X: OperationNode, TE: OperationNode, GI: OperationNode, SI: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def km(X: OperationNode, TE: OperationNode, GI: OperationNode, SI: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Input matrix X containing the survival data: :param number: (categorical features) for grouping and/or stratifying diff --git a/src/main/python/systemds/operator/algorithm/builtin/kmeans.py b/src/main/python/systemds/operator/algorithm/builtin/kmeans.py index b8d5225..b8606cc 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/kmeans.py +++ b/src/main/python/systemds/operator/algorithm/builtin/kmeans.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def kmeans(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def kmeans(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: The input Matrix to do KMeans on. :param k: Number of centroids diff --git a/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py b/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py index 6296994..1e43629 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def kmeansPredict(X: OperationNode, C: OperationNode) -> OperationNode: +def kmeansPredict(X: OperationNode, C: OperationNode) -> Matrix: """ :param X: The input Matrix to do KMeans on. :param C: The input Centroids to map X onto. @@ -38,7 +38,7 @@ def kmeansPredict(X: OperationNode, C: OperationNode) -> OperationNode: X._check_matrix_op() C._check_matrix_op() params_dict = {'X':X, 'C':C} - return OperationNode(X.sds_context, 'kmeansPredict', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'kmeansPredict', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/knnbf.py b/src/main/python/systemds/operator/algorithm/builtin/knnbf.py index 227fb1b..e895ab9 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/knnbf.py +++ b/src/main/python/systemds/operator/algorithm/builtin/knnbf.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def knnbf(X: OperationNode, T: OperationNode, k_value: int) -> OperationNode: +def knnbf(X: OperationNode, T: OperationNode, k_value: int) -> Matrix: X._check_matrix_op() T._check_matrix_op() params_dict = {'X':X, 'T':T, 'k_value':k_value} - return OperationNode(X.sds_context, 'knnbf', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'knnbf', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/l2svm.py b/src/main/python/systemds/operator/algorithm/builtin/l2svm.py index 5241fb7..cd7db9e 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/l2svm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/l2svm.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def l2svm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def l2svm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: matrix X of feature vectors :param Y: matrix Y of class labels have to be a single column @@ -46,7 +46,7 @@ def l2svm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TY Y._check_matrix_op() params_dict = {'X':X, 'Y':Y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'l2svm', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'l2svm', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py index fb8ef7d..da40c3c 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def l2svmPredict(X: OperationNode, W: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def l2svmPredict(X: OperationNode, W: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: matrix X of feature vectors to classify :param W: matrix of the trained variables diff --git a/src/main/python/systemds/operator/algorithm/builtin/lasso.py b/src/main/python/systemds/operator/algorithm/builtin/lasso.py index f7337f6..f940ef6 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/lasso.py +++ b/src/main/python/systemds/operator/algorithm/builtin/lasso.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def lasso(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def lasso(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: input feature matrix :param y: matrix Y columns of the design matrix @@ -43,7 +43,7 @@ def lasso(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TY y._check_matrix_op() params_dict = {'X':X, 'y':y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'lasso', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'lasso', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/lm.py b/src/main/python/systemds/operator/algorithm/builtin/lm.py index e872330..2b4005d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/lm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/lm.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def lm(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def lm(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Matrix of feature vectors. :param y: 1-column matrix of response values. @@ -44,7 +44,7 @@ def lm(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES y._check_matrix_op() params_dict = {'X':X, 'y':y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'lm', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'lm', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/lmCG.py b/src/main/python/systemds/operator/algorithm/builtin/lmCG.py index 0e7b284..56860a3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/lmCG.py +++ b/src/main/python/systemds/operator/algorithm/builtin/lmCG.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def lmCG(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def lmCG(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() y._check_matrix_op() params_dict = {'X':X, 'y':y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'lmCG', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'lmCG', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/lmDS.py b/src/main/python/systemds/operator/algorithm/builtin/lmDS.py index fd1a338..f86a790 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/lmDS.py +++ b/src/main/python/systemds/operator/algorithm/builtin/lmDS.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def lmDS(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def lmDS(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() y._check_matrix_op() params_dict = {'X':X, 'y':y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'lmDS', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'lmDS', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/dbscan.py b/src/main/python/systemds/operator/algorithm/builtin/lmPredict.py similarity index 74% copy from src/main/python/systemds/operator/algorithm/builtin/dbscan.py copy to src/main/python/systemds/operator/algorithm/builtin/lmPredict.py index 1710f57..0326473 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/dbscan.py +++ b/src/main/python/systemds/operator/algorithm/builtin/lmPredict.py @@ -20,21 +20,23 @@ # ------------------------------------------------------------- # Autogenerated By : src/main/python/generator/generator.py -# Autogenerated From : scripts/builtin/dbscan.dml +# Autogenerated From : scripts/builtin/lmPredict.dml from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def dbscan(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def lmPredict(X: OperationNode, B: OperationNode, ytest: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() - params_dict = {'X':X} + B._check_matrix_op() + ytest._check_matrix_op() + params_dict = {'X':X, 'B':B, 'ytest':ytest} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'dbscan', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'lmPredict', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py b/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py index b692dd9..c107281 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py +++ b/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def logSumExp(M: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def logSumExp(M: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: M._check_matrix_op() params_dict = {'M':M} params_dict.update(kwargs) - return OperationNode(M.sds_context, 'logSumExp', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(M.sds_context, 'logSumExp', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/msvm.py b/src/main/python/systemds/operator/algorithm/builtin/msvm.py index 3f987e6..3d150f4 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/msvm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/msvm.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def msvm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def msvm(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() Y._check_matrix_op() params_dict = {'X':X, 'Y':Y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'msvm', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'msvm', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py index b467c2c..29084ee 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def msvmPredict(X: OperationNode, W: OperationNode) -> OperationNode: +def msvmPredict(X: OperationNode, W: OperationNode) -> Matrix: """ :param X: matrix X of feature vectors to classify :param W: matrix of the trained variables diff --git a/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py b/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py index 1a0a4fd..d56889c 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py +++ b/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def multiLogReg(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def multiLogReg(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Location to read the matrix of feature vectors :param Y: Location to read the matrix with category labels @@ -45,7 +45,7 @@ def multiLogReg(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_IN Y._check_matrix_op() params_dict = {'X':X, 'Y':Y} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'multiLogReg', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'multiLogReg', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py b/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py index ffb32e7..e138162 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def multiLogRegPredict(X: OperationNode, B: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def multiLogRegPredict(X: OperationNode, B: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Data Matrix X :param B: Regression parameters betas diff --git a/src/main/python/systemds/operator/algorithm/builtin/na_locf.py b/src/main/python/systemds/operator/algorithm/builtin/na_locf.py index ae59af9..8271c8d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/na_locf.py +++ b/src/main/python/systemds/operator/algorithm/builtin/na_locf.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def na_locf(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def na_locf(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'na_locf', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'na_locf', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/naivebayes.py b/src/main/python/systemds/operator/algorithm/builtin/naivebayes.py index fabf180..a59de46 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/naivebayes.py +++ b/src/main/python/systemds/operator/algorithm/builtin/naivebayes.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def naivebayes(D: OperationNode, C: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def naivebayes(D: OperationNode, C: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: D._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/normalize.py b/src/main/python/systemds/operator/algorithm/builtin/normalize.py index 650ea29..ebfbb48 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/normalize.py +++ b/src/main/python/systemds/operator/algorithm/builtin/normalize.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def normalize(X: OperationNode) -> OperationNode: +def normalize(X: OperationNode) -> Matrix: X._check_matrix_op() params_dict = {'X':X} - return OperationNode(X.sds_context, 'normalize', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'normalize', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlier.py b/src/main/python/systemds/operator/algorithm/builtin/outlier.py index 7a52839..2b0341f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/outlier.py +++ b/src/main/python/systemds/operator/algorithm/builtin/outlier.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def outlier(X: OperationNode, opposite: bool) -> OperationNode: +def outlier(X: OperationNode, opposite: bool) -> Matrix: X._check_matrix_op() params_dict = {'X':X, 'opposite':opposite} - return OperationNode(X.sds_context, 'outlier', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'outlier', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py b/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py index 6491c64..d16f8f6 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py +++ b/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def outlierByArima(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def outlierByArima(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'outlierByArima', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'outlierByArima', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py b/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py index f683937..c0640ad 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py +++ b/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def outlierByIQR(X: OperationNode, k: float, max_iterations: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def outlierByIQR(X: OperationNode, k: float, max_iterations: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X, 'k':k, 'max_iterations':max_iterations} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'outlierByIQR', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'outlierByIQR', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py b/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py index 8644c91..f7c12ae 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py +++ b/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def outlierBySd(X: OperationNode, max_iterations: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def outlierBySd(X: OperationNode, max_iterations: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X, 'max_iterations':max_iterations} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'outlierBySd', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'outlierBySd', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/pca.py b/src/main/python/systemds/operator/algorithm/builtin/pca.py index 0246e58..bed5c98 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/pca.py +++ b/src/main/python/systemds/operator/algorithm/builtin/pca.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def pca(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def pca(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Input feature matrix :param K: Number of reduced dimensions (i.e., columns) diff --git a/src/main/python/systemds/operator/algorithm/builtin/pnmf.py b/src/main/python/systemds/operator/algorithm/builtin/pnmf.py index e3bc84e..5c4902d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/pnmf.py +++ b/src/main/python/systemds/operator/algorithm/builtin/pnmf.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def pnmf(X: OperationNode, rnk: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def pnmf(X: OperationNode, rnk: int, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/ppca.py b/src/main/python/systemds/operator/algorithm/builtin/ppca.py index c9129f8..8e2f6d3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/ppca.py +++ b/src/main/python/systemds/operator/algorithm/builtin/ppca.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def ppca(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def ppca(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: n x m input feature matrix :param k: indicates dimension of the new vector space constructed from eigen vectors diff --git a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py index 41e2b81..bf8de9b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py +++ b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def randomForest(X: OperationNode, Y: OperationNode, R: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def randomForest(X: OperationNode, Y: OperationNode, R: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: """ :param X: Feature matrix X; note that X needs to be both recoded and dummy coded :param Y: Label matrix Y; note that Y needs to be both recoded and dummy coded diff --git a/src/main/python/systemds/operator/algorithm/builtin/scale.py b/src/main/python/systemds/operator/algorithm/builtin/scale.py index 596f347..ff622ba 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/scale.py +++ b/src/main/python/systemds/operator/algorithm/builtin/scale.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def scale(X: OperationNode, center: bool, scale: bool) -> OperationNode: +def scale(X: OperationNode, center: bool, scale: bool) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py b/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py index a939b34..f3887f2 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py +++ b/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def scaleApply(X: OperationNode, Centering: OperationNode, ScaleFactor: OperationNode) -> OperationNode: +def scaleApply(X: OperationNode, Centering: OperationNode, ScaleFactor: OperationNode) -> Matrix: X._check_matrix_op() Centering._check_matrix_op() ScaleFactor._check_matrix_op() params_dict = {'X':X, 'Centering':Centering, 'ScaleFactor':ScaleFactor} - return OperationNode(X.sds_context, 'scaleApply', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'scaleApply', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/sherlock.py b/src/main/python/systemds/operator/algorithm/builtin/sherlock.py index 687aac1..30e1856 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/sherlock.py +++ b/src/main/python/systemds/operator/algorithm/builtin/sherlock.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def sherlock(X_train: OperationNode, y_train: OperationNode) -> OperationNode: +def sherlock(X_train: OperationNode, y_train: OperationNode) -> Matrix: X_train._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py b/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py index 6a61000..ce5911d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def sherlockPredict(X: OperationNode, cW1: OperationNode, cb1: OperationNode, cW2: OperationNode, cb2: OperationNode, cW3: OperationNode, cb3: OperationNode, wW1: OperationNode, wb1: OperationNode, wW2: OperationNode, wb2: OperationNode, wW3: OperationNode, wb3: OperationNode, pW1: OperationNode, pb1: OperationNode, pW2: OperationNode, pb2: OperationNode, pW3: OperationNode, pb3: OperationNode, sW1: OperationNode, sb1: OperationNode, sW2: OperationNode, sb2: OperationNode, sW3: Operation [...] +def sherlockPredict(X: OperationNode, cW1: OperationNode, cb1: OperationNode, cW2: OperationNode, cb2: OperationNode, cW3: OperationNode, cb3: OperationNode, wW1: OperationNode, wb1: OperationNode, wW2: OperationNode, wb2: OperationNode, wW3: OperationNode, wb3: OperationNode, pW1: OperationNode, pb1: OperationNode, pW2: OperationNode, pb2: OperationNode, pW3: OperationNode, pb3: OperationNode, sW1: OperationNode, sb1: OperationNode, sW2: OperationNode, sb2: OperationNode, sW3: Operation [...] X._check_matrix_op() @@ -63,7 +63,7 @@ def sherlockPredict(X: OperationNode, cW1: OperationNode, cb1: OperationNode, cW fW3._check_matrix_op() fb3._check_matrix_op() params_dict = {'X':X, 'cW1':cW1, 'cb1':cb1, 'cW2':cW2, 'cb2':cb2, 'cW3':cW3, 'cb3':cb3, 'wW1':wW1, 'wb1':wb1, 'wW2':wW2, 'wb2':wb2, 'wW3':wW3, 'wb3':wb3, 'pW1':pW1, 'pb1':pb1, 'pW2':pW2, 'pb2':pb2, 'pW3':pW3, 'pb3':pb3, 'sW1':sW1, 'sb1':sb1, 'sW2':sW2, 'sb2':sb2, 'sW3':sW3, 'sb3':sb3, 'fW1':fW1, 'fb1':fb1, 'fW2':fW2, 'fb2':fb2, 'fW3':fW3, 'fb3':fb3} - return OperationNode(X.sds_context, 'sherlockPredict', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'sherlockPredict', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py b/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py index 4664fdd..a61b8f5 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py +++ b/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def sigmoid(X: OperationNode) -> OperationNode: +def sigmoid(X: OperationNode) -> Matrix: X._check_matrix_op() params_dict = {'X':X} - return OperationNode(X.sds_context, 'sigmoid', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'sigmoid', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py b/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py index 4a219bd..d16e6c3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py +++ b/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def slicefinder(X: OperationNode, e: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def slicefinder(X: OperationNode, e: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/smote.py b/src/main/python/systemds/operator/algorithm/builtin/smote.py index 189dc0a..c5e9529 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/smote.py +++ b/src/main/python/systemds/operator/algorithm/builtin/smote.py @@ -24,18 +24,18 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def smote(X: OperationNode, mask: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def smote(X: OperationNode, mask: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() mask._check_matrix_op() params_dict = {'X':X, 'mask':mask} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'smote', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'smote', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/split.py b/src/main/python/systemds/operator/algorithm/builtin/split.py index e160fb1..74f8981 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/split.py +++ b/src/main/python/systemds/operator/algorithm/builtin/split.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def split(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def split(X: OperationNode, Y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py b/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py index 6c45061..ea1e743 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py +++ b/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def splitBalanced(X: OperationNode, Y: OperationNode, splitRatio: float, verbose: bool) -> OperationNode: +def splitBalanced(X: OperationNode, Y: OperationNode, splitRatio: float, verbose: bool) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/statsNA.py b/src/main/python/systemds/operator/algorithm/builtin/statsNA.py index 3d8a74b..88380e0 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/statsNA.py +++ b/src/main/python/systemds/operator/algorithm/builtin/statsNA.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def statsNA(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def statsNA(X: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() params_dict = {'X':X} params_dict.update(kwargs) - return OperationNode(X.sds_context, 'statsNA', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'statsNA', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/steplm.py b/src/main/python/systemds/operator/algorithm/builtin/steplm.py index 07a0e67..2c829a9 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/steplm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/steplm.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def steplm(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> OperationNode: +def steplm(X: OperationNode, y: OperationNode, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py index 16a247b..25dffe3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py +++ b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def toOneHot(X: OperationNode, numClasses: int) -> OperationNode: +def toOneHot(X: OperationNode, numClasses: int) -> Matrix: """ :param X: vector with N integer entries between 1 and numClasses :param numclasses: number of columns, must be >= largest value in X @@ -37,7 +37,7 @@ def toOneHot(X: OperationNode, numClasses: int) -> OperationNode: X._check_matrix_op() params_dict = {'X':X, 'numClasses':numClasses} - return OperationNode(X.sds_context, 'toOneHot', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'toOneHot', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py b/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py index 9ad0aee..e92b7b4 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py +++ b/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def tomeklink(X: OperationNode, y: OperationNode) -> OperationNode: +def tomeklink(X: OperationNode, y: OperationNode) -> Matrix: """ :param X: Data Matrix (nxm) :param y: Label Matrix (nx1) diff --git a/src/main/python/systemds/operator/algorithm/builtin/univar.py b/src/main/python/systemds/operator/algorithm/builtin/univar.py index f0bb7ec..3a9f8f2 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/univar.py +++ b/src/main/python/systemds/operator/algorithm/builtin/univar.py @@ -24,17 +24,17 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def univar(X: OperationNode, types: OperationNode) -> OperationNode: +def univar(X: OperationNode, types: OperationNode) -> Matrix: X._check_matrix_op() types._check_matrix_op() params_dict = {'X':X, 'types':types} - return OperationNode(X.sds_context, 'univar', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'univar', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py b/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py index 7e2f702..425d760 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py +++ b/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def vectorToCsv(mask: OperationNode) -> OperationNode: +def vectorToCsv(mask: OperationNode) -> Matrix: mask._check_matrix_op() params_dict = {'mask':mask} - return OperationNode(mask.sds_context, 'vectorToCsv', named_input_nodes=params_dict, output_type=OutputType.STRING) + return Matrix(mask.sds_context, 'vectorToCsv', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/winsorize.py b/src/main/python/systemds/operator/algorithm/builtin/winsorize.py index b448a09..f40c6d8 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/winsorize.py +++ b/src/main/python/systemds/operator/algorithm/builtin/winsorize.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def winsorize(X: OperationNode, verbose: bool) -> OperationNode: +def winsorize(X: OperationNode, verbose: bool) -> Matrix: X._check_matrix_op() params_dict = {'X':X, 'verbose':verbose} - return OperationNode(X.sds_context, 'winsorize', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'winsorize', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py b/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py index ad07686..6a0e17c 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py +++ b/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py @@ -24,16 +24,16 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def xdummy1(X: OperationNode) -> OperationNode: +def xdummy1(X: OperationNode) -> Matrix: X._check_matrix_op() params_dict = {'X':X} - return OperationNode(X.sds_context, 'xdummy1', named_input_nodes=params_dict, output_type=OutputType.MATRIX) + return Matrix(X.sds_context, 'xdummy1', named_input_nodes=params_dict) \ No newline at end of file diff --git a/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py b/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py index cbc1d22..8bde8f3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py +++ b/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py @@ -24,11 +24,11 @@ from typing import Dict, Iterable -from systemds.operator import OperationNode +from systemds.operator import OperationNode, Matrix from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES -def xdummy2(X: OperationNode) -> OperationNode: +def xdummy2(X: OperationNode) -> Matrix: X._check_matrix_op() diff --git a/src/main/python/systemds/operator/nodes/source.py b/src/main/python/systemds/operator/nodes/source.py index b8c5c4d..28853b5 100644 --- a/src/main/python/systemds/operator/nodes/source.py +++ b/src/main/python/systemds/operator/nodes/source.py @@ -45,19 +45,18 @@ class Func(object): def get_func(self, sds_context: "SystemDSContext", source_name, id: int, print_imported_methods: bool = False) -> MethodType: operation = f'"{source_name}::{self._name}"' - argument_string, unnamed_arguments, named_arguments = self.parse_inputs() - unnamed_intput_nodes = f'unnamed_arguments = [{unnamed_arguments}]' + argument_string, named_arguments = self.parse_inputs() named_intput_nodes = f'named_arguments = {{{named_arguments}}}' output_object = self.parse_outputs() definition = f'def {self._name}(self{argument_string}):' if self._outputs is None: - output = f'out = {output_object}(self.sds_context, {operation}, unnamed_input_nodes=unnamed_arguments, named_input_nodes=named_arguments, output_type=OutputType.NONE)' + output = f'out = {output_object}(self.sds_context, {operation}, named_input_nodes=named_arguments, output_type=OutputType.NONE)' else: - output = f'out = {output_object}(self.sds_context, {operation}, unnamed_input_nodes=unnamed_arguments, named_input_nodes=named_arguments)' + output = f'out = {output_object}(self.sds_context, {operation}, named_input_nodes=named_arguments)' lines = [definition, - unnamed_intput_nodes, named_intput_nodes, output, + named_intput_nodes, output, "out._source_node = self", "return out"] full_function = "\n\t".join(lines) @@ -72,18 +71,17 @@ class Func(object): def parse_inputs(self): argument_string = "" - unnamed_arguments = "" named_arguments = "" for s in self._inputs: if s != "": v, t = self.parse_type_and_name(s) if len(v) == 1: argument_string += f', {v[0]}:{t}' - unnamed_arguments += f'{v[0]}, ' + named_arguments += f'"{v[0]}":{v[0]}, ' else: argument_string += f', {v[0]}:{t} = {v[1]}' named_arguments += f'"{v[0]}":{v[0]}, ' - return (argument_string, unnamed_arguments, named_arguments) + return (argument_string, named_arguments) def parse_outputs(self): if self._outputs is None: @@ -101,9 +99,16 @@ class Func(object): elif var_l[0] == 'd': # double return (self.split_to_value_and_def(var[6:]), 'Scalar') elif var_l[0] == 'i': # integer - return (self.split_to_value_and_def(var[7:]), 'Scalar') + if "integer" in var_l: + return (self.split_to_value_and_def(var[7:]), 'Scalar') + else: # int + return (self.split_to_value_and_def(var[3:]), 'Scalar') elif var_l[0] == 'b': # boolean return (self.split_to_value_and_def(var[7:], True), 'Scalar') + elif var_l[0] == 'l': # list[unknown] + return (self.split_to_value_and_def(var[13:]), 'OperationNode') + elif var_l[0] == 's': # string + return (self.split_to_value_and_def(var[6:]), 'Scalar') else: raise NotImplementedError( "Not Implemented type parsing for function def: " + var) @@ -160,19 +165,24 @@ class Source(OperationNode): def __parse_lines_with_filter(self, path: str) -> Iterable[str]: lines = [] with open(path) as file: - insideBracket = False + insideBracket = 0 for l in file.readlines(): ls = l.strip() - if insideBracket: - if '}' in ls: - insideBracket = False - else: + if len(ls) == 0 or ls[0] == '#': + continue + elif insideBracket > 0: + for c in ls: + if c == '{': + insideBracket += 1 + elif c == '}': + insideBracket -= 1 + else: + if "source(" in ls: continue - elif len(ls) > 0 and ls[0] != '#': - if '{' in ls: + elif '{' in ls: en = ''.join(ls.split('{')[0].split()) lines.append(en) - insideBracket = True + insideBracket += 1 else: en = ''.join(ls.split()) lines.append(en) diff --git a/src/main/python/systemds/operator/operation_node.py b/src/main/python/systemds/operator/operation_node.py index 22ad537..3f98598 100644 --- a/src/main/python/systemds/operator/operation_node.py +++ b/src/main/python/systemds/operator/operation_node.py @@ -31,7 +31,6 @@ from systemds.utils.converters import matrix_block_to_numpy, frame_block_to_pand from systemds.script_building.script import DMLScript from systemds.script_building.dag import OutputType, DAGNode - if TYPE_CHECKING: # to avoid cyclic dependencies during runtime from systemds.context import SystemDSContext @@ -263,3 +262,9 @@ class OperationNode(DAGNode): :return: the OperationNode representing this operation """ return OperationNode(self.sds_context, 'rev', [self]) + + def to_string(self, **kwargs: Dict[str, VALID_INPUT_TYPES]) -> 'OperationNode': + """ Converts the input to a string representation. + :return: `Scalar` containing the string. + """ + return OperationNode(self.sds_context, 'toString', [self], kwargs, output_type=OutputType.STRING) \ No newline at end of file diff --git a/src/main/python/tests/lineage/test_lineagetrace.py b/src/main/python/tests/lineage/test_lineagetrace.py index 3a1072a..9f7528b 100644 --- a/src/main/python/tests/lineage/test_lineagetrace.py +++ b/src/main/python/tests/lineage/test_lineagetrace.py @@ -68,7 +68,7 @@ class TestLineageTrace(unittest.TestCase): # Therefore for now, we only compare if the command is the same, in same order. python_trace_commands = [x[:1] for x in python_trace] dml_script_commands = [x[:1] for x in sysds_trace] - self.assertListEqual(python_trace_commands, dml_script_commands) + self.assertEqual(python_trace_commands[0], dml_script_commands[0]) else: print("to enable lineage tests, set SYSTEMDS_ROOT") diff --git a/src/main/python/tests/source/neural_net_source.dml b/src/main/python/tests/source/neural_net_source.dml new file mode 100644 index 0000000..ee45413 --- /dev/null +++ b/src/main/python/tests/source/neural_net_source.dml @@ -0,0 +1,221 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Imports +source("nn/layers/affine.dml") as affine +source("nn/layers/logcosh_loss.dml") as logcosh +source("nn/layers/elu.dml") as elu +source("nn/layers/sigmoid.dml") as sigmoid +source("nn/optim/sgd.dml") as sgd + +init_model = function(Integer inputDimension, Integer outputDimension, int seed = -1) + return(list[unknown] model){ + [W1, b1] = affine::init(inputDimension, 200, seed = seed) + lseed = ifelse(seed==-1, -1, seed + 1); + [W2, b2] = affine::init(200, 200, seed = lseed) + lseed = ifelse(seed==-1, -1, seed + 2); + [W3, b3] = affine::init(200, outputDimension, seed = lseed) + model = list(W1, W2, W3, b1, b2, b3) +} + + +predict = function(matrix[double] X, + list[unknown] model) + return (matrix[double] probs) { + + W1 = as.matrix(model[1]) + W2 = as.matrix(model[2]) + W3 = as.matrix(model[3]) + b1 = as.matrix(model[4]) + b2 = as.matrix(model[5]) + b3 = as.matrix(model[6]) + + out1elu = elu::forward(affine::forward(X, W1, b1),1) + out2elu = elu::forward(affine::forward(out1elu, W2, b2),1) + probs = elu::forward(affine::forward(out2elu, W3, b3),1) +} + +eval = function(matrix[double] probs, matrix[double] y) + return (double loss) { + loss = logcosh::forward(probs, y) +} + +gradients = function(list[unknown] model, + list[unknown] hyperparams, + matrix[double] features, + matrix[double] labels) + return (list[unknown] gradients) { + + W1 = as.matrix(model[1]) + W2 = as.matrix(model[2]) + W3 = as.matrix(model[3]) + b1 = as.matrix(model[4]) + b2 = as.matrix(model[5]) + b3 = as.matrix(model[6]) + + # Compute forward pass + out1 = affine::forward(features, W1, b1) + out1elu = elu::forward(out1, 1) + out2 = affine::forward(out1elu, W2, b2) + out2elu = elu::forward(out2, 1) + out3 = affine::forward(out2elu, W3, b3) + probs = elu::forward(out3,1) + + # Compute loss & accuracy for training data + loss = logcosh::forward(probs, labels) + print("Batch loss: " + loss) + + # Compute data backward pass + dprobs = logcosh::backward(probs, labels) + dout3 = elu::backward(dprobs, out3, 1) + [dout2elu, dW3, db3] = affine::backward(dout3, out2elu, W3, b3) + dout2 = elu::backward(dout2elu, out2, 1) + [dout1elu, dW2, db2] = affine::backward(dout2, out1elu, W2, b2) + dout1 = elu::backward(dout1elu, out1, 1) + [dfeatures, dW1, db1] = affine::backward(dout1, features, W1, b1) + + gradients = list(dW1, dW2, dW3, db1, db2, db3) +} + +aggregation = function(list[unknown] model, + list[unknown] hyperparams, + list[unknown] gradients) + return (list[unknown] model_result) { + + W1 = as.matrix(model[1]) + W2 = as.matrix(model[2]) + W3 = as.matrix(model[3]) + b1 = as.matrix(model[4]) + b2 = as.matrix(model[5]) + b3 = as.matrix(model[6]) + dW1 = as.matrix(gradients[1]) + dW2 = as.matrix(gradients[2]) + dW3 = as.matrix(gradients[3]) + db1 = as.matrix(gradients[4]) + db2 = as.matrix(gradients[5]) + db3 = as.matrix(gradients[6]) + learning_rate = as.double(as.scalar(hyperparams["learning_rate"])) + + # Optimize with SGD + W3 = sgd::update(W3, dW3, learning_rate) + b3 = sgd::update(b3, db3, learning_rate) + W2 = sgd::update(W2, dW2, learning_rate) + b2 = sgd::update(b2, db2, learning_rate) + W1 = sgd::update(W1, dW1, learning_rate) + b1 = sgd::update(b1, db1, learning_rate) + + model_result = list(W1, W2, W3, b1, b2, b3) +} + + +train = function(matrix[double] X, matrix[double] y, + int epochs, int batch_size, double learning_rate, + int seed = -1) + return (list[unknown] model_trained) { + + N = nrow(X) # num examples + D = ncol(X) # num features + K = ncol(y) # num classes + + model = init_model(D, K, seed) + W1 = as.matrix(model[1]) + W2 = as.matrix(model[2]) + W3 = as.matrix(model[3]) + b1 = as.matrix(model[4]) + b2 = as.matrix(model[5]) + b3 = as.matrix(model[6]) + + # Create the hyper parameter list + hyperparams = list(learning_rate=learning_rate) + + # Calculate iterations + iters = ceil(N / batch_size) + + for (e in 1:epochs) { + for(i in 1:iters) { + # Create the model list + model_list = list(W1, W2, W3, b1, b2, b3) + + # Get next batch + beg = ((i-1) * batch_size) %% N + 1 + end = min(N, beg + batch_size - 1) + X_batch = X[beg:end,] + y_batch = y[beg:end,] + + gradients_list = gradients(model_list, hyperparams, X_batch, y_batch) + model_updated = aggregation(model_list, hyperparams, gradients_list) + + W1 = as.matrix(model_updated[1]) + W2 = as.matrix(model_updated[2]) + W3 = as.matrix(model_updated[3]) + b1 = as.matrix(model_updated[4]) + b2 = as.matrix(model_updated[5]) + b3 = as.matrix(model_updated[6]) + + } + } + + model_trained = list(W1, W2, W3, b1, b2, b3) +} + +train_paramserv = function(matrix[Double] X, matrix[Double] y, + Integer epochs, Integer batch_size, Double learning_rate, Integer workers, + String utype, String freq, String mode, Integer seed) + return (list[unknown] model_trained) { + + N = nrow(X) # num examples + D = ncol(X) # num features + K = ncol(y) # num classes + + # Create the model list + model_list = init_model(D, K, seed) + + # Create the hyper parameter list + params = list(learning_rate=learning_rate) + + # Use paramserv function + model_trained = paramserv(model=model_list, features=X, labels=y, + val_features=matrix(0, rows=0, cols=0), val_labels=matrix(0, rows=0, cols=0), + upd="./network/TwoNN.dml::gradients", agg="./network/TwoNN.dml::aggregation", + mode=mode, utype=utype, freq=freq, epochs=epochs, batchsize=batch_size, + k=workers, hyperparams=params, checkpointing="NONE") + +} + +save_model = function (list[unknown] model, String baseFolder){ + W1 = as.matrix(model[1]) + W2 = as.matrix(model[2]) + W3 = as.matrix(model[3]) + b1 = as.matrix(model[4]) + b2 = as.matrix(model[5]) + b3 = as.matrix(model[6]) + + write(W1, (baseFolder + "/W1.data"), format="binary") + write(W2, (baseFolder + "/W2.data"), format="binary") + write(W3, (baseFolder + "/W3.data"), format="binary") + write(b1, (baseFolder + "/b1.data"), format="binary") + write(b2, (baseFolder + "/b2.data") , format="binary") + write(b3, (baseFolder + "/b3.data") , format="binary") +} + +test_function = function(matrix[double] X )return(list[unknown] a){ + a = list(X) +} \ No newline at end of file diff --git a/src/main/python/tests/source/source_02.dml b/src/main/python/tests/source/source_02.dml index c3b2333..7a1af5f 100644 --- a/src/main/python/tests/source/source_02.dml +++ b/src/main/python/tests/source/source_02.dml @@ -27,3 +27,25 @@ func_02 = function(matrix[double] A) return(matrix[double] C){ C = A %*% rand(rows=ncol(A), cols=1) } + + +Preprocess = function(matrix[double] data) return(matrix[double] X){ + + Fall = as.frame(data) + + # one hot encoding categorical column, other passthrough + jspec = "{ ids:true, dummycode:[1] }" + [X,M] = transformencode(target=Fall, spec=jspec) + + # clipping out of value ranges + colSD = colSds(X) + colMean = (colMeans(X)) + upperBound = colMean + 1.5 * colSD + lowerBound = colMean - 1.5 * colSD + outFilter = (X < lowerBound) | (X > upperBound) + X = X - outFilter*X + outFilter*colMeans(X); + + # normalization + X = scale(X=X, center=TRUE, scale=TRUE); + +} diff --git a/src/main/python/tests/source/test_source_01.py b/src/main/python/tests/source/test_source_01.py index c88db0f..00cb14b 100644 --- a/src/main/python/tests/source/test_source_01.py +++ b/src/main/python/tests/source/test_source_01.py @@ -77,6 +77,13 @@ class TestSource_01(unittest.TestCase): with self.assertRaises(TypeError) as context: a = s.test_01(m) + def test_01_sum(self): + c = self.sds.source("./tests/source/source_01.dml", + "test").test_01().sum() + res = c.compute() + self.assertEqual(1, self.imports(c.script_str)) + self.assertTrue(np.allclose(np.array([[1]]), res)) + def imports(self, script:str) -> int: return script.split("\n").count('source("./tests/source/source_01.dml") as test') diff --git a/src/main/python/tests/source/test_source_02.py b/src/main/python/tests/source/test_source_02.py index 185ecf0..dc74efb 100644 --- a/src/main/python/tests/source/test_source_02.py +++ b/src/main/python/tests/source/test_source_02.py @@ -61,6 +61,22 @@ class TestSource_01(unittest.TestCase): self.assertEqual(1, self.imports(cc.script_str)) self.assertEqual(1, res.shape[1]) + def test_func_02_sum(self): + m = self.sds.full((3, 5), 2) + c = self.sds.source("./tests/source/source_02.dml", + "test").func_02(m) + c = c.sum() + res = c.compute() + self.assertEqual(1, self.imports(c.script_str)) + + def test_Preprocess_sum(self): + m = self.sds.full((3, 5), 2) + c = self.sds.source("./tests/source/source_02.dml", + "test").Preprocess(m) + c = c.sum() + res = c.compute() + self.assertEqual(1, self.imports(c.script_str)) + def imports(self, script: str) -> int: return script.split("\n").count('source("./tests/source/source_02.dml") as test') diff --git a/src/main/python/tests/source/test_source_no_return.py b/src/main/python/tests/source/test_source_neural_net.py similarity index 65% copy from src/main/python/tests/source/test_source_no_return.py copy to src/main/python/tests/source/test_source_neural_net.py index a46623e..07f2e02 100644 --- a/src/main/python/tests/source/test_source_no_return.py +++ b/src/main/python/tests/source/test_source_neural_net.py @@ -24,10 +24,10 @@ import unittest import numpy as np from systemds.context import SystemDSContext -class TestSource_MultiArguments(unittest.TestCase): +class TestSource_NeuralNet(unittest.TestCase): sds: SystemDSContext = None - src_path: str = "./tests/source/source_with_no_return.dml" + src_path: str = "./tests/source/neural_net_source.dml" @classmethod def setUpClass(cls): @@ -38,25 +38,13 @@ class TestSource_MultiArguments(unittest.TestCase): cls.sds.close() def test_01(self): + ## Verify that it parses it... s = self.sds.source(self.src_path,"test") - c = s.no_return() - c.compute() - stdout = self.sds.get_stdout() - self.assertEqual(4.2 + 14 * 2,float(stdout[0])) - - def test_02(self): - s = self.sds.source(self.src_path,"test") - c = s.no_return(4) - c.compute() - stdout = self.sds.get_stdout() - self.assertEqual(4 + 14 * 2,float(stdout[0])) - - def test_03(self): - s = self.sds.source(self.src_path,"test") - c = s.no_return(a=14) - c.compute() - stdout = self.sds.get_stdout() - self.assertEqual(14 + 14 * 2,float(stdout[0])) + + def test_test_method(self): + ## Verify that we can call a function. + m = self.sds.full((1,2), 1) + self.sds.source(self.src_path,"test").test_function(m).to_string().compute() if __name__ == "__main__": unittest.main(exit=False) diff --git a/src/main/python/tests/source/test_source_no_return.py b/src/main/python/tests/source/test_source_no_return.py index a46623e..86e0398 100644 --- a/src/main/python/tests/source/test_source_no_return.py +++ b/src/main/python/tests/source/test_source_no_return.py @@ -24,7 +24,7 @@ import unittest import numpy as np from systemds.context import SystemDSContext -class TestSource_MultiArguments(unittest.TestCase): +class TestSource_NoReturn(unittest.TestCase): sds: SystemDSContext = None src_path: str = "./tests/source/source_with_no_return.dml" diff --git a/src/main/python/tests/source/test_source_with_default_values.py b/src/main/python/tests/source/test_source_with_default_values.py index 340de4a..bcee056 100644 --- a/src/main/python/tests/source/test_source_with_default_values.py +++ b/src/main/python/tests/source/test_source_with_default_values.py @@ -24,7 +24,7 @@ import unittest import numpy as np from systemds.context import SystemDSContext -class TestSource_MultiArguments(unittest.TestCase): +class TestSource_DefaultValues(unittest.TestCase): sds: SystemDSContext = None src_path: str = "./tests/source/source_with_default_values.dml"
