[1/4] systemml git commit: [MINOR] Performance createvar instruction, cleanup data handles
Repository: systemml Updated Branches: refs/heads/master d69686273 -> 6a11413b1 http://git-wip-us.apache.org/repos/asf/systemml/blob/ffefd8e6/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java -- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java index d69837f..f283b69 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java @@ -60,7 +60,7 @@ public class TaskPartitionerFixedsize extends TaskPartitioner for( long i = lFrom; i<=lTo; ) { //create new task and add to list of tasks - Task lTask = new Task( type ); + Task lTask = new Task(_iterVarName, type); tasks.addLast(lTask); int corr = (lfnp1-- > 0)? 1:0; //correction for static partitioner @@ -71,9 +71,7 @@ public class TaskPartitionerFixedsize extends TaskPartitioner { //value based tasks for( long j=0; j<_taskSize+corr && i<=lTo; j++, i+=lIncr ) - { - lTask.addIteration(new IntObject(_iterVarName, i)); - } + lTask.addIteration(new IntObject(i)); } else { @@ -81,9 +79,9 @@ public class TaskPartitionerFixedsize extends TaskPartitioner long to = Math.min( i+(_taskSize-1+corr)*lIncr, lTo ); //range based tasks - lTask.addIteration(new IntObject(_iterVarName, i)); //from - lTask.addIteration(new IntObject(_iterVarName, to));//to - lTask.addIteration(new IntObject(_iterVarName, lIncr)); //increment + lTask.addIteration(new IntObject(i)); //from + lTask.addIteration(new IntObject(to));//to + lTask.addIteration(new IntObject(lIncr)); //increment i = to + lIncr; } @@ -112,7 +110,7 @@ public class TaskPartitionerFixedsize extends TaskPartitioner for( long i = lFrom; i<=lTo; ) { //create new task and add to list of tasks - Task lTask = new Task( type ); + Task lTask = new Task(_iterVarName, type); int corr = (lfnp1-- > 0)? 1:0; //correction for static partitioner @@ -122,9 +120,7 @@ public class TaskPartitionerFixedsize extends TaskPartitioner { //value based tasks for( long j=0; j<_taskSize+corr && i<=lTo; j++, i+=lIncr ) - { - lTask.addIteration(new IntObject(_iterVarName, i)); - } + lTask.addIteration(new IntObject(i)); } else { @@ -132,9 +128,9 @@ public class TaskPartitionerFixedsize extends TaskPartitioner long to = Math.min( i+(_taskSize-1+corr)*lIncr, lTo ); //range based tasks - lTask.addIteration(new IntObject(_iterVarName, i)); //from - lTask.addIteration(new IntObject(_iterVarName, to));//to - lTask.addIteration(new IntObject(_iterVarName, lIncr)); //increment + lTask.addIteration(new IntObject(i)); //from + lTask.addIteration(new IntObject(to)); //to + lTask.addIteration(new IntObject(lIncr)); //increment i = to + lIncr;
[2/4] systemml git commit: [MINOR] Performance createvar instruction, cleanup data handles
[MINOR] Performance createvar instruction, cleanup data handles This patch improves the performance of string concatenation for createvar instructions with unique filenames and removes the unnecessary variable name from all data objects (matrices, frames, scalars). Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ffefd8e6 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ffefd8e6 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ffefd8e6 Branch: refs/heads/master Commit: ffefd8e68defb7eb5412f92ddfd8e48046e6532c Parents: c961432 Author: Matthias Boehm Authored: Thu Nov 9 20:03:47 2017 -0800 Committer: Matthias Boehm Committed: Thu Nov 9 22:08:02 2017 -0800 -- .../apache/sysml/api/jmlc/PreparedScript.java | 8 +- .../api/mlcontext/MLContextConversionUtil.java | 138 +++ .../sysml/api/mlcontext/MLContextUtil.java | 38 ++--- .../runtime/controlprogram/ForProgramBlock.java | 12 +- .../controlprogram/ParForProgramBlock.java | 16 +-- .../runtime/controlprogram/ProgramBlock.java| 8 +- .../controlprogram/caching/CacheableData.java | 42 +++--- .../controlprogram/caching/FrameObject.java | 7 +- .../controlprogram/caching/MatrixObject.java| 13 +- .../context/ExecutionContext.java | 5 +- .../context/SparkExecutionContext.java | 22 +-- .../controlprogram/parfor/DataPartitioner.java | 9 +- .../controlprogram/parfor/ParWorker.java| 11 +- .../controlprogram/parfor/ProgramConverter.java | 12 +- .../parfor/RemoteDPParForSparkWorker.java | 4 +- .../parfor/RemoteDPParWorkerReducer.java| 4 +- .../parfor/ResultMergeLocalFile.java| 59 .../parfor/ResultMergeLocalMemory.java | 26 ++-- .../parfor/ResultMergeRemoteMR.java | 10 +- .../parfor/ResultMergeRemoteSpark.java | 20 ++- .../runtime/controlprogram/parfor/Task.java | 31 +++-- .../parfor/TaskPartitionerFactoring.java| 22 ++- .../parfor/TaskPartitionerFixedsize.java| 24 ++-- .../cp/AggregateUnaryCPInstruction.java | 8 +- .../runtime/instructions/cp/BooleanObject.java | 8 +- .../cp/CentralMomentCPInstruction.java | 3 +- .../cp/CovarianceCPInstruction.java | 4 +- .../sysml/runtime/instructions/cp/Data.java | 17 +-- .../runtime/instructions/cp/DoubleObject.java | 6 +- .../cp/FunctionCallCPInstruction.java | 2 - .../runtime/instructions/cp/IntObject.java | 8 +- .../runtime/instructions/cp/ScalarObject.java | 11 +- .../runtime/instructions/cp/StringObject.java | 8 +- .../cp/UaggOuterChainCPInstruction.java | 2 +- .../instructions/cp/VariableCPInstruction.java | 11 +- .../cpfile/MatrixIndexingCPFileInstruction.java | 7 +- .../ParameterizedBuiltinCPFileInstruction.java | 13 +- .../spark/CentralMomentSPInstruction.java | 3 +- .../spark/CheckpointSPInstruction.java | 2 +- .../spark/CovarianceSPInstruction.java | 3 +- .../spark/data/BroadcastObject.java | 4 +- .../instructions/spark/data/DatasetObject.java | 8 +- .../instructions/spark/data/LineageObject.java | 10 +- .../instructions/spark/data/RDDObject.java | 4 +- .../ExternalFunctionInvocationInstruction.java | 12 +- 45 files changed, 275 insertions(+), 420 deletions(-) -- http://git-wip-us.apache.org/repos/asf/systemml/blob/ffefd8e6/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java -- diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java index 3dc0db7..dac211e 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java +++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java @@ -142,7 +142,7 @@ public class PreparedScript * @throws DMLException if DMLException occurs */ public void setScalar(String varname, boolean scalar, boolean reuse) throws DMLException { - setScalar(varname, new BooleanObject(varname, scalar), reuse); + setScalar(varname, new BooleanObject(scalar), reuse); } /** @@ -165,7 +165,7 @@ public class PreparedScript * @throws DMLException if DMLException occurs */ public void setScalar(String varname, long scalar, boolean reuse) throws DMLException { - setScalar(varname, new IntObject(varname, scalar), reuse); + setScalar(varname, new IntObject(scalar), reuse); } /** Binds a scalar double to a registered input variable. @@ -187,7 +187,7 @@ public class PreparedScript *
[4/4] systemml git commit: [SYSTEMML-2009] Performance multi-threaded JMLC scoring (contention)
[SYSTEMML-2009] Performance multi-threaded JMLC scoring (contention) This patch makes a major performance improvement to multi-threaded JMLC scoring and script execution. For scripts with many small intermediates, concurrent JMLC prepared scripts were contended on sequence ID generators for unique matrix IDs, which is unnecessary because these IDs are only used in the bufferpool which is disabled through JMLC. On an end-to-end application with above characteristics, this changed improved performance from 487s to 94s (in a test environment w/ 24 vcores). Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/6a11413b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/6a11413b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/6a11413b Branch: refs/heads/master Commit: 6a11413b13d88712745ba88a7f58f5544a1018e4 Parents: ffefd8e Author: Matthias Boehm Authored: Thu Nov 9 21:46:06 2017 -0800 Committer: Matthias Boehm Committed: Thu Nov 9 22:08:03 2017 -0800 -- .../org/apache/sysml/api/jmlc/Connection.java | 1 + .../org/apache/sysml/conf/CompilerConfig.java | 2 ++ .../controlprogram/caching/CacheableData.java | 20 .../instructions/cp/VariableCPInstruction.java | 9 - 4 files changed, 19 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/systemml/blob/6a11413b/src/main/java/org/apache/sysml/api/jmlc/Connection.java -- diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java index 705d784..be75a95 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java @@ -112,6 +112,7 @@ public class Connection implements Closeable CompilerConfig cconf = new CompilerConfig(); cconf.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true); cconf.set(ConfigType.IGNORE_READ_WRITE_METADATA, true); + cconf.set(ConfigType.IGNORE_TEMPORARY_FILENAMES, true); cconf.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false); cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false); cconf.set(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, false); http://git-wip-us.apache.org/repos/asf/systemml/blob/6a11413b/src/main/java/org/apache/sysml/conf/CompilerConfig.java -- diff --git a/src/main/java/org/apache/sysml/conf/CompilerConfig.java b/src/main/java/org/apache/sysml/conf/CompilerConfig.java index bafe0ce..8c2c672 100644 --- a/src/main/java/org/apache/sysml/conf/CompilerConfig.java +++ b/src/main/java/org/apache/sysml/conf/CompilerConfig.java @@ -70,6 +70,7 @@ public class CompilerConfig //Data expression configuration (modified by mlcontext, jmlc apis); no read of meta //data on mlcontext (local) /jmlc (global); ignore unknowns on jmlc IGNORE_READ_WRITE_METADATA, // global skip meta data reads + IGNORE_TEMPORARY_FILENAMES, // global skip temporary filename modifications REJECT_READ_WRITE_UNKNOWNS, // ignore missing meta data MLCONTEXT, // execution via new MLContext @@ -99,6 +100,7 @@ public class CompilerConfig _bmap.put(ConfigType.ALLOW_CSE_PERSISTENT_READS, true); _bmap.put(ConfigType.IGNORE_UNSPECIFIED_ARGS, false); _bmap.put(ConfigType.IGNORE_READ_WRITE_METADATA, false); + _bmap.put(ConfigType.IGNORE_TEMPORARY_FILENAMES, false); _bmap.put(ConfigType.REJECT_READ_WRITE_UNKNOWNS, true); _bmap.put(ConfigType.MLCONTEXT, false); _bmap.put(ConfigType.CODEGEN_ENABLED, false); http://git-wip-us.apache.org/repos/asf/systemml/blob/6a11413b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java -- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java index 885fedb..9e787de 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java @@ -110,7 +110,7 @@ public abstract class CacheableData extends Data } /** Global flag indicating if caching is enabled (controls eviction) */ - private static boolean _activeFlag = false; + private static volatile boolean _activeFlag = false; /** Global seq
[3/4] systemml git commit: [SYSTEMML-1990] Generalized ctable rewrites (seq-table, const inputs)
[SYSTEMML-1990] Generalized ctable rewrites (seq-table, const inputs) This patch generalized the existing rewrite for table(seq(),X,...) to rexpand(X,...) to handle cases with unknown dimensions, including common scenarios with column indexing on X. Additionally, this patch also introduces a new rewrite for table with constant matrix inputs (i.e., table(X, matrix(7)) -> table(X,7)). Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/c9614324 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/c9614324 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/c9614324 Branch: refs/heads/master Commit: c96143248349b6c68253ef9b3777afd5e5ed62f2 Parents: d696862 Author: Matthias Boehm Authored: Thu Nov 9 16:31:58 2017 -0800 Committer: Matthias Boehm Committed: Thu Nov 9 22:08:02 2017 -0800 -- .../sysml/hops/rewrite/HopRewriteUtils.java | 27 ++- .../RewriteAlgebraicSimplificationDynamic.java | 11 ++- .../RewriteAlgebraicSimplificationStatic.java | 22 +- .../misc/RewriteCTableToRExpandTest.java| 83 ++-- .../RewriteCTableToRExpandLeftUnknownPos.dml| 28 +++ .../RewriteCTableToRExpandRightUnknownPos.dml | 28 +++ 6 files changed, 167 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/systemml/blob/c9614324/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java -- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java index 28b2189..66f4fc7 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java @@ -965,6 +965,15 @@ public class HopRewriteUtils || isLiteralOfValue(hop.getInput().get(1), val)); } + public static boolean isTernary(Hop hop, OpOp3 type) { + return hop instanceof TernaryOp && ((TernaryOp)hop).getOp()==type; + } + + public static boolean isTernary(Hop hop, OpOp3... types) { + return ( hop instanceof TernaryOp + && ArrayUtils.contains(types, ((TernaryOp) hop).getOp())); + } + public static boolean containsInput(Hop current, Hop probe) { return rContainsInput(current, probe, new HashSet()); } @@ -1052,6 +1061,15 @@ public class HopRewriteUtils return true; } + public static boolean isColumnRightIndexing(Hop hop) { + return hop instanceof IndexingOp + && ((IndexingOp) hop).isColLowerEqualsUpper() + && ((hop.dimsKnown() && hop.getDim1() == hop.getInput().get(0).getDim1()) + || (isLiteralOfValue(hop.getInput().get(1), 1) + && isUnary(hop.getInput().get(2), OpOp1.NROW) + && hop.getInput().get(2).getInput().get(0)==hop.getInput().get(0))); + } + public static boolean isFullColumnIndexing(LeftIndexingOp hop) { return hop.isColLowerEqualsUpper() && isLiteralOfValue(hop.getInput().get(2), 1) @@ -1112,9 +1130,7 @@ public class HopRewriteUtils Hop to = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_TO)); Hop incr = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_INCR)); return isLiteralOfValue(from, 1) && isLiteralOfValue(incr, 1) - && (isLiteralOfValue(to, row?input.getDim1():input.getDim2()) - || (to instanceof UnaryOp && ((UnaryOp)to).getOp()==(row? - OpOp1.NROW:OpOp1.NCOL) && to.getInput().get(0)==input)); + && isSizeExpressionOf(to, input, row); } return false; } @@ -1149,6 +1165,11 @@ public class HopRewriteUtils throw new HopsException("Failed to retrieve 'to' argument from basic 1-N sequence."); } + public static boolean isSizeExpressionOf(Hop size, Hop input, boolean row) { + return (input.dimsKnown() && isLiteralOfValue(size, row?input.getDim1():input.getDim2())) + || ((row ? isUnary(size, OpOp1.NROW) : isUnary(size, OpOp1.NCOL)) && (size.getInput().get(0)==input + || (isColumnRightIndexing(input) && size.getInput().get(0)==input.getInput().get(0; + } public static boolean hasOnlyWriteParents( Hop hop, boolean inclTransient, boolean inclPersistent
systemml git commit: [SYSTEMML-1978] Add PCA to Performance Test Suite
Repository: systemml Updated Branches: refs/heads/master bd139a575 -> d69686273 [SYSTEMML-1978] Add PCA to Performance Test Suite Closes #694 Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d6968627 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d6968627 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d6968627 Branch: refs/heads/master Commit: d69686273da8bf4dc09441ec34ef3863eb437629 Parents: bd139a5 Author: Krishna Kalyan Authored: Thu Nov 9 20:42:49 2017 +0100 Committer: Krishna Kalyan Committed: Thu Nov 9 20:42:49 2017 +0100 -- scripts/perftest/python/datagen.py | 17 + scripts/perftest/python/run_perftest.py | 11 +++ scripts/perftest/python/train.py| 15 +++ 3 files changed, 39 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/systemml/blob/d6968627/scripts/perftest/python/datagen.py -- diff --git a/scripts/perftest/python/datagen.py b/scripts/perftest/python/datagen.py index 54f2eff..55dd06d 100755 --- a/scripts/perftest/python/datagen.py +++ b/scripts/perftest/python/datagen.py @@ -215,6 +215,23 @@ def stats2_datagen(matrix_dim, matrix_type, datagen_dir, config_dir): return save_path +def dimreduction_datagen(matrix_dim, matrix_type, datagen_dir, config_dir): + +path_name = '.'.join(['dimreduction', matrix_type, str(matrix_dim)]) +datagen_write = join(datagen_dir, path_name) +save_path = join(config_dir, path_name) +row, col = split_rowcol(matrix_dim) + +R = row +C = col +OUT = join(datagen_write, 'X.data') + +config = dict(R=R, C=C, OUT=OUT, FMT=DATA_FORMAT) + +config_writer(save_path + '.json', config) +return save_path + + def config_packets_datagen(algo_payload, matrix_type, matrix_shape, datagen_dir, dense_algos, config_dir): """ This function has two responsibilities. Generate the configuration files for http://git-wip-us.apache.org/repos/asf/systemml/blob/d6968627/scripts/perftest/python/run_perftest.py -- diff --git a/scripts/perftest/python/run_perftest.py b/scripts/perftest/python/run_perftest.py index 6e87261..1f78a75 100755 --- a/scripts/perftest/python/run_perftest.py +++ b/scripts/perftest/python/run_perftest.py @@ -47,7 +47,8 @@ ML_ALGO = {'binomial': ['MultiLogReg', 'l2-svm', 'm-svm'], 'regression1': ['LinearRegDS', 'LinearRegCG'], 'regression2': ['GLM_poisson', 'GLM_gamma', 'GLM_binomial'], 'stats1': ['Univar-Stats', 'bivar-stats'], - 'stats2': ['stratstats']} + 'stats2': ['stratstats'], + 'dimreduction': ['PCA']} ML_GENDATA = {'binomial': 'genRandData4LogisticRegression', 'clustering': 'genRandData4Kmeans', @@ -55,7 +56,8 @@ ML_GENDATA = {'binomial': 'genRandData4LogisticRegression', 'regression1': 'genRandData4LogisticRegression', 'regression2': 'genRandData4LogisticRegression', 'stats1': 'genRandData4DescriptiveStats', - 'stats2': 'genRandData4StratStats'} + 'stats2': 'genRandData4StratStats', + 'dimreduction': 'genRandData4PCA'} ML_TRAIN = {'GLM_poisson': 'GLM', 'GLM_gamma': 'GLM', @@ -69,7 +71,8 @@ ML_TRAIN = {'GLM_poisson': 'GLM', 'm-svm': 'm-svm', 'l2-svm': 'l2-svm', 'MultiLogReg': 'MultiLogReg', -'naive-bayes': 'naive-bayes'} +'naive-bayes': 'naive-bayes', +'PCA': 'PCA'} ML_PREDICT = {'Kmeans': 'Kmeans-predict', 'LinearRegCG': 'GLM-predict', @@ -82,7 +85,7 @@ ML_PREDICT = {'Kmeans': 'Kmeans-predict', 'GLM_gamma': 'GLM-predict', 'GLM_binomial': 'GLM-predict'} -DENSE_TYPE_ALGOS = ['clustering', 'stats1', 'stats2'] +DENSE_TYPE_ALGOS = ['clustering', 'stats1', 'stats2', 'dimreduction'] # Responsible for execution and metric logging http://git-wip-us.apache.org/repos/asf/systemml/blob/d6968627/scripts/perftest/python/train.py -- diff --git a/scripts/perftest/python/train.py b/scripts/perftest/python/train.py index 4428e8f..907c2b9 100755 --- a/scripts/perftest/python/train.py +++ b/scripts/perftest/python/train.py @@ -338,6 +338,21 @@ def regression2_glm_poisson_train(save_folder_name, datagen_dir, train_dir, conf return data_folders +def dimreduction_pca_train(save_folder_name, datagen_dir, train_dir, config_dir): +save_path = join(config_dir, save_folder_name) +train_write = join(train_dir, save_folder_name) + +INPUT = join(datagen_dir, 'X.data') +SCALE = '1' +
systemml git commit: [HOTFIX][SYSTEMML-1990] Fix cbind/rbind folding test (exec type)
Repository: systemml Updated Branches: refs/heads/master 11723cde8 -> bd139a575 [HOTFIX][SYSTEMML-1990] Fix cbind/rbind folding test (exec type) Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/bd139a57 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/bd139a57 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/bd139a57 Branch: refs/heads/master Commit: bd139a575de8a57af40b5f0e8ae10403cd3c060e Parents: 11723cd Author: Matthias Boehm Authored: Thu Nov 9 11:39:52 2017 -0800 Committer: Matthias Boehm Committed: Thu Nov 9 11:39:52 2017 -0800 -- .../functions/misc/RewriteFoldRCBindTest.java | 28 +++- 1 file changed, 22 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/systemml/blob/bd139a57/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteFoldRCBindTest.java -- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteFoldRCBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteFoldRCBindTest.java index 83050bd..476f573 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteFoldRCBindTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteFoldRCBindTest.java @@ -21,7 +21,10 @@ package org.apache.sysml.test.integration.functions.misc; import org.junit.Assert; import org.junit.Test; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; @@ -48,26 +51,37 @@ public class RewriteFoldRCBindTest extends AutomatedTestBase @Test public void testRewriteFoldCBindNoRewrite() { - testRewriteFoldRCBind( TEST_NAME1, false ); + testRewriteFoldRCBind( TEST_NAME1, false, ExecType.CP ); } @Test public void testRewriteFoldCBindRewrite() { - testRewriteFoldRCBind( TEST_NAME1, true ); + testRewriteFoldRCBind( TEST_NAME1, true, ExecType.CP ); } @Test public void testRewriteFoldRBindNoRewrite() { - testRewriteFoldRCBind( TEST_NAME2, false ); + testRewriteFoldRCBind( TEST_NAME2, false, ExecType.CP ); } @Test public void testRewriteFoldRBindRewrite() { - testRewriteFoldRCBind( TEST_NAME2, true ); + testRewriteFoldRCBind( TEST_NAME2, true, ExecType.CP ); } - private void testRewriteFoldRCBind( String testname, boolean rewrites ) - { + private void testRewriteFoldRCBind( String testname, boolean rewrites, ExecType et ) + { + RUNTIME_PLATFORM platformOld = rtplatform; + switch( et ){ + case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; + case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; + default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break; + } + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; try { @@ -96,6 +110,8 @@ public class RewriteFoldRCBindTest extends AutomatedTestBase } finally { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + rtplatform = platformOld; } } }