systemml git commit: strip for new line

[kkalyan]

Repository: systemml
Updated Branches:
  refs/heads/master ce8ee96dc -> 660ba7630


strip for new line

Closes 706


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/660ba763
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/660ba763
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/660ba763

Branch: refs/heads/master
Commit: 660ba763074083cf63b876c42d4d00fe52620636
Parents: ce8ee96
Author: Krishna Kalyan 
Authored: Mon Dec 18 11:48:38 2017 +0100
Committer: Krishna Kalyan 
Committed: Mon Dec 18 11:48:38 2017 +0100

--
 scripts/perftest/python/utils_exec.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/660ba763/scripts/perftest/python/utils_exec.py
--
diff --git a/scripts/perftest/python/utils_exec.py 
b/scripts/perftest/python/utils_exec.py
index d51cf2d..04eea6a 100755
--- a/scripts/perftest/python/utils_exec.py
+++ b/scripts/perftest/python/utils_exec.py
@@ -101,7 +101,7 @@ def parse_hdfs_base(std_outs):
 hdfs_uri = None
 for line in std_outs:
 if line.startswith('hdfs://'):
-hdfs_uri = line
+hdfs_uri = line.strip()
 if hdfs_uri is None:
 sys.exit('HDFS URI not found')
 return hdfs_uri
@@ -160,7 +160,7 @@ def parse_hdfs_paths(std_outs):
 if 'No such file or directory' in i:
 break
 elif 'hdfs' in i:
-current_dir = i.split(' ')[-1]
+current_dir = i.split(' ')[-1].strip()
 hdfs_dir.append(current_dir)
 
 return hdfs_dir

systemml git commit: [SYSTEMML-1978] Add PCA to Performance Test Suite

[kkalyan]

Repository: systemml
Updated Branches:
  refs/heads/master bd139a575 -> d69686273


[SYSTEMML-1978]  Add PCA to Performance Test Suite

Closes #694


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d6968627
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d6968627
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d6968627

Branch: refs/heads/master
Commit: d69686273da8bf4dc09441ec34ef3863eb437629
Parents: bd139a5
Author: Krishna Kalyan 
Authored: Thu Nov 9 20:42:49 2017 +0100
Committer: Krishna Kalyan 
Committed: Thu Nov 9 20:42:49 2017 +0100

--
 scripts/perftest/python/datagen.py  | 17 +
 scripts/perftest/python/run_perftest.py | 11 +++
 scripts/perftest/python/train.py| 15 +++
 3 files changed, 39 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/d6968627/scripts/perftest/python/datagen.py
--
diff --git a/scripts/perftest/python/datagen.py 
b/scripts/perftest/python/datagen.py
index 54f2eff..55dd06d 100755
--- a/scripts/perftest/python/datagen.py
+++ b/scripts/perftest/python/datagen.py
@@ -215,6 +215,23 @@ def stats2_datagen(matrix_dim, matrix_type, datagen_dir, 
config_dir):
 return save_path
 
 
+def dimreduction_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):
+
+path_name = '.'.join(['dimreduction', matrix_type, str(matrix_dim)])
+datagen_write = join(datagen_dir, path_name)
+save_path = join(config_dir, path_name)
+row, col = split_rowcol(matrix_dim)
+
+R = row
+C = col
+OUT = join(datagen_write, 'X.data')
+
+config = dict(R=R, C=C, OUT=OUT, FMT=DATA_FORMAT)
+
+config_writer(save_path + '.json', config)
+return save_path
+
+
 def config_packets_datagen(algo_payload, matrix_type, matrix_shape, 
datagen_dir, dense_algos, config_dir):
 """
 This function has two responsibilities. Generate the configuration files 
for

http://git-wip-us.apache.org/repos/asf/systemml/blob/d6968627/scripts/perftest/python/run_perftest.py
--
diff --git a/scripts/perftest/python/run_perftest.py 
b/scripts/perftest/python/run_perftest.py
index 6e87261..1f78a75 100755
--- a/scripts/perftest/python/run_perftest.py
+++ b/scripts/perftest/python/run_perftest.py
@@ -47,7 +47,8 @@ ML_ALGO = {'binomial': ['MultiLogReg', 'l2-svm', 'm-svm'],
'regression1': ['LinearRegDS', 'LinearRegCG'],
'regression2': ['GLM_poisson', 'GLM_gamma', 'GLM_binomial'],
'stats1': ['Univar-Stats', 'bivar-stats'],
-   'stats2': ['stratstats']}
+   'stats2': ['stratstats'],
+   'dimreduction': ['PCA']}
 
 ML_GENDATA = {'binomial': 'genRandData4LogisticRegression',
   'clustering': 'genRandData4Kmeans',
@@ -55,7 +56,8 @@ ML_GENDATA = {'binomial': 'genRandData4LogisticRegression',
   'regression1': 'genRandData4LogisticRegression',
   'regression2': 'genRandData4LogisticRegression',
   'stats1': 'genRandData4DescriptiveStats',
-  'stats2': 'genRandData4StratStats'}
+  'stats2': 'genRandData4StratStats',
+  'dimreduction': 'genRandData4PCA'}
 
 ML_TRAIN = {'GLM_poisson': 'GLM',
 'GLM_gamma': 'GLM',
@@ -69,7 +71,8 @@ ML_TRAIN = {'GLM_poisson': 'GLM',
 'm-svm': 'm-svm',
 'l2-svm': 'l2-svm',
 'MultiLogReg': 'MultiLogReg',
-'naive-bayes': 'naive-bayes'}
+'naive-bayes': 'naive-bayes',
+'PCA': 'PCA'}
 
 ML_PREDICT = {'Kmeans': 'Kmeans-predict',
   'LinearRegCG': 'GLM-predict',
@@ -82,7 +85,7 @@ ML_PREDICT = {'Kmeans': 'Kmeans-predict',
   'GLM_gamma': 'GLM-predict',
   'GLM_binomial': 'GLM-predict'}
 
-DENSE_TYPE_ALGOS = ['clustering', 'stats1', 'stats2']
+DENSE_TYPE_ALGOS = ['clustering', 'stats1', 'stats2', 'dimreduction']
 
 
 # Responsible for execution and metric logging

http://git-wip-us.apache.org/repos/asf/systemml/blob/d6968627/scripts/perftest/python/train.py
--
diff --git a/scripts/perftest/python/train.py b/scripts/perftest/python/train.py
index 4428e8f..907c2b9 100755
--- a/scripts/perftest/python/train.py
+++ b/scripts/perftest/python/train.py
@@ -338,6 +338,21 @@ def regression2_glm_poisson_train(save_folder_name, 
datagen_dir, train_dir, conf
 return data_folders
 
 
+def dimreduction_pca_train(save_folder_name, datagen_dir, train_dir, 
config_dir):
+save_path = join(config_dir, save_folder_name)
+train_write = join(train_dir, save_folder_name)
+
+