Repository: incubator-systemml Updated Branches: refs/heads/master ea6e2fe39 -> f73673d59
Added python script to launch systemml in standalone mode Closes #461 Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f73673d5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f73673d5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f73673d5 Branch: refs/heads/master Commit: f73673d59383ac947111cb84787cfa4df3ca7344 Parents: ea6e2fe Author: Nakul Jindal <naku...@gmail.com> Authored: Fri Apr 21 14:25:50 2017 -0700 Committer: Nakul Jindal <naku...@gmail.com> Committed: Fri Apr 21 14:25:50 2017 -0700 ---------------------------------------------------------------------- bin/systemml-standalone.py | 199 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f73673d5/bin/systemml-standalone.py ---------------------------------------------------------------------- diff --git a/bin/systemml-standalone.py b/bin/systemml-standalone.py new file mode 100755 index 0000000..367bcdf --- /dev/null +++ b/bin/systemml-standalone.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +import os +import shutil +import sys +from os.path import join, exists + + +# error help print +def print_usage_and_exit(): + this_script = sys.argv[0] + print('Usage: ' + this_script + ' <dml-filename> [arguments]') + sys.exit(1) + + +# from http://stackoverflow.com/questions/1724693/find-a-file-in-python +def find_file(name, path): + for root, dirs, files in os.walk(path): + if name in files: + return join(root, name) + return None + + +if len(sys.argv) < 2: + print('Wrong usage') + print_usage_and_exit() + + +# find the systemML root path which contains the bin folder, the script folder and the target folder +# tolerate path with spaces +script_dir = os.path.dirname(os.path.realpath(__file__)) +project_root_dir = os.path.dirname(script_dir) +user_dir = os.getcwd() + +scripts_dir = join(project_root_dir, 'scripts') +build_dir = join(project_root_dir, 'target') +lib_dir = join(build_dir, 'lib') +dml_script_class = join(build_dir, 'classes', 'org', 'apache', 'sysml', 'api', 'DMLScript.class') +hadoop_home = join(lib_dir, 'hadoop') + + +build_err_msg = 'You must build the project before running this script.' +build_dir_err_msg = 'Could not find target directory ' + build_dir + '. ' + build_err_msg + +lib_dir_err_msg = 'Could not find required libraries.' + build_err_msg +dml_script_err_msg = 'Could not find ' + dml_script_class + '. ' + build_err_msg + +# check if the project had been built and the jar files exist +if not(exists(build_dir)): + print(build_dir_err_msg) + sys.exit(1) +if not(exists(lib_dir)): + print(lib_dir_err_msg) + sys.exit(1) +if not(exists(dml_script_class)): + print(dml_script_err_msg) + sys.exit(1) + +print('================================================================================') + + +# if the present working directory is the project root or bin folder, then use the temp folder as user.dir +if user_dir == project_root_dir or user_dir == join(project_root_dir, 'bin'): + user_dir = join(project_root_dir, 'temp') + print('Output dir: ' + user_dir) + +# if the SystemML-config.xml does not exist, create it from the template +systemml_config_path = join(project_root_dir, 'conf', 'SystemML-config.xml') +systemml_template_config_path = join(project_root_dir, 'conf', 'SystemML-config.xml.template') +if not(exists(systemml_config_path)): + shutil.copyfile(systemml_template_config_path, systemml_config_path) + print('... created ' + systemml_config_path) + +# if the log4j.properties do not exist, create them from the template +log4j_properties_path = join(project_root_dir, 'conf', 'log4j.properties') +log4j_template_properties_path = join(project_root_dir, 'conf', 'log4j.properties.template') +if not(exists(log4j_properties_path)): + shutil.copyfile(log4j_template_properties_path, log4j_properties_path) + print('... created ' + log4j_properties_path) + + +script_file = sys.argv[1] + +# if the script file path was omitted, try to complete the script path +if not(exists(script_file)): + script_file_name = os.path.abspath(script_file) + script_file_found = find_file(script_file, scripts_dir) + if script_file_found is None: + print('Could not find DML script: ' + script_file) + print_usage_and_exit() + else: + script_file = script_file_found + print('DML Script:' + script_file) + +# add libraries which were generated by the build to the classpath +systemml_jar = join(build_dir, 'classes') + + +# For the *nix and windows, os.pathsep works reliably +# however for cygwin, the pathsep is set for *nix, which is ':' +# but the underlying java, which is a windows program requires ';' +# also all arguments passed to the JVM need to be converted to windows style +# if in the cygwin environment +lib_dir_star = join(lib_dir, '*') +if sys.platform == 'cygwin': + classpath_sep = os.pathsep + classpath_sep = ';' + lib_dir = os.popen('cygpath -pw ' + lib_dir).read().strip() + lib_dir_star = '"' + lib_dir + "\*" + '"' + systemml_jar = '"' + os.popen('cygpath -pw ' + systemml_jar).read().strip() + '"' + hadoop_home = '"' + os.popen('cygpath -pw ' + hadoop_home).read().strip() + '"' + log4j_properties_path = '"' + os.popen('cygpath -pw ' + log4j_properties_path).read().strip() + '"' + user_dir = '"' + os.popen('cygpath -pw ' + user_dir).read().strip() + '"' + script_file = '"' + os.popen('cygpath -pw ' + script_file).read().strip() + '"' + systemml_config_path = '"' + os.popen('cygpath -pw ' + systemml_config_path).read().strip() + '"' + classpath = lib_dir_star + '\\' + classpath_sep + systemml_jar +else: + #classpath = '"' + lib_dir_star + '"' + os.pathsep + '"' + systemml_jar + '"' + classpath = lib_dir_star + os.pathsep + systemml_jar + + +# Set the HADOOP_HOME environment variable +if 'HADOOP_HOME' not in os.environ: + os.environ['HADOOP_HOME'] = hadoop_home + + +print('================================================================================') + +# Set default Java options +systemml_default_java_opts = \ + '-Xmx8g -Xms4g -Xmn1g ' + \ + '-cp ' + classpath + ' ' + \ + '-Dlog4j.configuration=file:' + log4j_properties_path + ' ' \ + '-Duser.dir=' + user_dir + + +# Reads in key-value pairs from the conf/systemml-env.sh file +def parse_env_file(env_file_path): + env_vars = {} + with open(env_file_path) as f: + for l in f: + l = l.strip() + if l and not(l.startswith('#')) and '=' in l: + k, v = l.split('=', 1) + k = k.strip() + v = v.strip() + if len(v) > 0: + # strip quotes + if v[0] == v[len(v) - 1] and v[0] in ['"', "'"]: + v = v[1:-1] + + env_vars[k] = v + return env_vars + + +# Add any custom Java options set by the user at command line, overriding defaults as necessary. +if 'SYSTEMML_JAVA_OPTS' in os.environ: + systemml_java_opts = os.environ['SYSTEMML_JAVA_OPTS'] + systemml_default_java_opts = systemml_default_java_opts + ' ' + systemml_java_opts + # del os.environ['SYSTEMML_JAVA_OPTS'] + +# Add any custom Java options set by the user in the environment variables file, +# overriding defaults as necessary. +systemml_env_path = join(project_root_dir, 'conf', 'systemml-env.sh') +if exists(systemml_env_path): + env_vars = parse_env_file(systemml_env_path) + os.environ.update(env_vars) + + +# Invoke the jar with options and arguments +cmd = ['java', systemml_default_java_opts, 'org.apache.sysml.api.DMLScript', '-f', script_file, '-exec singlenode', '-config', systemml_config_path] + sys.argv[2:] +# For debugging +# print(' '.join(cmd)) + +return_code = os.system(' '.join(cmd)) + +if return_code != 0: + print('Failed to run SystemML. Exit code :' + str(return_code)) + print(' '.join(cmd))