SLIDER-756 revise accumulo-slider script to work with slider client --install command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/670bfb32 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/670bfb32 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/670bfb32 Branch: refs/heads/feature/package_simplification_II Commit: 670bfb32fd26769fd5f08b4609f05fb6c915da45 Parents: 909b7c1 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Wed Mar 4 23:14:24 2015 -0800 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Wed Mar 4 23:14:24 2015 -0800 ---------------------------------------------------------------------- app-packages/accumulo/accumulo-slider | 53 --- app-packages/accumulo/accumulo-slider.py | 378 ------------------- .../accumulo/clientInstallConfig-default.json | 7 + app-packages/accumulo/metainfo.xml | 2 +- .../accumulo/package/files/accumulo-slider | 59 +++ .../accumulo/package/files/accumulo-slider.py | 182 +++++++++ .../accumulo/package/scripts/accumulo_client.py | 37 +- .../accumulo/package/scripts/client_params.py | 39 ++ .../package/templates/accumulo-slider-env.sh.j2 | 17 + app-packages/accumulo/pom.xml | 14 +- app-packages/accumulo/src/assembly/accumulo.xml | 8 +- .../funtest/accumulo/AccumuloScriptIT.groovy | 91 +++-- .../funtest/accumulo/AccumuloSliderShell.groovy | 7 - .../resources/clientInstallConfig-test.json | 8 + 14 files changed, 415 insertions(+), 487 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/accumulo-slider ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/accumulo-slider b/app-packages/accumulo/accumulo-slider deleted file mode 100644 index 8f9cb06..0000000 --- a/app-packages/accumulo/accumulo-slider +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Resolve links - $0 may be a softlink -PRG="${0}" - -while [ -h "${PRG}" ]; do - ls=`ls -ld "${PRG}"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "${PRG}"`/"$link" - fi -done - -# find python >= 2.6 -if [ -a /usr/bin/python2.6 ]; then - PYTHON=/usr/bin/python2.6 -fi - -if [ -z "$PYTHON" ]; then - PYTHON=/usr/bin/python -fi - -# check for version -majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1` -minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2` -numversion=$(( 10 * $majversion + $minversion)) -if (( $numversion < 26 )); then - echo "Need python version > 2.6" - exit 1 -fi - -ACCUMULO_SLIDER_BIN_DIR=`dirname ${PRG}` - -$PYTHON ${ACCUMULO_SLIDER_BIN_DIR}/accumulo-slider.py $@ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/accumulo-slider.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/accumulo-slider.py b/app-packages/accumulo/accumulo-slider.py deleted file mode 100644 index 1eccf59..0000000 --- a/app-packages/accumulo/accumulo-slider.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import json -import glob -import tempfile -import subprocess -import shutil - -SLIDER_DIR = os.getenv('SLIDER_HOME', None) -if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)): - print "Unable to find SLIDER_HOME. Please configure SLIDER_HOME before running accumulo-slider" - sys.exit(1) -SLIDER_CMD = os.path.join(SLIDER_DIR, 'bin', 'slider.py') - -TMP_DIR = os.path.join(tempfile.gettempdir(), "accumulo-slider-tmp."+str(os.getpid())) - -CMD_OPTS = {} - -def call(cmd): - print "Running: " + " ".join(cmd) - retcode = subprocess.call(cmd) - if retcode != 0: - raise Exception("return code from running %s was %d" % (cmd[0], retcode)) - -def exec_accumulo_command(command, args=[]): - ACCUMULO_DIR = os.getenv('ACCUMULO_HOME', None) - if ACCUMULO_DIR == None or (not os.path.exists(ACCUMULO_DIR)): - print "Unable to find ACCUMULO_HOME. Please configure ACCUMULO_HOME before running accumulo-slider " + command - sys.exit(1) - ACCUMULO_CMD = os.path.join(ACCUMULO_DIR, 'bin', 'accumulo') - - confdir = get_all_conf() - if command == 'shell' or command == 'admin': - cmd = [ACCUMULO_CMD, command, '--config-file', os.path.join(confdir, 'client.conf')] + list(args) - else: - cmd = [ACCUMULO_CMD, command] + list(args) - print "Setting ACCUMULO_CONF_DIR=" + confdir - os.putenv("ACCUMULO_CONF_DIR", confdir) - call(cmd) - -def exec_tool_command(jarfile, mainclass, args=[]): - ACCUMULO_DIR = os.getenv('ACCUMULO_HOME', None) - if ACCUMULO_DIR == None or (not os.path.exists(ACCUMULO_DIR)): - print "Unable to find ACCUMULO_HOME. Please configure ACCUMULO_HOME before running accumulo-slider tool" - sys.exit(1) - TOOL_CMD = os.path.join(ACCUMULO_DIR, 'bin', 'tool.sh') - - confdir = get_all_conf() - cmd = [TOOL_CMD, jarfile, mainclass] + list(args) - print "Setting ACCUMULO_CONF_DIR=" + confdir - os.putenv("ACCUMULO_CONF_DIR", confdir) - call(cmd) - -def jar(jarfile, *args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] jar jarfile [mainclass]] - OR: [accumulo-slider --appconf confdir jar jarfile [mainclass]] - Runs a class from a specified jar - """ - exec_accumulo_command("jar", jarfile, args=args) - -def classname(mainclass, *args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] classname mainclass] - OR: [accumulo-slider --appconf confdir classname mainclass] - Runs a specified class on the existing accumulo classpath - """ - exec_accumulo_command(mainclass, args=args) - -def shell(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] shell] - OR: [accumulo-slider --appconf confdir shell] - Runs an accumulo shell - """ - exec_accumulo_command("shell", args=args) - -def admin(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] admin cmd] - OR: [accumulo-slider --appconf confdir admin cmd] - Executes an admin command (run without cmd argument for a list of commands) - """ - exec_accumulo_command("admin", args=args) - -def classpath(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] classpath] - OR: [accumulo-slider --appconf confdir classpath] - Prints the classpath of the accumulo client install - """ - exec_accumulo_command("classpath", args=args) - -def info(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] info] - OR: [accumulo-slider --appconf confdir info] - Prints information about an accumulo instance (monitor, masters, and zookeepers) - """ - exec_accumulo_command("info", args=args) - -def rfileinfo(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] rfile-info rfilename] - OR: [accumulo-slider --appconf confdir rfile-info rfilename] - Prints information about a specified RFile - """ - exec_accumulo_command("rfile-info", args=args) - -def logininfo(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] login-info] - OR: [accumulo-slider --appconf confdir login-info] - Prints the supported authentication token types for the accumulo instance - """ - exec_accumulo_command("login-info", args=args) - -def createtoken(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] create-token] - OR: [accumulo-slider --appconf confdir create-token] - Saves a given accumulo authentication token to a file - """ - exec_accumulo_command("create-token", args=args) - -def version(*args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] version] - OR: [accumulo-slider --appconf confdir version] - Prints the version of the accumulo client install - """ - exec_accumulo_command("version", args=args) - -def tool(jarfile, mainclass, *args): - """Syntax: [accumulo-slider --app appname[ --appconf confdir] tool jarfile classname] - OR: [accumulo-slider --appconf confdir tool jarfile classname] - Runs a mapreduce job using accumulo's tool.sh - """ - exec_tool_command(jarfile, mainclass, args=args) - -def quicklinks(): - """Syntax: [accumulo-slider --app appname quicklinks] - Prints the quicklinks information of accumulo-slider registry - """ - global CMD_OPTS - if not 'app_name' in CMD_OPTS.keys(): - print_usage() - sys.exit(1) - - cmd = [SLIDER_CMD, "registry", "--getconf", "quicklinks", "--format", "json", - "--name", CMD_OPTS['app_name']] - - if 'user' in CMD_OPTS.keys(): - cmd.append( "--user "+CMD_OPTS['user']) - - call(cmd) - -def proxies(): - """Syntax: [accumulo-slider --app appname proxies] - Prints the componentinstancedata information of accumulo-slider registry - """ - global CMD_OPTS - if not 'app_name' in CMD_OPTS.keys(): - print_usage() - sys.exit(1) - - cmd = [SLIDER_CMD, "registry", "--getconf", "componentinstancedata", - "--format", "json", "--name", CMD_OPTS['app_name']] - - if 'user' in CMD_OPTS.keys(): - cmd.append( "--user "+CMD_OPTS['user']) - - call(cmd) - -def install(dir): - """Syntax: [accumulo-slider --app appname install dir] - Installs a fully configured accumulo client in the specified dir - The resulting client may be used on its own without accumulo-slider - """ - global CMD_OPTS - if not 'app_name' in CMD_OPTS.keys(): - print_usage() - sys.exit(1) - if os.path.exists(dir): - raise Exception("Install dir must not exist: " + dir) - - global TMP_DIR - workdir = os.path.join(TMP_DIR, 'install-work-dir') - - statusfile = os.path.join(workdir, 'status.json') - cmd = [SLIDER_CMD, "status", CMD_OPTS['app_name'], "--out", statusfile] - call(cmd) - - infile = open(statusfile) - try: - content = json.load(infile) - finally: - infile.close() - - appdef = content['options']['application.def'] - appdeffile = appdef[appdef.rfind('/')+1:] - cmd = ["hadoop", "fs", "-copyToLocal", appdef, workdir] - call(cmd) - - cmd = ["unzip", os.path.join(workdir, appdeffile), "-d", workdir] - call(cmd) - - gzfile = glob.glob(os.path.join(workdir, 'package', 'files', 'accumulo*gz')) - if len(gzfile) != 1: - raise Exception("got " + gzfile + " from glob") - cmd = ["tar", "xvzf", gzfile[0], '-C', workdir] - call(cmd) - - tmp_accumulo = glob.glob(os.path.join(workdir, 'accumulo-[0-9]*')) - if len(tmp_accumulo) != 1: - raise Exception("got " + tmp_accumulo + " from glob") - tmp_accumulo = tmp_accumulo[0] - - confdir = os.path.join(tmp_accumulo, 'conf') - tmpconf = os.path.join(workdir, 'conf-tmp') - shutil.move(confdir, tmpconf) - make_conf(os.path.join(tmpconf, 'templates'), confdir) - - libdir = os.path.join(tmp_accumulo, 'lib') - for jar in glob.glob(os.path.join(workdir, 'package', 'files', '*jar')): - shutil.move(jar, libdir) - shutil.move(tmp_accumulo, dir) - -def get_all_conf(): - """Syntax: [accumulo-slider --app appname [--appconf confdir] getconf] - Downloads configuration for an accumulo instance - If --appconf is specified, creates the specified conf dir and populates it - """ - ACCUMULO_CONF_DIR = os.getenv('ACCUMULO_CONF_DIR', None) - if ACCUMULO_CONF_DIR == None or (not os.path.exists(ACCUMULO_CONF_DIR)): - ACCUMULO_DIR = os.getenv('ACCUMULO_HOME', None) - if ACCUMULO_DIR == None or (not os.path.exists(ACCUMULO_DIR)): - print "Unable to find ACCUMULO_HOME. Please configure ACCUMULO_HOME before running this command" - sys.exit(1) - ACCUMULO_CONF_DIR = os.path.join(ACCUMULO_DIR, 'conf') - - global CMD_OPTS - global TMP_DIR - confdir = os.path.join(TMP_DIR, 'conf') - if 'app_conf' in CMD_OPTS.keys(): - confdir = CMD_OPTS['app_conf'] - if os.path.exists(confdir): - print "Using existing app conf instead of downloading it again: " + confdir - return confdir - - if not 'app_name' in CMD_OPTS.keys(): - print_usage() - sys.exit(1) - - make_conf(ACCUMULO_CONF_DIR, confdir) - return confdir - -def make_conf(oldconf, newconf): - client_file = os.path.join(newconf, 'client.conf') - site_file = os.path.join(newconf, 'accumulo-site.xml') - env_file = os.path.join(newconf, 'accumulo-env.sh') - env_json = os.path.join(newconf, 'accumulo-env.json') - - print "Copying base conf from " + oldconf + " to " + newconf - shutil.copytree(oldconf, newconf) - - try_remove(client_file) - try_remove(site_file) - try_remove(env_file) - try_remove(env_json) - - get_conf("client", "properties", client_file) - get_conf("accumulo-site", "xml", site_file) - get_conf("accumulo-env", "json", env_json) - - infile = open(env_json) - outfile = open(env_file, 'w') - try: - content = json.load(infile) - outfile.write(content['content']) - finally: - outfile.close() - infile.close() - -def try_remove(path): - try: - os.remove(path) - except: - if os.path.exists(path): - raise - -def get_conf(confname, fileformat, destfile): - if os.path.exists(destfile): - print "Conf file " + destfile + " already exists, remove it to re-download" - return - - cmd = [SLIDER_CMD, "registry", "--getconf", confname, "--format", - fileformat, "--dest", destfile, "--name", CMD_OPTS['app_name']] - if 'user' in CMD_OPTS.keys(): - cmd.append("--user " + CMD_OPTS['user']) - - call(cmd) - if not os.path.exists(destfile): - raise Exception("Failed to read slider deployed accumulo config " + confname) - -def print_commands(): - """Print all client commands and link to documentation""" - print "Commands:\n\t", "\n\t".join(sorted(COMMANDS.keys())) - print "\nHelp:", "\n\thelp", "\n\thelp <command>" - -def print_usage(command=None): - """Print one help message or list of available commands""" - if command != None: - if COMMANDS.has_key(command): - print (COMMANDS[command].__doc__ or - "No documentation provided for <%s>" % command) - else: - print "<%s> is not a valid command" % command - else: - print "Usage:" - print "accumulo-slider --app <name>[ --appconf <confdir> --user <username>] <command>" - print " The option --appconf creates a conf dir that can be reused;" - print " on subsequent calls to accumulo-slider, --app can be left off if" - print " --appconf is specified. If --appconf is not specified, a" - print " temporary conf dir is created each time accumulo-slider is run." - print_commands() - -def unknown_command(*args): - print "Unknown command: [accumulo-slider %s]" % ' '.join(sys.argv[1:]) - print_usage() - -COMMANDS = {"shell": shell, "tool": tool, "admin": admin, "classpath": classpath, - "info": info, "version": version, "jar": jar, "classname": classname, - "quicklinks" : quicklinks, "proxies": proxies, "getconf": get_all_conf, - "rfile-info": rfileinfo, "login-info": logininfo, "create-token": createtoken, - "install": install, "help": print_usage} - -def parse_config_opts(args): - curr = args[:] - curr.reverse() - global CMD_OPTS - args_list = [] - while len(curr) > 0: - token = curr.pop() - if token == "--app": - CMD_OPTS['app_name'] = curr.pop() if (len(curr) != 0) else None - elif token == "--user": - CMD_OPTS['user'] = curr.pop() if (len(curr) != 0) else None - elif token == "--appconf": - CMD_OPTS['app_conf'] = curr.pop() if (len(curr) != 0) else None - else: - args_list.append(token) - return args_list - -def main(): - args = parse_config_opts(sys.argv[1:]) - if len(args) < 1: - print_usage() - sys.exit(-1) - COMMAND = args[0] - ARGS = args[1:] - try: - (COMMANDS.get(COMMAND, unknown_command))(*ARGS) - finally: - global CMD_OPTS - if os.path.exists(TMP_DIR): - print "Cleaning up tmp dir " + TMP_DIR - shutil.rmtree(TMP_DIR) - -if __name__ == "__main__": - main() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/clientInstallConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/clientInstallConfig-default.json b/app-packages/accumulo/clientInstallConfig-default.json new file mode 100644 index 0000000..02176dc --- /dev/null +++ b/app-packages/accumulo/clientInstallConfig-default.json @@ -0,0 +1,7 @@ +{ + "schema":"http://example.org/specification/v2.0.0", + "global":{ + "client_root": "{app_install_dir}/accumulo-${accumulo.version}", + "slider_home_dir": "${slider.dir}" + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml index f0ae34d..b425aef 100644 --- a/app-packages/accumulo/metainfo.xml +++ b/app-packages/accumulo/metainfo.xml @@ -140,7 +140,7 @@ <component> <name>ACCUMULO_PROXY</name> - <category>CLIENT</category> + <category>SLAVE</category> <componentExports> <componentExport> <name>proxies</name> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/package/files/accumulo-slider ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/files/accumulo-slider b/app-packages/accumulo/package/files/accumulo-slider new file mode 100644 index 0000000..0d203d7 --- /dev/null +++ b/app-packages/accumulo/package/files/accumulo-slider @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Resolve links - $0 may be a softlink +PRG="${0}" + +while [ -h "${PRG}" ]; do + ls=`ls -ld "${PRG}"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "${PRG}"`/"$link" + fi +done + +# find python >= 2.6 +if [ -a /usr/bin/python2.6 ]; then + PYTHON=/usr/bin/python2.6 +fi + +if [ -z "$PYTHON" ]; then + PYTHON=/usr/bin/python +fi + +# check for version +majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1` +minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2` +numversion=$(( 10 * $majversion + $minversion)) +if (( $numversion < 26 )); then + echo "Need python version > 2.6" + exit 1 +fi + +ACCUMULO_BIN_DIR=`dirname ${PRG}` +ACCUMULO_HOME=`cd ${ACCUMULO_BIN_DIR}/..;pwd` +ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}" + +if [ -f "${ACCUMULO_CONF_DIR}/accumulo-slider-env.sh" ]; then + . "${ACCUMULO_CONF_DIR}/accumulo-slider-env.sh" +fi + +$PYTHON ${ACCUMULO_BIN_DIR}/accumulo-slider.py $@ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/package/files/accumulo-slider.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/files/accumulo-slider.py b/app-packages/accumulo/package/files/accumulo-slider.py new file mode 100644 index 0000000..e685293 --- /dev/null +++ b/app-packages/accumulo/package/files/accumulo-slider.py @@ -0,0 +1,182 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import json +import glob +import tempfile +import subprocess +import shutil + +SLIDER_DIR = os.getenv('SLIDER_HOME', None) +if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)): + print "Unable to find SLIDER_HOME. Please configure SLIDER_HOME before running accumulo-slider" + sys.exit(1) +SLIDER_CMD = os.path.join(SLIDER_DIR, 'bin', 'slider.py') + +CMD_OPTS = {} + +def call(cmd): + print "Running: " + " ".join(cmd) + retcode = subprocess.call(cmd) + if retcode != 0: + raise Exception("return code from running %s was %d" % (cmd[0], retcode)) + +def quicklinks(): + """Syntax: [accumulo-slider --app appname quicklinks] + Prints the quicklinks information of accumulo-slider registry + """ + global CMD_OPTS + if not 'app_name' in CMD_OPTS.keys(): + print_usage() + sys.exit(1) + + cmd = [SLIDER_CMD, "registry", "--getconf", "quicklinks", "--format", "json", + "--name", CMD_OPTS['app_name']] + + if 'user' in CMD_OPTS.keys(): + cmd.append( "--user "+CMD_OPTS['user']) + + call(cmd) + +def proxies(): + """Syntax: [accumulo-slider --app appname proxies] + Prints the componentinstancedata information of accumulo-slider registry + """ + global CMD_OPTS + if not 'app_name' in CMD_OPTS.keys(): + print_usage() + sys.exit(1) + + cmd = [SLIDER_CMD, "registry", "--getconf", "componentinstancedata", + "--format", "json", "--name", CMD_OPTS['app_name']] + + if 'user' in CMD_OPTS.keys(): + cmd.append( "--user "+CMD_OPTS['user']) + + call(cmd) + +def get_all_conf(): + """Syntax: [accumulo-slider --app appname --appconf confdir getconf] + Downloads configuration for an accumulo instance to a specified conf dir, + overwriting if the files already exist + """ + if not 'app_conf' in CMD_OPTS.keys(): + print_usage() + sys.exit(1) + confdir = CMD_OPTS['app_conf'] + + if not 'app_name' in CMD_OPTS.keys(): + print_usage() + sys.exit(1) + + client_file = os.path.join(confdir, 'client.conf') + site_file = os.path.join(confdir, 'accumulo-site.xml') + env_file = os.path.join(confdir, 'accumulo-env.sh') + env_json = os.path.join(confdir, 'accumulo-env.json') + + if os.path.exists(client_file): + os.remove(client_file) + if os.path.exists(site_file): + os.remove(site_file) + if os.path.exists(env_file): + os.remove(env_file) + if os.path.exists(env_json): + os.remove(env_json) + + get_conf("client", "properties", client_file) + get_conf("accumulo-site", "xml", site_file) + get_conf("accumulo-env", "json", env_json) + + infile = open(env_json) + outfile = open(env_file, 'w') + try: + content = json.load(infile) + outfile.write(content['content']) + finally: + outfile.close() + infile.close() + +def get_conf(confname, fileformat, destfile): + if os.path.exists(destfile): + raise Exception("conf file %s was removed but still exists" % (destfile)) + + cmd = [SLIDER_CMD, "registry", "--getconf", confname, "--format", + fileformat, "--dest", destfile, "--name", CMD_OPTS['app_name']] + if 'user' in CMD_OPTS.keys(): + cmd.append("--user " + CMD_OPTS['user']) + + call(cmd) + if not os.path.exists(destfile): + raise Exception("Failed to read slider deployed accumulo config " + confname) + +def print_commands(): + """Print all client commands and link to documentation""" + print "Commands:\n\t", "\n\t".join(sorted(COMMANDS.keys())) + print "\nHelp:", "\n\thelp", "\n\thelp <command>" + +def print_usage(command=None): + """Print one help message or list of available commands""" + if command != None: + if COMMANDS.has_key(command): + print (COMMANDS[command].__doc__ or + "No documentation provided for <%s>" % command) + else: + print "<%s> is not a valid command" % command + else: + print "Usage:" + print "accumulo-slider --app <name> [--user <username>] quicklinks" + print "accumulo-slider --app <name> [--user <username>] proxies" + print "accumulo-slider --app <name> --appconf <confdir> [--user <username>] getconf" + print_commands() + +def unknown_command(*args): + print "Unknown command: [accumulo-slider %s]" % ' '.join(sys.argv[1:]) + print_usage() + +COMMANDS = {"quicklinks" : quicklinks, "proxies": proxies, "getconf": get_all_conf, + "help": print_usage} + +def parse_config_opts(args): + curr = args[:] + curr.reverse() + global CMD_OPTS + args_list = [] + while len(curr) > 0: + token = curr.pop() + if token == "--app": + CMD_OPTS['app_name'] = curr.pop() if (len(curr) != 0) else None + elif token == "--user": + CMD_OPTS['user'] = curr.pop() if (len(curr) != 0) else None + elif token == "--appconf": + CMD_OPTS['app_conf'] = curr.pop() if (len(curr) != 0) else None + else: + args_list.append(token) + return args_list + +def main(): + args = parse_config_opts(sys.argv[1:]) + if len(args) < 1: + print_usage() + sys.exit(-1) + COMMAND = args[0] + ARGS = args[1:] + (COMMANDS.get(COMMAND, unknown_command))(*ARGS) + +if __name__ == "__main__": + main() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/package/scripts/accumulo_client.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/accumulo_client.py b/app-packages/accumulo/package/scripts/accumulo_client.py index f50addf..2da02dc 100644 --- a/app-packages/accumulo/package/scripts/accumulo_client.py +++ b/app-packages/accumulo/package/scripts/accumulo_client.py @@ -18,22 +18,43 @@ limitations under the License. """ -import sys from resource_management import * -from accumulo_configuration import setup_conf_dir - - class AccumuloClient(Script): def install(self, env): + import client_params + env.set_params(client_params) self.install_packages(env) - self.configure(env) + Directory(client_params.conf_dir, + content=format("{conf_dir}/templates")) + jarname = "SliderAccumuloUtils.jar" + File(format("{client_root}/lib/{jarname}"), + mode=0644, + content=StaticFile(jarname) + ) + File(format("{bin_dir}/accumulo-slider"), + content=StaticFile("accumulo-slider"), + mode=0755 + ) + File(format("{bin_dir}/accumulo-slider.py"), + content=StaticFile("accumulo-slider.py"), + mode=0755 + ) + TemplateConfig(format("{conf_dir}/accumulo-slider-env.sh"), + mode=0755 + ) + if client_params.app_name: + Execute( format("{bin_dir}/accumulo-slider " + "--appconf {client_root}/conf --app {app_name} getconf ")) def configure(self, env): - import params - env.set_params(params) + pass - setup_conf_dir(name='client') + def start(self, env): + pass + + def stop(self, env): + pass def status(self, env): pass http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/package/scripts/client_params.py ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/scripts/client_params.py b/app-packages/accumulo/package/scripts/client_params.py new file mode 100644 index 0000000..1fc5219 --- /dev/null +++ b/app-packages/accumulo/package/scripts/client_params.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os + +config = Script.get_config() + +app_install_dir = config['configurations']['global']['app_install_dir'] +client_root = config['configurations']['global']['client_root'] +bin_dir = os.path.join(client_root, 'bin') +conf_dir = os.path.join(client_root, 'conf') + +app_name = None +if 'app_name' in config['configurations']['global']: + app_name = config['configurations']['global']['app_name'] + +slider_home_dir = config['configurations']['global']['slider_home_dir'] +if os.environ.has_key('SLIDER_CONF_DIR'): + slider_conf_dir = os.environ.get('SLIDER_CONF_DIR') +else: + slider_conf_dir = os.path.join(slider_home_dir, 'conf') http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/package/templates/accumulo-slider-env.sh.j2 ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/package/templates/accumulo-slider-env.sh.j2 b/app-packages/accumulo/package/templates/accumulo-slider-env.sh.j2 new file mode 100644 index 0000000..3cca1a8 --- /dev/null +++ b/app-packages/accumulo/package/templates/accumulo-slider-env.sh.j2 @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export SLIDER_HOME={{slider_home_dir}} +export SLIDER_CONF_DIR={{slider_conf_dir}} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/pom.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml index c062f95..678ace5 100644 --- a/app-packages/accumulo/pom.xml +++ b/app-packages/accumulo/pom.xml @@ -38,7 +38,7 @@ <slider.bin.dir>../../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir> <test.app.pkg.dir>${project.build.directory}</test.app.pkg.dir> <test.app.resources.dir>${project.build.directory}/test-config</test.app.resources.dir> - <test.app.scripts.dir>${project.build.directory}/${app.package.name}</test.app.scripts.dir> + <test.client.install.dir>${project.build.directory}/${work.dir}</test.client.install.dir> <!-- these properties are used in the default and the test appConfigs --> <hadoop.dir>/usr/lib/hadoop</hadoop.dir> <hdfs.dir>/usr/lib/hadoop-hdfs</hdfs.dir> @@ -48,6 +48,8 @@ <app.java.home>${java.home}</app.java.home> <app.user>yarn</app.user> <app.user.group>hadoop</app.user.group> + <!-- default slider home for clientInstallConfig --> + <slider.dir>${slider.bin.dir}</slider.dir> <!-- these are for accumulo processes --> <accumulo.keytab /> <accumulo.principal /> @@ -65,6 +67,7 @@ <yarn.dir>/usr/hdp/current/hadoop-yarn-client</yarn.dir> <mapred.dir>/usr/hdp/current/hadoop-mapreduce-client</mapred.dir> <zk.dir>/usr/hdp/current/zookeeper-client</zk.dir> + <slider.dir>/usr/hdp/current/slider-client</slider.dir> </properties> </profile> <profile> @@ -215,7 +218,8 @@ <test.app.pkg.file>${app.package.name}.zip</test.app.pkg.file> <test.app.pkg.name>ACCUMULO</test.app.pkg.name> <test.app.resources.dir>${test.app.resources.dir}</test.app.resources.dir> - <test.app.scripts.dir>${test.app.scripts.dir}</test.app.scripts.dir> + <test.client.install.dir>${test.client.install.dir}</test.client.install.dir> + <test.client.home.dir>${test.client.install.dir}/accumulo-${accumulo.version}</test.client.home.dir> </systemPropertyVariables> </configuration> </plugin> @@ -290,6 +294,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-examples-simple</artifactId> + <version>${accumulo.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.thrift</groupId> <artifactId>libthrift</artifactId> <version>0.9.1</version> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/src/assembly/accumulo.xml ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/assembly/accumulo.xml b/app-packages/accumulo/src/assembly/accumulo.xml index 3bc0e1f..d63fb28 100644 --- a/app-packages/accumulo/src/assembly/accumulo.xml +++ b/app-packages/accumulo/src/assembly/accumulo.xml @@ -42,15 +42,15 @@ <fileMode>0755</fileMode> </file> <file> - <source>metainfo.xml</source> + <source>clientInstallConfig-default.json</source> <outputDirectory>/</outputDirectory> <filtered>true</filtered> <fileMode>0755</fileMode> </file> <file> - <source>accumulo-slider</source> + <source>metainfo.xml</source> <outputDirectory>/</outputDirectory> - <filtered>false</filtered> + <filtered>true</filtered> <fileMode>0755</fileMode> </file> <file> @@ -78,8 +78,8 @@ <exclude>target/**</exclude> <exclude>appConfig-default.json</exclude> <exclude>appConfig-secured-default.json</exclude> + <exclude>clientInstallConfig-default.json</exclude> <exclude>metainfo.xml</exclude> - <exclude>accumulo-slider</exclude> </excludes> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy index 2ba06af..4fea600 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy @@ -17,18 +17,25 @@ package org.apache.slider.funtest.accumulo import groovy.util.logging.Slf4j +import org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter +import org.apache.accumulo.examples.simple.helloworld.ReadData +import org.apache.hadoop.registry.client.api.RegistryConstants import org.apache.slider.api.ClusterDescription import org.apache.slider.client.SliderClient import org.apache.slider.funtest.framework.AccumuloSliderShell +import org.apache.slider.funtest.framework.FuntestProperties +import org.apache.slider.funtest.framework.SliderShell import org.junit.BeforeClass -import java.nio.ByteBuffer - @Slf4j class AccumuloScriptIT extends AccumuloBasicIT { - public static final String RESOURCES_DIR = sysprop("test.app.resources.dir") - public static final String ACCUMULO_HOME = RESOURCES_DIR + "/install" - public static final String ACCUMULO_CONF = sysprop("test.app.resources.dir") + "/conf" + public static final String CLIENT_INSTALL_DIR = sysprop("test.client.install.dir") + public static final String CLIENT_HOME_DIR = sysprop("test.client.home.dir") + public static final String CLIENT_INSTALL_CONF = sysprop("test.app.resources.dir") + "/clientInstallConfig-test.json" + + public static final File ACCUMULO_SLIDER_SCRIPT = new File(CLIENT_HOME_DIR + "/bin", "accumulo-slider").canonicalFile + public static final File ACCUMULO_SCRIPT = new File(CLIENT_HOME_DIR + "/bin", "accumulo").canonicalFile + public static final File ACCUMULO_TOOL_SCRIPT = new File(CLIENT_HOME_DIR + "/bin", "tool.sh").canonicalFile @Override public String getClusterName() { @@ -42,47 +49,63 @@ class AccumuloScriptIT extends AccumuloBasicIT { @BeforeClass public static void setShell() { - AccumuloSliderShell.scriptFile = new File(sysprop("test.app.scripts.dir"), - "accumulo-slider").canonicalFile AccumuloSliderShell.setEnv("SLIDER_HOME", SLIDER_TAR_DIR) AccumuloSliderShell.setEnv("SLIDER_CONF_DIR", SLIDER_CONF_DIR) - AccumuloSliderShell.setEnv("ACCUMULO_HOME", ACCUMULO_HOME) + } + + public static AccumuloSliderShell accumulo_slider(String cmd) { + AccumuloSliderShell.scriptFile = ACCUMULO_SLIDER_SCRIPT + return AccumuloSliderShell.run(0, cmd) + } + + public static AccumuloSliderShell accumulo(String cmd) { + AccumuloSliderShell.scriptFile = ACCUMULO_SCRIPT + return AccumuloSliderShell.run(0, cmd) + } + + public static AccumuloSliderShell tool(String cmd) { + AccumuloSliderShell.scriptFile = ACCUMULO_TOOL_SCRIPT + return AccumuloSliderShell.run(0, cmd) } @Override public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) { String clusterName = getClusterName() - AccumuloSliderShell.run(0, "--app $clusterName quicklinks") - AccumuloSliderShell.run(0, "--app $clusterName install $ACCUMULO_HOME") - AccumuloSliderShell.run(0, "--app $clusterName --appconf $ACCUMULO_CONF getconf") - runBoth("shell -u $USER -p $PASSWORD -e tables") - runBoth("login-info") - AccumuloSliderShell info = runOne("info") + SliderShell shell = slider(EXIT_SUCCESS, + [ + ACTION_CLIENT, ARG_INSTALL, + ARG_PACKAGE, TEST_APP_PKG_DIR+"/"+TEST_APP_PKG_FILE, + ARG_DEST, CLIENT_INSTALL_DIR, + ARG_CONFIG, CLIENT_INSTALL_CONF + ]) + logShell(shell) + + accumulo_slider("--app $clusterName quicklinks") + + accumulo("shell -u $USER -p $PASSWORD -e tables") + accumulo("login-info") + + AccumuloSliderShell info = accumulo("info") String monitor = getMonitorUrl(sliderClient, getClusterName()) assert info.outputContains(monitor.substring(monitor.indexOf("://")+3)), "accumulo info output did not contain monitor" - runOne("version") - runOne("classpath") - runOne("create-token -u $USER -p $PASSWORD -f $RESOURCES_DIR/token") - runOne("admin checkTablets") - runOne("admin listInstances") - runOne("admin ping") - runOne("admin dumpConfig -a -d $RESOURCES_DIR") - runOne("admin volumes") - - // TODO: test tool, jar, classname, rfile-info - // runOne("shell -u $USER -p $PASSWORD -e \"createtable testtable\"") - } + accumulo("version") + accumulo("classpath") + accumulo("admin checkTablets") + accumulo("admin listInstances") + accumulo("admin ping") + accumulo("admin volumes") - public AccumuloSliderShell runOne(String cmd) { - return AccumuloSliderShell.run(0, "--appconf $ACCUMULO_CONF $cmd") - } - - public void runBoth(String cmd) { - String clusterName = getClusterName() - AccumuloSliderShell.run(0, "--app $clusterName $cmd") - AccumuloSliderShell.run(0, "--appconf $ACCUMULO_CONF $cmd") + String zookeepers = SLIDER_CONFIG.get( + RegistryConstants.KEY_REGISTRY_ZK_QUORUM, + FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS) + String instance = tree.global.get("site.client.instance.name") + accumulo("shell -u $USER -p $PASSWORD -e \"createtable test1\"") + accumulo(InsertWithBatchWriter.class.getName() + " -i $instance -z " + + "$zookeepers -u $USER -p $PASSWORD -t test1") + accumulo(ReadData.class.getName() + " -i $instance -z $zookeepers -u " + + "$USER -p $PASSWORD -t test1 --startKey row_0 --endKey row_101") } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy index 09b8a39..cb7ad6b 100644 --- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy +++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSliderShell.groovy @@ -18,13 +18,6 @@ package org.apache.slider.funtest.framework -import org.apache.bigtop.itest.shell.Shell -import org.apache.slider.core.exceptions.SliderException -import org.apache.slider.common.tools.SliderUtils -import org.slf4j.Logger -import org.slf4j.LoggerFactory - - class AccumuloSliderShell extends ShellBase { public static File scriptFile; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/670bfb32/app-packages/accumulo/src/test/resources/clientInstallConfig-test.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/resources/clientInstallConfig-test.json b/app-packages/accumulo/src/test/resources/clientInstallConfig-test.json new file mode 100644 index 0000000..c8431ce --- /dev/null +++ b/app-packages/accumulo/src/test/resources/clientInstallConfig-test.json @@ -0,0 +1,8 @@ +{ + "schema":"http://example.org/specification/v2.0.0", + "global":{ + "client_root": "{app_install_dir}/accumulo-${accumulo.version}", + "slider_home_dir": "${slider.dir}", + "app_name": "test_script" + } +}