So autotest-local can effectively run under autotest-remote. Handle client push in a way that it can also install the entry points in system wide locations, and code can also figure out those system wide locations are present.
Signed-off-by: Lucas Meneghel Rodrigues <[email protected]> --- client/autotest-daemon | 57 ++++++++++++++++++++++ client/autotest-daemon-monitor | 61 ++++++++++++++++++++++++ client/autotest-local-streamhandler | 33 +++++++++++++ client/setup.py | 5 +- server/autotest_remote.py | 94 ++++++++++++++++++++++++++++++++----- 5 files changed, 238 insertions(+), 12 deletions(-) create mode 100755 client/autotest-daemon create mode 100755 client/autotest-daemon-monitor create mode 100755 client/autotest-local-streamhandler diff --git a/client/autotest-daemon b/client/autotest-daemon new file mode 100755 index 0000000..71f288c --- /dev/null +++ b/client/autotest-daemon @@ -0,0 +1,57 @@ +#!/usr/bin/python + +try: + import autotest.common +except ImportError: + import common +import sys, os, subprocess, fcntl + +from autotest.client import os_dep + +try: + autotest = os_dep.command('autotest-local') +except ValueError: + bindir = os.path.dirname(__file__) + autotest = os.path.join(bindir, 'autotest') + +logdir = sys.argv[1] + + +# We want to simulate the behaviour of autotest_client, where fd3 would be +# routed to stderr and fd1 & fd2 to stdout + +# HACK: grab fd3 for now +os.dup2(2, 3) + +# open up log files to use for std* +stdout = open(os.path.join(logdir, 'stdout'), 'a', 0) +stderr = open(os.path.join(logdir, 'stderr'), 'a', 0) + +# set up the file descriptors now, simulating the old behaviour +os.dup2(stdout.fileno(), 1) +os.dup2(stdout.fileno(), 2) +os.dup2(stderr.fileno(), 3) + +# we don't need the file objects any more +stdout.close() +stderr.close() + + +args = [autotest] + sys.argv[2:] +if '-H' not in args: + args[1:1] = ['-H', 'autoserv'] +cmd = ' '.join(args) + +# open up a log file for saving off the exit code +exit_file = open(os.path.join(logdir, 'exit_code'), 'w', 0) +fcntl.flock(exit_file, fcntl.LOCK_EX) + +# touch a 'started' file to indicate we've been initialized +open(os.path.join(logdir, 'started'), 'w').close() + +# run the actual autotest client and write the exit code into the log file +exit_code = subprocess.call(cmd, shell=True) +exit_file.write('%+04d' % exit_code) +exit_file.flush() +fcntl.flock(exit_file, fcntl.LOCK_UN) +exit_file.close() diff --git a/client/autotest-daemon-monitor b/client/autotest-daemon-monitor new file mode 100755 index 0000000..628bc1a --- /dev/null +++ b/client/autotest-daemon-monitor @@ -0,0 +1,61 @@ +#!/usr/bin/python + +try: + import autotest.common +except ImportError: + import common +import sys, os, signal, time, subprocess, fcntl + +logdir = sys.argv[1] +stdout_start = int(sys.argv[2]) # number of bytes we can skip on stdout +stderr_start = int(sys.argv[3]) # nubmer of bytes we can skip on stderr + +# if any of our tail processes die, the monitor should die too +def kill_self(signum, frame): + os.kill(os.getpid(), signal.SIGTERM) +signal.signal(signal.SIGCHLD, kill_self) + +devnull = open(os.devnull, 'w') + +# launch some tail processes to pump the std* streams +def launch_tail(filename, outstream, start): + path = os.path.join(logdir, filename) + argv = ['tail', '--retry', '--follow=name', '--bytes=+%d' % start, path] + # stdout=sys.stdout fails on pre-2.5 python (bug in subprocess module) + if outstream != subprocess.PIPE and outstream.fileno() == 1: + return subprocess.Popen(argv, stderr=devnull) + else: + return subprocess.Popen(argv, stdout=outstream, stderr=devnull) +stdout_pump = launch_tail('stdout', sys.stdout, stdout_start) +stderr_pump = launch_tail('stderr', sys.stderr, stderr_start) + +# wait for logdir/started to exist to be sure autotestd is started +start_time = time.time() +started_file_path = os.path.join(logdir, 'started') +while not os.path.exists(started_file_path): + time.sleep(1) + if time.time() - start_time >= 30: + raise Exception("autotestd failed to start in %s" % logdir) + +# watch the exit code file for an exit +exit_code_file = open(os.path.join(logdir, 'exit_code')) +fcntl.flock(exit_code_file, fcntl.LOCK_EX) +try: + exit_code = exit_code_file.read() + if len(exit_code) != 4: + exit_code = -signal.SIGKILL # autotestd was nuked + else: + exit_code = int(exit_code) +finally: + fcntl.flock(exit_code_file, fcntl.LOCK_UN) + exit_code_file.close() + +# tail runs in 1s polling loop, so give them a chance to finish +time.sleep(2) +# clear the SIGCHLD handler so that killing the tails doesn't kill us +signal.signal(signal.SIGCHLD, signal.SIG_DFL) +os.kill(stdout_pump.pid, signal.SIGTERM) +os.kill(stderr_pump.pid, signal.SIGTERM) + +# exit (with the same code as autotestd) +sys.exit(exit_code) diff --git a/client/autotest-local-streamhandler b/client/autotest-local-streamhandler new file mode 100755 index 0000000..0762e8d --- /dev/null +++ b/client/autotest-local-streamhandler @@ -0,0 +1,33 @@ +#!/usr/bin/python +try: + import autotest.common +except ImportError: + import common + +import sys, os, subprocess + +# We want to set the output (stdout&stderr) of the autotest binary onto our +# stdout channel. We went to get the status stream of autotest back on our +# stderr channel - we set up fd 3 for this, and harness_simple.py can +# open it later. + +# Set up file descriptor 3 as a copy of our stderr. This is the status channel +os.dup2(2,3) +# Join our stderr in with our stdout +os.dup2(1,2) + +from autotest.client import os_dep + +try: + autotest = os_dep.command('autotest-local') +except ValueError: + autodir = os.path.dirname(sys.argv[0]) + autotest = os.path.join(autodir, 'autotest') + +args = [autotest] + sys.argv[1:] +if '-H' not in args: + args.insert(1, '-H simple') +cmd = ' '.join(args) +exit_code = subprocess.call(cmd, shell=True, stderr=subprocess.STDOUT, + close_fds=False) +sys.exit(exit_code) # pass on the exit status from autotest diff --git a/client/setup.py b/client/setup.py index f2d1c44..a37b3e9 100644 --- a/client/setup.py +++ b/client/setup.py @@ -60,7 +60,10 @@ setup(name='autotest', 'autotest.client', 'autotest', ], - scripts=[os.path.join(client_dir, 'autotest-local')], + scripts=[os.path.join(client_dir, 'autotest-local'), + os.path.join(client_dir, 'autotest-local-streamhandler'), + os.path.join(client_dir, 'autotest-daemon'), + os.path.join(client_dir, 'autotest-daemon-monitor')], data_files=[('/etc/autotest', [autotest_dir + '/global_config.ini', autotest_dir + '/shadow_config.ini', ]), diff --git a/server/autotest_remote.py b/server/autotest_remote.py index 704ce5f..368b01b 100644 --- a/server/autotest_remote.py +++ b/server/autotest_remote.py @@ -2,6 +2,7 @@ import re, os, sys, traceback, time, glob, tempfile, logging from autotest.server import installable_object, prebuild, utils +from autotest.client import os_dep from autotest.client.shared import base_job, log, error, autotemp from autotest.client.shared import global_config, packages from autotest.client.shared import utils as client_utils @@ -65,10 +66,25 @@ class BaseAutotest(installable_object.InstallableObject): logging.debug('Using existing host autodir: %s', autodir) return autodir + + system_wide = True + autotest_system_wide = '/usr/bin/autotest-local' + try: + host.run('test -x %s' % utils.sh_escape(autotest_system_wide)) + logging.info("System wide install detected") + except: + system_wide = False + for path in Autotest.get_client_autodir_paths(host): try: - autotest_binary = os.path.join(path, 'autotest') - host.run('test -x %s' % utils.sh_escape(autotest_binary)) + try: + autotest_binary = os.path.join(path, 'autotest') + host.run('test -x %s' % utils.sh_escape(autotest_binary)) + except error.AutoservRunError: + if system_wide: + pass + else: + raise host.run('test -w %s' % utils.sh_escape(path)) logging.debug('Found existing autodir at %s', path) return path @@ -166,11 +182,26 @@ class BaseAutotest(installable_object.InstallableObject): def _install_using_send_file(self, host, autodir): + system_wide = True + try: + autotest_local = os_dep.command('autotest-local') + autotest_local_streamhandler = os_dep.command('autotest-local-streamhandler') + autotest_daemon = os_dep.command('autotest-daemon') + autotest_daemon_monitor = os_dep.command('autotest-daemon-monitor') + except: + system_wide = False + dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"]) light_files = [os.path.join(self.source_material, f) for f in os.listdir(self.source_material) if f not in dirs_to_exclude] + if system_wide: + light_files.append(autotest_local) + light_files.append(autotest_local_streamhandler) + light_files.append(autotest_daemon) + light_files.append(autotest_daemon_monitor) + # there should be one and only one grubby tarball grubby_glob = os.path.join(self.source_material, "deps/grubby/grubby-*.tar.bz2") @@ -486,12 +517,20 @@ class _BaseRun(object): def verify_machine(self): - binary = os.path.join(self.autodir, 'autotest') + system_wide = True + binary = os.path.join('/usr/bin/autotest-local') try: - self.host.run('ls %s > /dev/null 2>&1' % binary) + self.host.run('test -x %s' % binary) except: - raise error.AutoservInstallError( - "Autotest does not appear to be installed") + system_wide = False + + if not system_wide: + binary = os.path.join(self.autodir, 'autotest') + try: + self.host.run('test -x %s' % binary) + except: + raise error.AutoservInstallError( + "Autotest does not appear to be installed") if not self.parallel_flag: tmpdir = os.path.join(self.autodir, 'tmp') @@ -517,23 +556,56 @@ class _BaseRun(object): def get_background_cmd(self, section): - cmd = ['nohup', os.path.join(self.autodir, 'autotest_client')] + system_wide = True + system_wide_client_path = '/usr/bin/autotest-local-streamhandler' + try: + self.host.run('test -x %s' % system_wide_client_path) + except: + system_wide = False + + if system_wide: + cmd = ['nohup', system_wide_client_path] + else: + cmd = ['nohup', os.path.join(self.autodir, 'autotest_client')] cmd += self.get_base_cmd_args(section) cmd += ['>/dev/null', '2>/dev/null', '&'] return ' '.join(cmd) def get_daemon_cmd(self, section, monitor_dir): - cmd = ['nohup', os.path.join(self.autodir, 'autotestd'), - monitor_dir, '-H autoserv'] + system_wide = True + system_wide_client_path = '/usr/bin/autotest-daemon' + try: + self.host.run('test -x %s' % system_wide_client_path) + except: + system_wide = False + + if system_wide: + cmd = ['nohup', system_wide_client_path, + monitor_dir, '-H autoserv'] + else: + cmd = ['nohup', os.path.join(self.autodir, 'autotestd'), + monitor_dir, '-H autoserv'] + cmd += self.get_base_cmd_args(section) cmd += ['>/dev/null', '2>/dev/null', '&'] return ' '.join(cmd) def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read): - cmd = [os.path.join(self.autodir, 'autotestd_monitor'), - monitor_dir, str(stdout_read), str(stderr_read)] + system_wide = True + system_wide_client_path = '/usr/bin/autotest-daemon-monitor' + try: + system_wide = self.host.run('test -x %s' % system_wide_client_path) + except: + system_wide = False + + if system_wide: + cmd = [system_wide_client_path, + monitor_dir, str(stdout_read), str(stderr_read)] + else: + cmd = [os.path.join(self.autodir, 'autotestd_monitor'), + monitor_dir, str(stdout_read), str(stderr_read)] return ' '.join(cmd) -- 1.7.11.4 _______________________________________________ Autotest-kernel mailing list [email protected] https://www.redhat.com/mailman/listinfo/autotest-kernel
