Author: wade
Date: 2007-07-20 21:15:13 -0400 (Fri, 20 Jul 2007)
New Revision: 82404
Modified:
trunk/release/packaging/
trunk/release/packaging/build
trunk/release/packaging/do-install-zip-pkgs
trunk/release/packaging/do-msvn-tar
trunk/release/packaging/install-deps
trunk/release/packaging/jail-do
trunk/release/packaging/jail-do-all
trunk/release/packaging/mktarball
trunk/release/pyutils/buildenv.py
trunk/release/pyutils/config.py
trunk/release/pyutils/launch_process.py
trunk/release/pyutils/network.py
trunk/release/pyutils/packaging.py
trunk/release/pyutils/utils.py
Log:
new monobuild feature: ability for anyone (well, with vpn access) to
interrupt builds, and ability for anyone to lock a client machine so
monobuild won't farm a build to that client, will post more instructions
on the list
logger changes:
-had to use my_logger. Any place the var 'logger' was being used,
it was overwriting the class name (oops)
jail-do:
-can log in using old syntax. To execute a command, you must use "run
<command>"
-several new actions: interrupting and locking build clients
Lots of locking changes to do best effort locking from multiple hosts
Remove some shell-isms and use python code instead
With interruptability, had option that some commands can't be
interrupted (ie: killing an rpm command could trash the rpm db,
also during svn checkouts, don't kill svn processes)
Also gives ability to use remove execution code internally
without locking ourselves out
config.py:
-clean up client and package lists
Property changes on: trunk/release/packaging
___________________________________________________________________
Name: svn:ignore
- packages
snapshot_packages
snapshot_zip_packages
zip_packages
external_zip_pkg
external_packages
logs
sources
snapshot_sources
tarball_map
+ packages
snapshot_packages
snapshot_zip_packages
zip_packages
external_zip_pkg
external_packages
logs
sources
snapshot_sources
tarball_map
distro_source_key
.smbpass
Modified: trunk/release/packaging/build
===================================================================
--- trunk/release/packaging/build 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/build 2007-07-21 01:15:13 UTC (rev 82404)
@@ -46,6 +46,9 @@
# Some tests are reporting 255 on killed processes?
if code == utils.KILLED_EXIT_CODE or code == 214 or code == 137 or code
== 255:
return "timeout"
+ # Most of the time interrupts will show as 'timeout'
+ elif code == utils.INTERRUPTED_EXIT_CODE:
+ return "interrupted"
elif code:
if test:
return "testfailure"
@@ -68,7 +71,7 @@
log_obj = logger.Logger(filename=LOGFILE,
print_screen=print_screen)
build_info.update_step(name, state="inprogress",
log=os.path.basename(LOGFILE), start=utils.get_time(), finish="")
- (result, output) = conf.buildenv.execute_code("%s %s " %
(step_env_pre_cmd, code), working_dir=remote_src_dir, env=conf.env_vars,
logger=log_obj, output_timeout=timeout, max_output_size=max_output_size)
+ (result, output) = conf.buildenv.execute_code("%s %s " %
(step_env_pre_cmd, code), working_dir=remote_src_dir, env=conf.env_vars,
my_logger=log_obj, output_timeout=timeout, max_output_size=max_output_size)
# Get results, if any
results_dir = os.path.join(config.build_info_dir,
build_info.rel_files_dir, 'files', 'steps')
@@ -238,7 +241,7 @@
else:
install_deps_H_or_R = HEAD_or_RELEASE
print "Installing dependencies..."
-(code, output) = utils.launch_process("./install-deps --HEAD_or_RELEASE=%s %s
%s" % (install_deps_H_or_R, conf.conf_lock_filename, package_name),
logger=log_obj)
+(code, output) = utils.launch_process("./install-deps --HEAD_or_RELEASE=%s %s
%s" % (install_deps_H_or_R, conf.conf_lock_filename, package_name),
my_logger=log_obj)
if code:
conf.buildenv.unlock_env()
build_info.update_step("install-deps", state="failure",
finish=utils.get_time())
@@ -292,8 +295,10 @@
step_env_pre_cmd = ""
# Clean up
-conf.buildenv.execute_command('rm -Rf %s/steps' %
(conf.env_vars['build_location']) )
-conf.buildenv.execute_command('rm -Rf %s; mkdir -p -m777 %s' %
(temp_build_dir, temp_build_dir) )
+conf.buildenv.remove_path('%s/steps' % conf.env_vars['build_location'] )
+conf.buildenv.remove_path(temp_build_dir)
+conf.buildenv.make_path(temp_build_dir)
+
conf.buildenv.copy_to(files_to_copy, temp_build_dir, compress=0)
conf.buildenv.copy_to(support_files_to_copy, conf.env_vars['build_location'],
compress=0)
@@ -314,7 +319,7 @@
build_info.update_build(state="failure", finish=utils.get_time())
sys.exit(7)
-(code, output) = conf.buildenv.execute_command(build_command,
env=conf.env_vars, logger=log_obj, output_timeout=timeout,
max_output_size=max_output_size)
+(code, output) = conf.buildenv.execute_command(build_command,
env=conf.env_vars, my_logger=log_obj, output_timeout=timeout,
max_output_size=max_output_size)
if code:
build_failed(conf, code, LOGFILE, build_info, package)
@@ -330,12 +335,12 @@
code = 0
output = ""
for file in files:
- (tmp_code, tmp_output) = utils.launch_process("unzip -qt " +
file, logger=log_obj)
+ (tmp_code, tmp_output) = utils.launch_process("unzip -qt " +
file, my_logger=log_obj)
if tmp_code: code = 1
output += tmp_output
else:
files = glob.glob(package_version_path + os.sep + "*.rpm")
- (code, output) = utils.launch_process("rpm -K " + " ".join(files),
logger=log_obj)
+ (code, output) = utils.launch_process("rpm -K " + " ".join(files),
my_logger=log_obj)
if code:
log_obj.log("Verifying packages failed... (usually because of lack of
disk space or incomplete copy)\n")
Modified: trunk/release/packaging/do-install-zip-pkgs
===================================================================
--- trunk/release/packaging/do-install-zip-pkgs 2007-07-21 00:26:17 UTC (rev
82403)
+++ trunk/release/packaging/do-install-zip-pkgs 2007-07-21 01:15:13 UTC (rev
82404)
@@ -23,7 +23,7 @@
dest = sys.argv[1]
reloc = sys.argv[2]
-package_files = sys.argv[3:]
+package_files = glob.glob(sys.argv[3] + os.sep + "*")
print " *** Installing to %s for destination %s *** " % (dest, reloc)
Modified: trunk/release/packaging/do-msvn-tar
===================================================================
--- trunk/release/packaging/do-msvn-tar 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/do-msvn-tar 2007-07-21 01:15:13 UTC (rev 82404)
@@ -9,15 +9,23 @@
import shutil
import glob
import distutils.dir_util
+import getpass
sys.path.append('.')
import packaging
+# Some things we'll need to run as root
+EXEC_AS_ROOT = 'sudo'
+
(HEAD_or_RELEASE, package, version) = sys.argv[1:]
os.chdir('source_dist_creator')
+# they key needs to be owned by the user so ssh doesn't reject the key
+os.system('%s chown %s distro_source_key' % (EXEC_AS_ROOT, getpass.getuser()))
+os.system("%s mount -t proc none /proc &> /dev/null" % EXEC_AS_ROOT)
+
temp_dir = os.getcwd()
dummy_env = ""
Modified: trunk/release/packaging/install-deps
===================================================================
--- trunk/release/packaging/install-deps 2007-07-21 00:26:17 UTC (rev
82403)
+++ trunk/release/packaging/install-deps 2007-07-21 01:15:13 UTC (rev
82404)
@@ -60,7 +60,7 @@
log_obj = logger.Logger(filename=config.devnull)
# skip_alternates, because locking is handled in build and mktarball (if we
had better locking, this wouldn't be needed)
-target_conf = packaging.buildconf(target, logger=log_obj, skip_alternates=True)
+target_conf = packaging.buildconf(target, my_logger=log_obj,
skip_alternates=True)
if build_source:
build_deps = False
@@ -104,12 +104,14 @@
build_location = target_conf.env_vars['build_location']
# Clean out deps area (/tmp/build_deps does not use build_location for
relocation purposes)
-(code, output) = target_conf.buildenv.execute_command("rm -rf
%s/install-packages /tmp/build_deps" % (build_location) )
-if code:
+(code, output) = target_conf.buildenv.remove_path("/tmp/build_deps")
+(code2, output2) = target_conf.buildenv.remove_path(build_location + os.sep +
'install-packages')
+if code or code2:
print "Error cleaning up... exiting"
sys.exit(1)
-target_conf.buildenv.execute_command("mkdir -p -m777 %s/install-packages" %
build_location)
+target_conf.buildenv.make_path("%s/install-packages" % build_location)
+
# copy packages
# It would be nice to see the output...
target_conf.buildenv.copy_to(files, '%s/install-packages' % build_location,
compress=0)
@@ -119,7 +121,7 @@
target_conf.buildenv.copy_to(['do-install-zip-pkgs',
'../pyutils/utils.py', '../pyutils/rpm2cpio.py'], build_location)
# These must NOT be in the build_location, otherwise it will throw off
the relocation
- (code, output) = target_conf.buildenv.execute_command(
"%s/do-install-zip-pkgs /tmp/build_deps /tmp/build_deps %s/install-packages/*"
% (build_location, build_location) )
+ (code, output) = target_conf.buildenv.execute_command(
"%s/do-install-zip-pkgs /tmp/build_deps /tmp/build_deps %s/install-packages" %
(build_location, build_location) )
else:
@@ -133,7 +135,8 @@
target_conf.buildenv.execute_command('%s/remove-rpms %s' %
(build_location, " ".join(packs_to_remove)), exec_as_root=1)
code = 0
if files:
- (code, output) = target_conf.buildenv.execute_command('rpm -Uvh
%s/install-packages/*.rpm' % build_location, exec_as_root=1)
+ # Make this non-interruptable because we don't want the rpm db
trashed
+ (code, output) = target_conf.buildenv.execute_command('rpm -Uvh
%s/install-packages/*.rpm' % build_location, exec_as_root=1,
interruptable=False)
# Exit with return code...
sys.exit(code)
Modified: trunk/release/packaging/jail-do
===================================================================
--- trunk/release/packaging/jail-do 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/jail-do 2007-07-21 01:15:13 UTC (rev 82404)
@@ -1,28 +1,73 @@
#!/usr/bin/env python
import sys
-import string
sys.path += ['../pyutils']
import buildenv
import packaging
+def usage():
+ print "Usage: ./jail-do <distro conf> <action> [ <action args> ]"
+ print " where <action> is one of: login | run | lock | unlock |
is_locked | interrupt"
+ print " Example: ./jail-do exec suse-102-i586 run ls"
+ print " 'run' is the only action that takes <action args>,"
+ print " which will run that command"
+ print " 'unlock' also removes interrupt file flag"
+
if len(sys.argv) < 2:
- print "Usage: ./jail-do <distro conf> [command]"
- print " Example: ./jail-do suse-93-i586 ls"
- print " If no [command] is given, an interactive login will proceed"
+ usage()
sys.exit(1)
-conf = packaging.buildconf(sys.argv[1], skip_alternates=True)
+env_name = sys.argv[1]
+action_options = []
+
+# Default to login action if none is specified
+if len(sys.argv) == 2:
+ action = "login"
+else:
+ action = sys.argv[2]
+ if len(sys.argv) > 3:
+ action_options = sys.argv[3:]
+
+
+conf = packaging.buildconf(env_name, skip_alternates=True)
conf.buildenv.print_output=True
-if len(sys.argv) > 2:
- command = string.join(sys.argv[2:], ' ')
+if action == "login":
+ print "Logging in"
+ conf.buildenv.login()
+elif action == "run":
+ command = ' '.join(action_options)
sys.stderr.write("Executing: %s\n" % command)
status, output = conf.buildenv.execute_command(command)
+ if status == 255:
+ print "Process was killed!"
sys.exit(status)
+elif action == "lock":
+ if conf.buildenv.lock_env():
+ print "Locked successfully"
+ else:
+ print "Failed to lock"
+ sys.exit(1)
+elif action == "unlock":
+ if conf.buildenv.unlock_env(clear_interrupt=True):
+ print "Unlocked successfully, interrupt lock cleared if existed"
+ else:
+ print "Failed to unlock"
+ sys.exit(1)
+elif action == "is_locked":
+ if conf.buildenv.is_locked():
+ # return 0 (true) if it is locked
+ print "Locked"
+ else:
+ # return 1 (false) if it's not
+ print "Unlocked"
+ sys.exit(1)
+elif action == "interrupt":
+ (code, output) = conf.buildenv.interrupt()
+ sys.exit(code)
else:
- print "Logging in"
- conf.buildenv.login()
+ print "Unknown action: " + action
+ sys.exit(1)
Modified: trunk/release/packaging/jail-do-all
===================================================================
Modified: trunk/release/packaging/mktarball
===================================================================
--- trunk/release/packaging/mktarball 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/mktarball 2007-07-21 01:15:13 UTC (rev 82404)
@@ -72,7 +72,7 @@
log_obj.log("Sources path: %s\n" % package_obj.source_fullpath)
log_obj.log("Using distro: %s\n" % distro)
-conf = packaging.buildconf(distro, logger=log_obj)
+conf = packaging.buildconf(distro, my_logger=log_obj)
if conf.buildenv.is_locked():
log_obj.log("%s jail is busy\n" % distro )
@@ -84,7 +84,7 @@
repo.add_file(HEAD_or_RELEASE, package, version, "inprogress")
if package_obj.info.has_key('MONO_DEPS'):
- (status, output) = utils.launch_process('./install-deps --build_source
%s %s' % (conf.conf_lock_filename, package), logger=log_obj)
+ (status, output) = utils.launch_process('./install-deps --build_source
%s %s' % (conf.conf_lock_filename, package), my_logger=log_obj)
if status:
conf.buildenv.unlock_env()
log_obj.log("Dependency installation failed\n")
@@ -94,7 +94,7 @@
temp_dir = conf.env_vars['build_location'] + os.sep + 'source_dist_creator'
# Clean up
-conf.buildenv.execute_command("rm -Rf %s" % temp_dir)
+conf.buildenv.remove_path(temp_dir)
# Copy some files over and execute!
# Also copy over def_alias files
@@ -111,10 +111,8 @@
conf.buildenv.copy_to(files_to_copy, temp_dir, mode='scp')
log_obj.log("Starting to build source dist...\n")
-# they key needs to be owned by the user so ssh doesn't reject the key
-(status, output) = conf.buildenv.execute_command('chown %s
%s/distro_source_key' % (conf.info['username'], temp_dir), exec_as_root=1)
-(status, output) = conf.buildenv.execute_command('mount /proc &> /dev/null',
exec_as_root=1)
-(status, output) = conf.buildenv.execute_command('%s/do-msvn-tar %s %s %s' %
(temp_dir, HEAD_or_RELEASE, package, version) )
+# Make this non-interruptable because we don't want the source cache corrupted
+(status, output) = conf.buildenv.execute_command('%s/do-msvn-tar %s %s %s' %
(temp_dir, HEAD_or_RELEASE, package, version), interruptable=False )
if status:
conf.buildenv.unlock_env()
Modified: trunk/release/pyutils/buildenv.py
===================================================================
--- trunk/release/pyutils/buildenv.py 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/buildenv.py 2007-07-21 01:15:13 UTC (rev 82404)
@@ -7,9 +7,6 @@
"""
import os
-import config
-import network
-import utils
import socket
import distutils.dir_util
import shutil
@@ -17,13 +14,16 @@
import tempfile
import getpass
+import config
+import network
+import utils
import logger
import pdb
class buildenv:
- def __init__(self, username="", hostname="", root_dir="",
lock_filename="", target_command_prefix="", env=config.env_vars, logger="",
login_mode='ssh', copy_mode='scp', exec_mode='ssh'):
+ def __init__(self, username="", hostname="", root_dir="",
lock_filename="", target_command_prefix="", env=config.env_vars, my_logger="",
login_mode='ssh', copy_mode='scp', exec_mode='ssh'):
self.login_mode=login_mode
self.copy_mode=copy_mode
@@ -57,11 +57,15 @@
# Custom identifier to use at various places
self.uid = "[EMAIL PROTECTED]:%s" % (self.username,
self.hostname, root_dir)
+ # Host lock filename
if lock_filename:
- self.lock_filename = lock_filename
+ self.host_lock_filename = lock_filename
else:
- self.lock_filename = self.uid
+ self.host_lock_filename = self.uid
+ # Client lock filename
+ self.client_lock_filename = self.build_location + os.sep +
"locked"
+
# Optional...
self.root_dir = root_dir
self.env = env
@@ -73,12 +77,12 @@
# initialize objects for later
self.modes = {}
- self.modes['ssh'] = network.ssh(username=username,
hostname=hostname, env=self.env, logger=logger)
- self.modes['scp'] = network.scp(username=username,
hostname=hostname, env=self.env, logger=logger)
- self.modes['smbclient'] = network.smbclient(username=username,
hostname=hostname, env=self.env, logger=logger)
- self.modes['local'] = local(username=username,
hostname=hostname, env=self.env, logger=logger)
+ self.modes['ssh'] = network.ssh(username=username,
hostname=hostname, env=self.env, my_logger=my_logger)
+ self.modes['scp'] = network.scp(username=username,
hostname=hostname, env=self.env, my_logger=my_logger)
+ self.modes['smbclient'] = network.smbclient(username=username,
hostname=hostname, env=self.env, my_logger=my_logger)
+ self.modes['local'] = local(username=username,
hostname=hostname, env=self.env, my_logger=my_logger)
- self.logger = logger
+ self.logger = my_logger
# chroot options
if root_dir:
@@ -88,7 +92,13 @@
# Set flag so we don't copy over and over for execute_command
self.exec_util_files_copied = False
+ # empty logger to use at various places
+ self.empty_logger = logger.Logger(config.devnull,
print_screen=0)
+ # Filenames for process jobs
+ self.interrupted_file = self.build_location + os.sep +
'interrupted'
+ self.pgkill_file = self.build_location + os.sep + 'pgid_kill'
+
def login(self, mode=""):
if not mode:
@@ -99,13 +109,14 @@
mode = self.modes[mode].login(command)
- def execute_command(self, command, working_dir="", mode="",
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="",
env={}, logger=""):
+ def execute_command(self, command, working_dir="", mode="",
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="",
env={}, my_logger="", interruptable=True):
+ """setting interruptable to False is for internal and really
important commands (ie: killing rpm installs/removes could cause havoc) Using
that flag doesn't write a pgid file"""
if not working_dir:
working_dir = self.build_location
- if not logger:
- logger = self.logger
+ if not my_logger:
+ my_logger = self.logger
if not mode:
mode = self.exec_mode
@@ -117,6 +128,10 @@
if not env:
env = self.env
+ # If true, but no filename passed in, use this default filename
+ if interruptable == True:
+ interruptable = self.pgkill_file
+
# launch process options. pass everything through launch
process, and have it save the pgid to enable process interruptions
# TODO: this doesn't play nicely with local commands, because
this code is in the process group.
# We need the local execute to be in a different process group
for this to work
@@ -129,6 +144,8 @@
launch_options += " --terminate_reg=%s" % terminate_reg
if working_dir:
launch_options += " --working_dir=%s" % working_dir
+ if interruptable:
+ launch_options += " --interruptable=%s" % interruptable
# Build environment string to pass to launch_process
if env:
@@ -150,11 +167,14 @@
# Double quotes are very meticulous
command = '%s %s %s "%s %s %s"' % (self.root_dir_options,
self.env['shell_path'], "-c", self.target_command_prefix, sudo_opts, command)
- return self.modes[mode].execute_command(command, logger=logger)
+ return self.modes[mode].execute_command(command,
my_logger=my_logger)
- def execute_code(self, code, working_dir="", interpreter="",
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="",
env={}, logger=""):
+ def execute_code(self, code, working_dir="", interpreter="",
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="",
env={}, my_logger="", interruptable=True):
""" interpreter can be anything that works... mcs *.cs; mono,
perl, python, etc... """
+ if not my_logger:
+ my_logger = self.logger
+
# Default to shell code, but can be overwritten
if not interpreter:
interpreter = self.env['shell_path']
@@ -185,8 +205,9 @@
args['max_output_size'] = max_output_size
args['terminate_reg'] = terminate_reg
args['env'] = env
- args['logger'] = logger
+ args['my_logger'] = my_logger
args['working_dir'] = working_dir
+ args['interruptable'] = interruptable
command = "%s %s" % (interpreter, self.build_location + os.sep
+ file_basename)
code, output = self.execute_command(command, **args)
@@ -197,12 +218,32 @@
return code, output
def make_path(self, dir, mode=""):
-
+ """Create a path
+ This must be implemented in the lower layers since
execute_* depend on it"""
if not mode:
mode = self.exec_mode
return self.modes[mode].make_path(self.root_dir + dir)
+ def remove_path(self, dir):
+ """Remove a path. Since no internal functions depend on this,
it can be a higher layer function"""
+
+ remove_path_code = """
+import os
+import shutil
+
+path = '%s'
+
+if os.path.exists(path):
+ # Newer versions of python's rmtree will remove files, but not all
versions
+ if os.path.isdir(path):
+ shutil.rmtree(path)
+ else:
+ os.unlink(path)
+""" % dir
+
+ return self.execute_code(remove_path_code,
interpreter=self.env['python_path'], interruptable=False)
+
def copy_to(self, src, dest, compress=True, mode=""):
if not mode:
@@ -217,7 +258,7 @@
print "No files to copy..."
return 0, ""
- # Make sure it exists
+ ## Make sure it exists (turns into circular recursion at this
point)
self.make_path(dest)
return self.modes[mode].copy_to(src, self.root_dir + dest,
compress=compress)
@@ -245,37 +286,129 @@
return self.modes[mode].copy_from(src, dest, compress=compress)
- def interrupt_command(self):
- pass
+ def interrupt(self):
+ """Write interrupted file flag, read pgid_kill file, and blow
process group away!"""
+
+ flag_code = """
+import os
+
+path = '%s'
+
+fd = open(path, 'w')
+fd.write("")
+fd.close()""" % self.interrupted_file
+
+ (code, output) = self.execute_code(flag_code,
interpreter=self.env['python_path'], interruptable=False)
+
+ if code:
+ return (code, output)
+
+ kill_code = """
+import os
+import signal
+import sys
+
+path = '%s'
+client_lock = '%s'
+
+if not os.path.exists(path):
+ if os.path.exists(client_lock):
+ print "Client is busy and uninterruptable, try again in a few
minutes"
+ sys.exit(1)
+ print "Client is not running an interruptable command, exiting..."
+ print " (it could be running an non-interruptable command, but the
client isn't locked)"
+ sys.exit(0)
+
+
+fd = open(path)
+pgid = int(fd.read())
+fd.close()
+os.unlink(path)
+
+# Kill the process group
+print "Killing process group: %%d" %% pgid
+try:
+ os.kill(-pgid, signal.SIGKILL)
+except OSError:
+ print "Error killing process group %%d (already gone?)" %% pgid
+ sys.exit(1)
+""" % (self.pgkill_file, self.client_lock_filename)
+
+ return self.execute_code(kill_code,
interpreter=self.env['python_path'], interruptable=False)
# TODO: use os level locking here
def lock_env(self):
- fd = open(self.lock_filename, 'w')
+ fd = open(self.host_lock_filename, 'w')
fd.write("")
fd.close()
+
+ # also put file in jail to signify that it's locked (method not
perfect, but should be good enough)
+ #self.execute_command("touch %s" % self.client_lock_filename)
+
+ lock_code = """
+path = '%s'
+
+fd = open(path, 'w')
+fd.write("")
+fd.close()
+
+""" % self.client_lock_filename
+
+ self.execute_code(lock_code,
interpreter=self.env['python_path'], interruptable=False)
+
return True
- def unlock_env(self):
- if os.path.exists(self.lock_filename):
- os.unlink(self.lock_filename)
+ def unlock_env(self, clear_interrupt=False):
+ if os.path.exists(self.host_lock_filename):
+ os.unlink(self.host_lock_filename)
+ host_ret = True
+ else:
+ print "unlock_env: host lockfile already removed"
+ host_ret = False
+
+ # also remove interrupt file if it is there
+ if clear_interrupt:
+ (ret, output) = self.remove_path(self.interrupted_file)
+ (client_ret, output) =
self.remove_path(self.client_lock_filename)
+
+ # Return False on failure
+ if host_ret or not client_ret:
return True
else:
- print "Build environment already unlocked"
return False
def is_locked(self):
- return os.path.exists(self.lock_filename)
+ # Check host lockfile
+ if os.path.exists(self.host_lock_filename):
+ host_ret = True
+ else:
+ host_ret = False
+
+ path_exists_code = """
+import os
+import sys
+
+path = '%s'
+
+if not os.path.exists(path):
+ sys.exit(1)
+""" % self.client_lock_filename
+
+ (client_ret, output) = self.execute_code(path_exists_code,
interpreter=self.env['python_path'], interruptable=False)
+
+ # if host lock or client lock exists, we're locked
+ if host_ret or not client_ret:
+ return True
+ else:
+ return False
+
def offline(self):
- old_logger = self.logger
- self.logger = logger.Logger(config.devnull, print_screen=0)
# Run some arbitrary command that will always return true
- (code, output) = self.execute_command("ls")
+ (code, output) = self.execute_command("ls",
my_logger=self.empty_logger, interruptable=False)
- self.logger = old_logger
-
if code: return 1
else: return 0
@@ -283,20 +416,24 @@
class local:
"""Class implementing the same interface as a network object, but the
jail or machine is on the host machine"""
- def __init__(self, username, hostname, env="", logger=""):
+ def __init__(self, username, hostname, env="", my_logger=""):
self.username = username
self.hostname = hostname
- self.logger = logger
+ self.logger = my_logger
def login(self, command):
os.system(command)
- def execute_command(self, command, logger=""):
+ def execute_command(self, command, my_logger=""):
+
+ if not my_logger:
+ my_logger = self.logger
+
# TODO: execute this under a new pgid so only it gets killed,
not us all
# check
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012
# Or, put that code in the utils.launch_process routine, which
would make it more robust
- return utils.launch_process(command, logger=logger)
+ return utils.launch_process(command, my_logger=my_logger)
# For local copyings, you can use the same commands (scp, tar), just
not over ssh (scp?)
# TODO: But how to know what to use... scp or tar, because they behave
differently
@@ -305,7 +442,7 @@
# difference: # Note: tar mode appends src path of file to dest (just
the way tar works)"""
# tar mode handles symbolic links and preserving time stamps, unlike
scp.
- def copy_to(self, src, dest, compress=True, logger=""):
+ def copy_to(self, src, dest, compress=True, my_logger=""):
results = []
# Glob each of the src filesnames to allow wildcards
new_src = []
@@ -329,11 +466,11 @@
return 0, "\n".join(results)
- def copy_from(self, src, dest, compress=True, logger=""):
+ def copy_from(self, src, dest, compress=True, my_logger=""):
""" No need to duplicate this, since we're on a local machine,
use the same method"""
- return self.copy_to(src, dest, compress=compress, logger=logger)
+ return self.copy_to(src, dest, compress=compress,
my_logger=my_logger)
- def make_path(self, dir):
+ def make_path(self, dir):
"""Create a path"""
error = 0
@@ -342,5 +479,3 @@
except:
error = 1
return error
-
-
Modified: trunk/release/pyutils/config.py
===================================================================
--- trunk/release/pyutils/config.py 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/config.py 2007-07-21 01:15:13 UTC (rev 82404)
@@ -63,7 +63,20 @@
td_num_sequential = 10
# static list of packages to create tarballs for
-td_packages = ['mono', 'libgdiplus', 'mono-basic', 'mono-tools',
'monodevelop', 'monodoc', 'xsp', 'gecko-sharp-2.0', 'mono-debugger',
'gtk-sharp', 'mod_mono', 'olive' ]
+td_packages = """
+ gecko-sharp2
+ gtk-sharp
+ libgdiplus
+ mod_mono
+ mono
+ mono-basic
+ mono-debugger
+ mono-tools
+ monodevelop
+ monodoc
+ olive
+ xsp
+""".split()
# builds each and every checkin if true, otherwise, only build the latest
checkin
td_sequential = False
@@ -85,20 +98,20 @@
# List of platforms/packages
sd_latest_build_distros = """
- sles-9-i586
- sles-9-x86_64
- win-4-i386
+ debian-4-arm
+ debian-4-sparc
macos-10-ppc
macos-10-x86
- sunos-8-sparc
- sunos-10-x86
+ sles-9-i586
sles-9-ia64
+ sles-9-ppc
sles-9-s390
sles-9-s390x
- sles-9-ppc
- debian-4-arm
- debian-4-sparc
+ sles-9-x86_64
+ sunos-8-sparc
+ sunos-10-x86
suse-100-i586
+ win-4-i386
""".split()
sd_latest_build_packages = td_packages
##############################################
@@ -135,6 +148,7 @@
#'shell_path': '/bin/sh',
# We depend on bash (ie: env string in execute_command)
'shell_path': '/bin/bash',
+ 'python_path': 'python', # full path can be set in conf file
for individual jails
}
######################################
Modified: trunk/release/pyutils/launch_process.py
===================================================================
--- trunk/release/pyutils/launch_process.py 2007-07-21 00:26:17 UTC (rev
82403)
+++ trunk/release/pyutils/launch_process.py 2007-07-21 01:15:13 UTC (rev
82404)
@@ -7,7 +7,7 @@
import utils
def usage():
- print "Usage: ./launch_process.py [--terminate_reg=<reg string>]
[--output_timeout=<secs>] [--max_output_size=<num chars>]
[--kill_process_group] <command>"
+ print "Usage: ./launch_process.py [--terminate_reg=<reg string>]
[--output_timeout=<secs>] [--max_output_size=<num chars>]
[--kill_process_group] [--interruptable] <command>"
def commandline():
@@ -17,7 +17,7 @@
# Options for launch process
try:
- opts, command = getopt.getopt(sys.argv[1:], "", [
"terminate_reg=", "output_timeout=", "max_output_size=", "kill_process_group",
"env=", "working_dir=" ])
+ opts, command = getopt.getopt(sys.argv[1:], "", [
"terminate_reg=", "output_timeout=", "max_output_size=", "kill_process_group",
"env=", "working_dir=", "interruptable=" ])
except getopt.GetoptError:
usage()
sys.exit(1)
@@ -44,6 +44,8 @@
print "Failed to chdir: " + value
print "Exiting..."
sys.exit(1)
+ if option == "--interruptable":
+ args['interruptable'] = value
command = " ".join(command)
code, output = utils.launch_process(command, **args)
Modified: trunk/release/pyutils/network.py
===================================================================
--- trunk/release/pyutils/network.py 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/network.py 2007-07-21 01:15:13 UTC (rev 82404)
@@ -13,13 +13,13 @@
class ssh:
- def __init__(self, username, hostname, env="", logger=""):
+ def __init__(self, username, hostname, env="", my_logger=""):
# Ignore host checks, and blowfish might be faster and less cpu
intensive
self.options = '-o "BatchMode yes" -o "StrictHostKeyChecking
no" -o "Cipher blowfish" '
self.username = username
self.hostname = hostname
- self.logger = logger
+ self.logger = my_logger
# Get tar paths (required from copy to/from)
if env and env.has_key('tar_path'):
@@ -34,20 +34,26 @@
# Can't use utils.launch_process here because we need an
interactive terminal
return os.system(command)
- def execute_command(self, command, logger=""):
+ def execute_command(self, command, my_logger=""):
+ if not my_logger:
+ my_logger = self.logger
+
# Escape these so they get interpolated by the remote machine
command = command.replace('$', '\$')
# Put single quotes around the command so the whole command
gets passed over ssh
# (must use single quotes here since command may have double
quotes (shell allows double quotes inside of single, without escaping)
command = "ssh %s [EMAIL PROTECTED] '%s'" % (self.options,
self.username, self.hostname, command)
- return utils.launch_process(command, logger=logger)
+ return utils.launch_process(command, my_logger=my_logger)
- def copy_to(self, src, dest, compress=True, logger=""):
+ def copy_to(self, src, dest, compress=True, my_logger=""):
"""Args: src (list of strings), dest, Returns: (exit_code,
output).
Optional args: compress=True of False"""
+ if not my_logger:
+ my_logger = self.logger
+
src = " ".join(src)
if compress:
@@ -59,9 +65,9 @@
# Note: the -f - option to the remote tar is required for
solaris tar, otherwise it tries to read from a tape
command = "%s -%spc %s | ssh %s [EMAIL PROTECTED] 'cd %s ; %s
-%spvxf - ' " % (self.local_tar_path, compress_option, src, self.options,
self.username, self.hostname, dest, self.remote_tar_path, compress_option )
- return utils.launch_process(command, logger=logger)
+ return utils.launch_process(command, my_logger=my_logger)
- def copy_from(self, src, dest, compress=True, logger=""):
+ def copy_from(self, src, dest, compress=True, my_logger=""):
"""Args: (src (list), dest) Returns: (exit_code, output).
Optional args: compress=0 or 1
@@ -69,6 +75,9 @@
# I guess I could also use zip/unzip... oh well (What does
this mean?)
# Note: tar mode appends src path of file to dest (just the
way tar works)"""
+ if not my_logger:
+ my_logger = self.logger
+
if compress:
# CompressionLevel only works for protocol 1...
(bummer)
compress_option = "z"
@@ -84,30 +93,33 @@
# Note: the -f - option to the remote tar is required for
solaris tar, otherwise it tries to read from a tape
command = "cd %s; ssh %s [EMAIL PROTECTED] ' %s %s -%spcf - %s
' | %s -%spvxf - " % (dest, self.options, self.username, self.hostname,
self.remote_tar_path, tar_dir_options, compress_option, files,
self.local_tar_path, compress_tar_option )
- return utils.launch_process(command, logger=logger)
+ return utils.launch_process(command, my_logger=my_logger)
def make_path(self, dir):
- """Args: (dir) Returns: (mkdir exit code)."""
+ """Args: (dir) Returns: (mkdir exit code and output)."""
return self.execute_command("mkdir -p -m777 " + dir)
class scp:
- def __init__(self, username, hostname, env="", logger=""):
+ def __init__(self, username, hostname, env="", my_logger=""):
# Ignore host checks, and blowfish might be faster and less cpu
intensive
self.options = '-o "StrictHostKeyChecking no" -o "Cipher
blowfish" '
self.username = username
self.hostname = hostname
- self.logger = logger
+ self.logger = my_logger
- def copy_to(self, src, dest, compress=True, logger=""):
+ def copy_to(self, src, dest, compress=True, my_logger=""):
"""Args: src (list of strings), dest, Returns: (exit_code,
output).
Optional args: compress=0 or 1 (but doesn't actually do
anything in ssh protocol 2... oh well
Note, in scp mode, recursive is on by default"""
+ if not my_logger:
+ my_logger = self.logger
+
src = " ".join(src)
if compress:
@@ -118,14 +130,17 @@
command = "scp -r %s %s [EMAIL PROTECTED]:%s" % (self.options +
compress_option, src, self.username, self.hostname, dest)
- return utils.launch_process(command, logger=logger)
+ return utils.launch_process(command, my_logger=my_logger)
- def copy_from(self, src, dest, compress=True, logger=""):
+ def copy_from(self, src, dest, compress=True, my_logger=""):
"""Args: (src (list), dest) Returns: (exit_code, output).
Optional args: compress=0 or 1, mode=tar or scp.
# I guess I could also use zip/unzip... oh well (??)"""
+ if not my_logger:
+ my_logger = self.logger
+
if compress:
# CompressionLevel only works for protocol 1... (bummer)
compress_option = ' -o "Compression yes" -o
"CompressionLevel 9" '
@@ -138,23 +153,26 @@
files += " [EMAIL PROTECTED]:%s " % (self.username,
self.hostname, file)
command = "scp -r %s %s %s" % (self.options + compress_option,
files, dest)
- return utils.launch_process(command, logger=logger)
+ return utils.launch_process(command, my_logger=my_logger)
# TODO: implement code for this
# Needs to be tested
class smbclient:
- def __init__(self, username, hostname, env="", logger=""):
+ def __init__(self, username, hostname, env="", my_logger=""):
self.username = username
self.hostname = hostname
- self.logger = logger
+ self.logger = my_logger
# Copy to jail
- def copy_to(self, src, dest, compress=True, logger=""):
+ def copy_to(self, src, dest, compress=True, my_logger=""):
"""Args: src (list), dest, Returns: (exit_code, output)."""
+ if not my_logger:
+ my_logger = self.logger
+
# this is the original 'scp' mode
# fixed: You can't mput files outside the current local dir
(going to have to chdir to each dir and issue separate smbclient commands)
# Kinda ugly, but it works!
@@ -175,11 +193,13 @@
#command = "%s -spc %s | smbclient //%s/%s -A %s -U %s -D %s
-Trqx -" % (self.local_tar_path, src, self.host, self.SMB_SHARE,
config.smb_passfile, self.user, dest)
- return utils.launch_process(command,
print_output=self.print_output)
+ return utils.launch_process(command, my_logger=my_logger)
- def copy_from(self, src, dest, compress=True, logger=""):
+ def copy_from(self, src, dest, compress=True, my_logger=""):
"""Args: (src (list), dest) Returns: (exit_code, output)."""
+ if not my_logger:
+ my_logger = self.logger
# fixed: You can't mput files outside the current local dir
(going to have to chdir to each dir and issue separate smbclient commands)
# Kinda ugly, but it works!
@@ -193,6 +213,6 @@
command += "cd %s; smbclient //%s/%s -A %s -U %s -D %s
-c 'prompt; recurse; mget %s' ; cd %s ;" % (dest, self.host, self.SMB_SHARE,
config.smb_passfile, self.user, dir, filename, current_dir)
- return utils.launch_process(command,
print_output=self.print_output)
+ return utils.launch_process(command, my_logger=my_logger)
Modified: trunk/release/pyutils/packaging.py
===================================================================
--- trunk/release/pyutils/packaging.py 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/packaging.py 2007-07-21 01:15:13 UTC (rev 82404)
@@ -17,7 +17,7 @@
class buildconf:
- def __init__(self, conf_file_name, logger="", skip_alternates=False):
+ def __init__(self, conf_file_name, my_logger="", skip_alternates=False):
"""skip_alternates is for times like jail-do, where I don't
care if the jail is locked or not"""
# Parse out name and counter (alternate)
@@ -42,7 +42,7 @@
# Construct arguments
buildenv_args = {}
- buildenv_args['logger'] = logger
+ buildenv_args['my_logger'] = my_logger
# Find a buildenv that isn't locked
while True:
@@ -78,8 +78,7 @@
if not check_alternates:
break
- # TODO: also check for being offline
- if not self.buildenv.is_locked() or skip_alternates:
+ if skip_alternates or self.buildenv.offline() or not
self.buildenv.is_locked():
# we're done, use that buildenv
break
Modified: trunk/release/pyutils/utils.py
===================================================================
--- trunk/release/pyutils/utils.py 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/utils.py 2007-07-21 01:15:13 UTC (rev 82404)
@@ -49,6 +49,7 @@
debug=0
KILLED_EXIT_CODE=-42
+INTERRUPTED_EXIT_CODE=10 # Should be arbitrary... ?
match_all = re.compile('')
@@ -337,7 +338,7 @@
fd.close()
-def launch_process(command, capture_stderr=1, print_output=1, print_command=0,
terminate_reg="", logger="", output_timeout=0, kill_process_group=0,
max_output_size=0):
+def launch_process(command, capture_stderr=1, print_output=1, print_command=0,
terminate_reg="", my_logger="", output_timeout=0, kill_process_group=0,
max_output_size=0, interruptable=False):
"""Execute a command, return output (stdout and optionally stderr), and
optionally print output the process is being run.
Returns a tuple: exit code, output
@@ -367,10 +368,16 @@
print_command=1
# turn off output if a logger is used
- if logger: print_output=0
+ if my_logger: print_output=0
if print_command: print command
+ if interruptable:
+ interrupted_file = os.path.dirname(interruptable) + os.sep +
"interrupted"
+ if os.path.exists(interrupted_file):
+ print "** Failing to execute command '%s' because
'interrupted' file is present: %s" % (command, interrupted_file )
+ sys.exit(INTERRUPTED_EXIT_CODE)
+
if capture_stderr:
process = popen2.Popen4(command)
else:
@@ -379,6 +386,12 @@
# Close unnecessary handles
process.tochild.close()
+
+ if interruptable:
+ fd = open(interruptable, 'w')
+ fd.write(str(os.getpgrp()))
+ fd.close()
+
if output_timeout:
flags = fcntl.fcntl(process.fromchild.fileno(), fcntl.F_GETFL)
flags = flags | os.O_NONBLOCK
@@ -438,6 +451,10 @@
# Kill the process
try:
+ # clean up interruptable file if it's
there
+ if interruptable and
os.path.exists(interruptable):
+ os.unlink(interruptable)
+
# Kill the process group of the current
process...
if kill_process_group:
print "** Killing process group
(committing suicide) ... **"
@@ -470,8 +487,8 @@
sys.stdout.flush()
# This doesn't work on macosx... ?
#process.fromchild.flush()
- if logger:
- logger.log(line)
+ if my_logger:
+ my_logger.log(line)
collected += [ line ]
@@ -485,6 +502,12 @@
if killed:
exit_code = KILLED_EXIT_CODE
+ # Clean up interruptable file? naw... shouldn't need to
+ # actually we do, one to remove file if process isn't running,
+ # also if we're root (install rpms), we need to remove this while we
have rights
+ if interruptable and os.path.exists(interruptable):
+ os.unlink(interruptable)
+
# Concat array elements into a string
# strip the whitespace off the end (makes it behave more like
commands.getxxx)
return exit_code, "".join(collected).rstrip()
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches