Author: wade
Date: 2007-07-20 21:15:13 -0400 (Fri, 20 Jul 2007)
New Revision: 82404

Modified:
   trunk/release/packaging/
   trunk/release/packaging/build
   trunk/release/packaging/do-install-zip-pkgs
   trunk/release/packaging/do-msvn-tar
   trunk/release/packaging/install-deps
   trunk/release/packaging/jail-do
   trunk/release/packaging/jail-do-all
   trunk/release/packaging/mktarball
   trunk/release/pyutils/buildenv.py
   trunk/release/pyutils/config.py
   trunk/release/pyutils/launch_process.py
   trunk/release/pyutils/network.py
   trunk/release/pyutils/packaging.py
   trunk/release/pyutils/utils.py
Log:
new monobuild feature: ability for anyone (well, with vpn access) to
interrupt builds, and ability for anyone to lock a client machine so
monobuild won't farm a build to that client, will post more instructions
on the list

logger changes:
-had to use my_logger.  Any place the var 'logger' was being used,
 it was overwriting the class name (oops)

jail-do:
-can log in using old syntax.  To execute a command, you must use "run
<command>"
-several new actions: interrupting and locking build clients

Lots of locking changes to do best effort locking from multiple hosts

Remove some shell-isms and use python code instead

With interruptability, had option that some commands can't be
interrupted (ie: killing an rpm command could trash the rpm db,
  also during svn checkouts, don't kill svn processes)
 Also gives ability to use remove execution code internally
  without locking ourselves out

config.py:
-clean up client and package lists




Property changes on: trunk/release/packaging
___________________________________________________________________
Name: svn:ignore
   - packages
snapshot_packages
snapshot_zip_packages
zip_packages
external_zip_pkg
external_packages
logs
sources
snapshot_sources
tarball_map

   + packages
snapshot_packages
snapshot_zip_packages
zip_packages
external_zip_pkg
external_packages
logs
sources
snapshot_sources
tarball_map
distro_source_key
.smbpass


Modified: trunk/release/packaging/build
===================================================================
--- trunk/release/packaging/build       2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/build       2007-07-21 01:15:13 UTC (rev 82404)
@@ -46,6 +46,9 @@
        #  Some tests are reporting 255 on killed processes?
        if code == utils.KILLED_EXIT_CODE or code == 214 or code == 137 or code 
== 255:
                return "timeout"
+       # Most of the time interrupts will show as 'timeout'
+       elif code == utils.INTERRUPTED_EXIT_CODE:
+               return "interrupted"
        elif code:
                if test:
                        return "testfailure"
@@ -68,7 +71,7 @@
                log_obj = logger.Logger(filename=LOGFILE, 
print_screen=print_screen)
                build_info.update_step(name, state="inprogress", 
log=os.path.basename(LOGFILE), start=utils.get_time(), finish="")
 
-               (result, output) = conf.buildenv.execute_code("%s %s " % 
(step_env_pre_cmd, code), working_dir=remote_src_dir, env=conf.env_vars, 
logger=log_obj, output_timeout=timeout, max_output_size=max_output_size)
+               (result, output) = conf.buildenv.execute_code("%s %s " % 
(step_env_pre_cmd, code), working_dir=remote_src_dir, env=conf.env_vars, 
my_logger=log_obj, output_timeout=timeout, max_output_size=max_output_size)
 
                # Get results, if any
                results_dir = os.path.join(config.build_info_dir, 
build_info.rel_files_dir, 'files', 'steps')
@@ -238,7 +241,7 @@
 else:
        install_deps_H_or_R = HEAD_or_RELEASE
 print "Installing dependencies..."
-(code, output) = utils.launch_process("./install-deps --HEAD_or_RELEASE=%s %s 
%s" % (install_deps_H_or_R, conf.conf_lock_filename, package_name), 
logger=log_obj)
+(code, output) = utils.launch_process("./install-deps --HEAD_or_RELEASE=%s %s 
%s" % (install_deps_H_or_R, conf.conf_lock_filename, package_name), 
my_logger=log_obj)
 if code:
        conf.buildenv.unlock_env()
        build_info.update_step("install-deps", state="failure", 
finish=utils.get_time())
@@ -292,8 +295,10 @@
        step_env_pre_cmd = ""
 
 # Clean up
-conf.buildenv.execute_command('rm -Rf %s/steps' % 
(conf.env_vars['build_location']) )
-conf.buildenv.execute_command('rm -Rf %s; mkdir -p -m777 %s' % 
(temp_build_dir, temp_build_dir) )
+conf.buildenv.remove_path('%s/steps' % conf.env_vars['build_location'] )
+conf.buildenv.remove_path(temp_build_dir)
+conf.buildenv.make_path(temp_build_dir)
+
 conf.buildenv.copy_to(files_to_copy, temp_build_dir, compress=0)
 conf.buildenv.copy_to(support_files_to_copy, conf.env_vars['build_location'], 
compress=0)
 
@@ -314,7 +319,7 @@
        build_info.update_build(state="failure", finish=utils.get_time())
        sys.exit(7)
 
-(code, output) = conf.buildenv.execute_command(build_command, 
env=conf.env_vars, logger=log_obj, output_timeout=timeout, 
max_output_size=max_output_size)
+(code, output) = conf.buildenv.execute_command(build_command, 
env=conf.env_vars, my_logger=log_obj, output_timeout=timeout, 
max_output_size=max_output_size)
 if code:
        build_failed(conf, code, LOGFILE, build_info, package)
 
@@ -330,12 +335,12 @@
        code = 0
        output = ""
        for file in files:
-               (tmp_code, tmp_output) = utils.launch_process("unzip -qt " + 
file, logger=log_obj)
+               (tmp_code, tmp_output) = utils.launch_process("unzip -qt " + 
file, my_logger=log_obj)
                if tmp_code: code = 1
                output += tmp_output
 else:
        files = glob.glob(package_version_path + os.sep + "*.rpm")
-       (code, output) = utils.launch_process("rpm -K " + " ".join(files), 
logger=log_obj)
+       (code, output) = utils.launch_process("rpm -K " + " ".join(files), 
my_logger=log_obj)
 
 if code:
        log_obj.log("Verifying packages failed... (usually because of lack of 
disk space or incomplete copy)\n")

Modified: trunk/release/packaging/do-install-zip-pkgs
===================================================================
--- trunk/release/packaging/do-install-zip-pkgs 2007-07-21 00:26:17 UTC (rev 
82403)
+++ trunk/release/packaging/do-install-zip-pkgs 2007-07-21 01:15:13 UTC (rev 
82404)
@@ -23,7 +23,7 @@
 dest = sys.argv[1]
 reloc = sys.argv[2]
 
-package_files = sys.argv[3:]
+package_files = glob.glob(sys.argv[3] + os.sep + "*")
 
 print " *** Installing to %s for destination %s *** " % (dest, reloc)
 

Modified: trunk/release/packaging/do-msvn-tar
===================================================================
--- trunk/release/packaging/do-msvn-tar 2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/do-msvn-tar 2007-07-21 01:15:13 UTC (rev 82404)
@@ -9,15 +9,23 @@
 import shutil
 import glob
 import distutils.dir_util
+import getpass
 
 sys.path.append('.')
 
 import packaging
 
+# Some things we'll need to run as root
+EXEC_AS_ROOT = 'sudo'
+
 (HEAD_or_RELEASE, package, version) = sys.argv[1:]
 
 os.chdir('source_dist_creator')
 
+# they key needs to be owned by the user so ssh doesn't reject the key
+os.system('%s chown %s distro_source_key' % (EXEC_AS_ROOT, getpass.getuser()))
+os.system("%s mount -t proc none /proc &> /dev/null" % EXEC_AS_ROOT)
+
 temp_dir = os.getcwd()
 
 dummy_env = ""

Modified: trunk/release/packaging/install-deps
===================================================================
--- trunk/release/packaging/install-deps        2007-07-21 00:26:17 UTC (rev 
82403)
+++ trunk/release/packaging/install-deps        2007-07-21 01:15:13 UTC (rev 
82404)
@@ -60,7 +60,7 @@
 
 log_obj = logger.Logger(filename=config.devnull)
 # skip_alternates, because locking is handled in build and mktarball (if we 
had better locking, this wouldn't be needed)
-target_conf = packaging.buildconf(target, logger=log_obj, skip_alternates=True)
+target_conf = packaging.buildconf(target, my_logger=log_obj, 
skip_alternates=True)
 
 if build_source:
        build_deps = False
@@ -104,12 +104,14 @@
 build_location = target_conf.env_vars['build_location']
 
 # Clean out deps area (/tmp/build_deps does not use build_location for 
relocation purposes)
-(code, output) = target_conf.buildenv.execute_command("rm -rf 
%s/install-packages /tmp/build_deps" % (build_location) )
-if code:
+(code, output) = target_conf.buildenv.remove_path("/tmp/build_deps")
+(code2, output2) = target_conf.buildenv.remove_path(build_location + os.sep + 
'install-packages')
+if code or code2:
        print "Error cleaning up... exiting"
        sys.exit(1)
-target_conf.buildenv.execute_command("mkdir -p -m777 %s/install-packages" % 
build_location)
 
+target_conf.buildenv.make_path("%s/install-packages" % build_location)
+
 # copy packages
 # It would be nice to see the output...
 target_conf.buildenv.copy_to(files, '%s/install-packages' % build_location, 
compress=0)
@@ -119,7 +121,7 @@
 
        target_conf.buildenv.copy_to(['do-install-zip-pkgs', 
'../pyutils/utils.py', '../pyutils/rpm2cpio.py'], build_location)
        # These must NOT be in the build_location, otherwise it will throw off 
the relocation
-        (code, output) = target_conf.buildenv.execute_command( 
"%s/do-install-zip-pkgs /tmp/build_deps /tmp/build_deps %s/install-packages/*" 
% (build_location, build_location) )
+        (code, output) = target_conf.buildenv.execute_command( 
"%s/do-install-zip-pkgs /tmp/build_deps /tmp/build_deps %s/install-packages" % 
(build_location, build_location) )
 
 else:
 
@@ -133,7 +135,8 @@
                target_conf.buildenv.execute_command('%s/remove-rpms %s' % 
(build_location, " ".join(packs_to_remove)), exec_as_root=1)
        code = 0
        if files:
-               (code, output) = target_conf.buildenv.execute_command('rpm -Uvh 
%s/install-packages/*.rpm' % build_location, exec_as_root=1)
+               # Make this non-interruptable because we don't want the rpm db 
trashed
+               (code, output) = target_conf.buildenv.execute_command('rpm -Uvh 
%s/install-packages/*.rpm' % build_location, exec_as_root=1, 
interruptable=False)
 
 # Exit with return code...
 sys.exit(code)

Modified: trunk/release/packaging/jail-do
===================================================================
--- trunk/release/packaging/jail-do     2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/jail-do     2007-07-21 01:15:13 UTC (rev 82404)
@@ -1,28 +1,73 @@
 #!/usr/bin/env python
 
 import sys
-import string
 
 sys.path += ['../pyutils']
 
 import buildenv
 import packaging
 
+def usage():
+       print "Usage: ./jail-do <distro conf> <action> [ <action args> ]"
+       print "  where <action> is one of: login | run | lock | unlock | 
is_locked | interrupt"
+       print "  Example: ./jail-do exec suse-102-i586 run ls"
+       print "  'run' is the only action that takes <action args>,"
+       print "         which will run that command"
+       print "  'unlock' also removes interrupt file flag"
+
 if len(sys.argv) < 2:
-       print "Usage: ./jail-do <distro conf> [command]"
-       print "  Example: ./jail-do suse-93-i586 ls"
-       print "  If no [command] is given, an interactive login will proceed"
+       usage()
        sys.exit(1)
 
-conf = packaging.buildconf(sys.argv[1], skip_alternates=True)
+env_name = sys.argv[1]
+action_options = []
+
+# Default to login action if none is specified
+if len(sys.argv) == 2:
+       action = "login"
+else:
+       action = sys.argv[2]
+       if len(sys.argv) > 3:
+               action_options = sys.argv[3:]
+
+
+conf = packaging.buildconf(env_name, skip_alternates=True)
 conf.buildenv.print_output=True
 
-if len(sys.argv) > 2:
-       command = string.join(sys.argv[2:], ' ')
+if   action == "login":
+       print "Logging in"
+       conf.buildenv.login()
+elif action == "run":
+       command = ' '.join(action_options)
        sys.stderr.write("Executing: %s\n" % command)
        status, output = conf.buildenv.execute_command(command)
+       if status == 255:
+               print "Process was killed!"
        sys.exit(status)
+elif action == "lock":
+       if conf.buildenv.lock_env():
+               print "Locked successfully"
+       else:
+               print "Failed to lock"
+               sys.exit(1)
+elif action == "unlock":
+       if conf.buildenv.unlock_env(clear_interrupt=True):
+               print "Unlocked successfully, interrupt lock cleared if existed"
+       else:
+               print "Failed to unlock"
+               sys.exit(1)
+elif action == "is_locked":
+       if conf.buildenv.is_locked():
+               # return 0 (true) if it is locked
+               print "Locked"
+       else:
+               # return 1 (false) if it's not
+               print "Unlocked"
+               sys.exit(1)
+elif action == "interrupt":
+       (code, output) = conf.buildenv.interrupt()
+       sys.exit(code)
 else:
-       print "Logging in"
-       conf.buildenv.login()
+       print "Unknown action: " + action
+       sys.exit(1)
 

Modified: trunk/release/packaging/jail-do-all
===================================================================

Modified: trunk/release/packaging/mktarball
===================================================================
--- trunk/release/packaging/mktarball   2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/packaging/mktarball   2007-07-21 01:15:13 UTC (rev 82404)
@@ -72,7 +72,7 @@
 log_obj.log("Sources path: %s\n" % package_obj.source_fullpath)
 log_obj.log("Using distro: %s\n" % distro)
 
-conf = packaging.buildconf(distro, logger=log_obj)
+conf = packaging.buildconf(distro, my_logger=log_obj)
 
 if conf.buildenv.is_locked():
        log_obj.log("%s jail is busy\n" % distro )
@@ -84,7 +84,7 @@
 repo.add_file(HEAD_or_RELEASE, package, version, "inprogress")
 
 if package_obj.info.has_key('MONO_DEPS'):
-       (status, output) = utils.launch_process('./install-deps --build_source 
%s %s' % (conf.conf_lock_filename, package), logger=log_obj)
+       (status, output) = utils.launch_process('./install-deps --build_source 
%s %s' % (conf.conf_lock_filename, package), my_logger=log_obj)
        if status:
                conf.buildenv.unlock_env()
                log_obj.log("Dependency installation failed\n")
@@ -94,7 +94,7 @@
 temp_dir = conf.env_vars['build_location'] + os.sep + 'source_dist_creator'
 
 # Clean up
-conf.buildenv.execute_command("rm -Rf %s" % temp_dir)
+conf.buildenv.remove_path(temp_dir)
 
 # Copy some files over and execute!
 # Also copy over def_alias files
@@ -111,10 +111,8 @@
 conf.buildenv.copy_to(files_to_copy, temp_dir, mode='scp')
 
 log_obj.log("Starting to build source dist...\n")
-# they key needs to be owned by the user so ssh doesn't reject the key
-(status, output) = conf.buildenv.execute_command('chown %s 
%s/distro_source_key' % (conf.info['username'], temp_dir), exec_as_root=1)
-(status, output) = conf.buildenv.execute_command('mount /proc &> /dev/null', 
exec_as_root=1)
-(status, output) = conf.buildenv.execute_command('%s/do-msvn-tar %s %s %s' % 
(temp_dir, HEAD_or_RELEASE, package, version) )
+# Make this non-interruptable because we don't want the source cache corrupted
+(status, output) = conf.buildenv.execute_command('%s/do-msvn-tar %s %s %s' % 
(temp_dir, HEAD_or_RELEASE, package, version), interruptable=False )
 
 if status:
         conf.buildenv.unlock_env()

Modified: trunk/release/pyutils/buildenv.py
===================================================================
--- trunk/release/pyutils/buildenv.py   2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/buildenv.py   2007-07-21 01:15:13 UTC (rev 82404)
@@ -7,9 +7,6 @@
 """
 
 import os
-import config
-import network
-import utils
 import socket
 import distutils.dir_util
 import shutil
@@ -17,13 +14,16 @@
 import tempfile
 import getpass
 
+import config
+import network
+import utils
 import logger
 
 import pdb
 
 class buildenv:
 
-        def __init__(self, username="", hostname="", root_dir="", 
lock_filename="", target_command_prefix="", env=config.env_vars, logger="", 
login_mode='ssh', copy_mode='scp', exec_mode='ssh'):
+        def __init__(self, username="", hostname="", root_dir="", 
lock_filename="", target_command_prefix="", env=config.env_vars, my_logger="", 
login_mode='ssh', copy_mode='scp', exec_mode='ssh'):
 
                self.login_mode=login_mode
                self.copy_mode=copy_mode
@@ -57,11 +57,15 @@
                # Custom identifier to use at various places
                self.uid = "[EMAIL PROTECTED]:%s" % (self.username, 
self.hostname, root_dir)
 
+               # Host lock filename
                if lock_filename:
-                       self.lock_filename = lock_filename
+                       self.host_lock_filename = lock_filename
                else:
-                       self.lock_filename = self.uid
+                       self.host_lock_filename = self.uid
 
+               # Client lock filename
+               self.client_lock_filename = self.build_location + os.sep + 
"locked"
+
                 # Optional...
                 self.root_dir = root_dir
                 self.env = env
@@ -73,12 +77,12 @@
 
                # initialize objects for later
                self.modes = {}
-               self.modes['ssh'] = network.ssh(username=username, 
hostname=hostname, env=self.env, logger=logger)
-               self.modes['scp'] = network.scp(username=username, 
hostname=hostname, env=self.env, logger=logger)
-               self.modes['smbclient'] = network.smbclient(username=username, 
hostname=hostname, env=self.env, logger=logger)
-               self.modes['local'] = local(username=username, 
hostname=hostname, env=self.env, logger=logger)
+               self.modes['ssh'] = network.ssh(username=username, 
hostname=hostname, env=self.env, my_logger=my_logger)
+               self.modes['scp'] = network.scp(username=username, 
hostname=hostname, env=self.env, my_logger=my_logger)
+               self.modes['smbclient'] = network.smbclient(username=username, 
hostname=hostname, env=self.env, my_logger=my_logger)
+               self.modes['local'] = local(username=username, 
hostname=hostname, env=self.env, my_logger=my_logger)
 
-               self.logger = logger
+               self.logger = my_logger
 
                # chroot options
                if root_dir:
@@ -88,7 +92,13 @@
                # Set flag so we don't copy over and over for execute_command
                self.exec_util_files_copied = False
 
+               # empty logger to use at various places
+               self.empty_logger = logger.Logger(config.devnull, 
print_screen=0)
 
+               # Filenames for process jobs
+               self.interrupted_file = self.build_location + os.sep + 
'interrupted'
+               self.pgkill_file = self.build_location + os.sep + 'pgid_kill'
+
        def login(self, mode=""):
 
                if not mode:
@@ -99,13 +109,14 @@
 
                mode = self.modes[mode].login(command)
 
-       def execute_command(self, command, working_dir="", mode="", 
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="", 
env={}, logger=""):
+       def execute_command(self, command, working_dir="", mode="", 
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="", 
env={}, my_logger="", interruptable=True):
+               """setting interruptable to False is for internal and really 
important commands (ie: killing rpm installs/removes could cause havoc)  Using 
that flag doesn't write a pgid file"""
 
                if not working_dir:
                        working_dir = self.build_location
 
-               if not logger:
-                       logger = self.logger
+               if not my_logger:
+                       my_logger = self.logger
 
                if not mode:
                        mode = self.exec_mode
@@ -117,6 +128,10 @@
                if not env:
                        env = self.env
 
+               # If true, but no filename passed in, use this default filename
+               if interruptable == True:
+                       interruptable = self.pgkill_file
+
                # launch process options. pass everything through launch 
process, and have it save the pgid to enable process interruptions
                # TODO: this doesn't play nicely with local commands, because 
this code is in the process group.  
                #  We need the local execute to be in a different process group 
for this to work
@@ -129,6 +144,8 @@
                        launch_options += " --terminate_reg=%s" % terminate_reg
                if working_dir:
                        launch_options += " --working_dir=%s" % working_dir
+               if interruptable:
+                       launch_options += " --interruptable=%s" % interruptable
 
                # Build environment string to pass to launch_process
                if env:
@@ -150,11 +167,14 @@
                # Double quotes are very meticulous
                command = '%s %s %s "%s %s %s"' % (self.root_dir_options, 
self.env['shell_path'], "-c",  self.target_command_prefix, sudo_opts, command)
 
-               return self.modes[mode].execute_command(command, logger=logger)
+               return self.modes[mode].execute_command(command, 
my_logger=my_logger)
 
-       def execute_code(self, code, working_dir="", interpreter="", 
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="", 
env={}, logger=""):
+       def execute_code(self, code, working_dir="", interpreter="", 
exec_as_root=False, output_timeout=0, max_output_size=0, terminate_reg="", 
env={}, my_logger="", interruptable=True):
                """ interpreter can be anything that works... mcs *.cs; mono, 
perl, python, etc... """
 
+                if not my_logger:
+                       my_logger = self.logger
+
                # Default to shell code, but can be overwritten
                if not interpreter:
                        interpreter = self.env['shell_path']
@@ -185,8 +205,9 @@
                args['max_output_size'] = max_output_size
                args['terminate_reg'] = terminate_reg
                args['env'] = env
-               args['logger'] = logger
+               args['my_logger'] = my_logger
                args['working_dir'] = working_dir
+               args['interruptable'] = interruptable
 
                command = "%s %s" % (interpreter, self.build_location + os.sep 
+ file_basename)
                code, output = self.execute_command(command, **args)
@@ -197,12 +218,32 @@
                return code, output
 
        def make_path(self, dir, mode=""):
-
+               """Create a path
+                       This must be implemented in the lower layers since 
execute_* depend on it"""
                if not mode:
                        mode = self.exec_mode
 
                return self.modes[mode].make_path(self.root_dir + dir)
 
+        def remove_path(self, dir):
+               """Remove a path.  Since no internal functions depend on this, 
it can be a higher layer function"""
+
+               remove_path_code = """
+import os
+import shutil
+
+path = '%s'
+
+if os.path.exists(path):
+       # Newer versions of python's rmtree will remove files, but not all 
versions
+       if os.path.isdir(path):
+               shutil.rmtree(path)
+       else:
+               os.unlink(path)
+""" % dir
+
+               return self.execute_code(remove_path_code, 
interpreter=self.env['python_path'], interruptable=False)
+
         def copy_to(self, src, dest, compress=True, mode=""):
 
                 if not mode:
@@ -217,7 +258,7 @@
                        print "No files to copy..."
                        return 0, ""
 
-               # Make sure it exists
+               ## Make sure it exists (turns into circular recursion at this 
point)
                self.make_path(dest)
 
                return self.modes[mode].copy_to(src, self.root_dir + dest, 
compress=compress)
@@ -245,37 +286,129 @@
 
                 return self.modes[mode].copy_from(src, dest, compress=compress)
 
-       def interrupt_command(self):
-               pass
+       def interrupt(self):
+               """Write interrupted file flag, read pgid_kill file, and blow 
process group away!"""
+
+               flag_code = """
+import os
+
+path = '%s'
+
+fd = open(path, 'w')
+fd.write("")
+fd.close()""" % self.interrupted_file
+
+               (code, output) = self.execute_code(flag_code, 
interpreter=self.env['python_path'], interruptable=False)
+
+               if code:
+                       return (code, output)
+
+               kill_code = """
+import os
+import signal
+import sys
+
+path = '%s'
+client_lock = '%s'
+
+if not os.path.exists(path):
+       if os.path.exists(client_lock):
+               print "Client is busy and uninterruptable, try again in a few 
minutes"
+               sys.exit(1)
+       print "Client is not running an interruptable command, exiting..."
+       print "  (it could be running an non-interruptable command, but the 
client isn't locked)"
+       sys.exit(0)
+
+
+fd = open(path)
+pgid = int(fd.read())
+fd.close()
+os.unlink(path)
+
+# Kill the process group
+print "Killing process group: %%d" %% pgid
+try:
+       os.kill(-pgid, signal.SIGKILL)
+except OSError:
+       print "Error killing process group %%d (already gone?)" %% pgid
+       sys.exit(1)
+""" % (self.pgkill_file, self.client_lock_filename)
+
+               return self.execute_code(kill_code, 
interpreter=self.env['python_path'], interruptable=False)
        
 
        # TODO: use os level locking here
        def lock_env(self):
-               fd = open(self.lock_filename, 'w')
+               fd = open(self.host_lock_filename, 'w')
                fd.write("")
                fd.close()
+
+               # also put file in jail to signify that it's locked (method not 
perfect, but should be good enough)
+               #self.execute_command("touch %s" % self.client_lock_filename)
+
+               lock_code = """
+path = '%s'
+
+fd = open(path, 'w')
+fd.write("")
+fd.close()
+
+""" % self.client_lock_filename
+
+               self.execute_code(lock_code, 
interpreter=self.env['python_path'], interruptable=False)
+
                return True
 
-       def unlock_env(self):
-               if os.path.exists(self.lock_filename):
-                       os.unlink(self.lock_filename)
+       def unlock_env(self, clear_interrupt=False):
+               if os.path.exists(self.host_lock_filename):
+                       os.unlink(self.host_lock_filename)
+                       host_ret = True
+               else:
+                       print "unlock_env: host lockfile already removed"
+                       host_ret = False
+
+               # also remove interrupt file if it is there
+               if clear_interrupt:
+                       (ret, output) = self.remove_path(self.interrupted_file)
+               (client_ret, output) = 
self.remove_path(self.client_lock_filename)
+
+               # Return False on failure
+               if host_ret or not client_ret:
                        return True
                else:
-                       print "Build environment already unlocked"
                        return False
 
        def is_locked(self):
-               return os.path.exists(self.lock_filename)
 
+               # Check host lockfile
+               if os.path.exists(self.host_lock_filename):
+                       host_ret = True
+               else:
+                       host_ret = False
+
+               path_exists_code = """
+import os
+import sys
+
+path = '%s'
+
+if not os.path.exists(path):
+       sys.exit(1)
+""" % self.client_lock_filename
+
+               (client_ret, output) = self.execute_code(path_exists_code, 
interpreter=self.env['python_path'], interruptable=False)
+
+               # if host lock or client lock exists, we're locked
+               if host_ret or not client_ret:
+                       return True
+               else:
+                       return False
+
        def offline(self):
-               old_logger = self.logger
-               self.logger = logger.Logger(config.devnull, print_screen=0)
 
                # Run some arbitrary command that will always return true
-               (code, output) = self.execute_command("ls")
+               (code, output) = self.execute_command("ls", 
my_logger=self.empty_logger, interruptable=False)
 
-               self.logger = old_logger
-
                if code: return 1
                else:    return 0
 
@@ -283,20 +416,24 @@
 class local:
        """Class implementing the same interface as a network object, but the 
jail or machine is on the host machine"""
 
-       def __init__(self, username, hostname, env="", logger=""):
+       def __init__(self, username, hostname, env="", my_logger=""):
                 self.username = username
                 self.hostname = hostname
-                self.logger = logger
+                self.logger = my_logger
 
        def login(self, command):
 
                os.system(command)
 
-       def execute_command(self, command, logger=""):
+       def execute_command(self, command, my_logger=""):
+
+               if not my_logger:
+                       my_logger = self.logger
+
                # TODO: execute this under a new pgid so only it gets killed, 
not us all
                # check 
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012
                # Or, put that code in the utils.launch_process routine, which 
would make it more robust
-               return utils.launch_process(command, logger=logger)
+               return utils.launch_process(command, my_logger=my_logger)
 
        # For local copyings, you can use the same commands (scp, tar), just 
not over ssh (scp?)
        # TODO: But how to know what to use... scp or tar, because they behave 
differently
@@ -305,7 +442,7 @@
        # difference: # Note: tar mode appends src path of file to dest (just 
the way tar works)"""
        # tar mode handles symbolic links and preserving time stamps, unlike 
scp.
 
-        def copy_to(self, src, dest, compress=True, logger=""):
+        def copy_to(self, src, dest, compress=True, my_logger=""):
                results = []
                # Glob each of the src filesnames to allow wildcards
                new_src = []
@@ -329,11 +466,11 @@
 
                return 0, "\n".join(results)
 
-        def copy_from(self, src, dest, compress=True, logger=""):
+        def copy_from(self, src, dest, compress=True, my_logger=""):
                """ No need to duplicate this, since we're on a local machine, 
use the same method"""
-               return self.copy_to(src, dest, compress=compress, logger=logger)
+               return self.copy_to(src, dest, compress=compress, 
my_logger=my_logger)
 
-        def make_path(self, dir):
+       def make_path(self, dir):
                """Create a path"""
 
                error = 0
@@ -342,5 +479,3 @@
                except:
                        error = 1
                return error
-
-

Modified: trunk/release/pyutils/config.py
===================================================================
--- trunk/release/pyutils/config.py     2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/config.py     2007-07-21 01:15:13 UTC (rev 82404)
@@ -63,7 +63,20 @@
 td_num_sequential = 10
 
 # static list of packages to create tarballs for
-td_packages = ['mono', 'libgdiplus', 'mono-basic', 'mono-tools', 
'monodevelop', 'monodoc', 'xsp', 'gecko-sharp-2.0', 'mono-debugger', 
'gtk-sharp', 'mod_mono', 'olive' ]
+td_packages = """
+       gecko-sharp2
+       gtk-sharp
+       libgdiplus
+       mod_mono
+       mono
+       mono-basic
+       mono-debugger
+       mono-tools
+       monodevelop
+       monodoc
+       olive
+       xsp
+""".split()
 
 # builds each and every checkin if true, otherwise, only build the latest 
checkin
 td_sequential = False
@@ -85,20 +98,20 @@
 
 # List of platforms/packages
 sd_latest_build_distros = """
-       sles-9-i586
-       sles-9-x86_64
-       win-4-i386
+       debian-4-arm
+       debian-4-sparc
        macos-10-ppc
        macos-10-x86
-       sunos-8-sparc
-       sunos-10-x86
+       sles-9-i586
        sles-9-ia64
+       sles-9-ppc
        sles-9-s390
        sles-9-s390x
-       sles-9-ppc
-       debian-4-arm
-       debian-4-sparc
+       sles-9-x86_64
+       sunos-8-sparc
+       sunos-10-x86
        suse-100-i586
+       win-4-i386
 """.split()
 sd_latest_build_packages = td_packages
 ##############################################
@@ -135,6 +148,7 @@
        #'shell_path':          '/bin/sh',
        # We depend on bash (ie: env string in execute_command)
        'shell_path':           '/bin/bash',
+       'python_path':          'python', # full path can be set in conf file 
for individual jails
 }
 
 ######################################

Modified: trunk/release/pyutils/launch_process.py
===================================================================
--- trunk/release/pyutils/launch_process.py     2007-07-21 00:26:17 UTC (rev 
82403)
+++ trunk/release/pyutils/launch_process.py     2007-07-21 01:15:13 UTC (rev 
82404)
@@ -7,7 +7,7 @@
 import utils
 
 def usage():
-       print "Usage: ./launch_process.py [--terminate_reg=<reg string>] 
[--output_timeout=<secs>] [--max_output_size=<num chars>] 
[--kill_process_group] <command>"
+       print "Usage: ./launch_process.py [--terminate_reg=<reg string>] 
[--output_timeout=<secs>] [--max_output_size=<num chars>] 
[--kill_process_group] [--interruptable] <command>"
 
 def commandline():
 
@@ -17,7 +17,7 @@
 
        # Options for launch process
        try:
-               opts, command = getopt.getopt(sys.argv[1:], "", [ 
"terminate_reg=", "output_timeout=", "max_output_size=", "kill_process_group", 
"env=", "working_dir=" ])
+               opts, command = getopt.getopt(sys.argv[1:], "", [ 
"terminate_reg=", "output_timeout=", "max_output_size=", "kill_process_group", 
"env=", "working_dir=", "interruptable=" ])
        except getopt.GetoptError:
                 usage()
                sys.exit(1)
@@ -44,6 +44,8 @@
                                print "Failed to chdir: " + value
                                print "Exiting..."
                                sys.exit(1)
+               if option == "--interruptable":
+                       args['interruptable'] = value
 
        command = " ".join(command)
        code, output = utils.launch_process(command, **args)

Modified: trunk/release/pyutils/network.py
===================================================================
--- trunk/release/pyutils/network.py    2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/network.py    2007-07-21 01:15:13 UTC (rev 82404)
@@ -13,13 +13,13 @@
 
 class ssh:
 
-       def __init__(self, username, hostname, env="", logger=""):
+       def __init__(self, username, hostname, env="", my_logger=""):
                # Ignore host checks, and blowfish might be faster and less cpu 
intensive
                self.options = '-o "BatchMode yes" -o "StrictHostKeyChecking 
no" -o "Cipher blowfish" '
 
                self.username = username
                self.hostname = hostname
-               self.logger = logger
+               self.logger = my_logger
 
                # Get tar paths (required from copy to/from)
                if env and env.has_key('tar_path'):
@@ -34,20 +34,26 @@
                # Can't use utils.launch_process here because we need an 
interactive terminal
                 return os.system(command)
 
-        def execute_command(self, command, logger=""):
+        def execute_command(self, command, my_logger=""):
 
+               if not my_logger:
+                       my_logger = self.logger
+
                # Escape these so they get interpolated by the remote machine
                command = command.replace('$', '\$')
 
                # Put single quotes around the command so the whole command 
gets passed over ssh
                # (must use single quotes here since command may have double 
quotes (shell allows double quotes inside of single, without escaping)
                command = "ssh %s [EMAIL PROTECTED] '%s'" % (self.options, 
self.username, self.hostname, command)
-                return utils.launch_process(command, logger=logger)
+                return utils.launch_process(command, my_logger=my_logger)
 
-       def copy_to(self, src, dest, compress=True, logger=""):
+       def copy_to(self, src, dest, compress=True, my_logger=""):
                """Args: src (list of strings), dest, Returns: (exit_code, 
output).
                Optional args: compress=True of False"""
 
+               if not my_logger:
+                       my_logger = self.logger
+
                src = " ".join(src)
 
                if compress:
@@ -59,9 +65,9 @@
 
                # Note: the -f - option to the remote tar is required for 
solaris tar, otherwise it tries to read from a tape
                command = "%s -%spc %s | ssh %s [EMAIL PROTECTED] 'cd %s ; %s 
-%spvxf - ' " % (self.local_tar_path, compress_option, src, self.options, 
self.username, self.hostname, dest, self.remote_tar_path, compress_option )
-               return utils.launch_process(command, logger=logger)
+               return utils.launch_process(command, my_logger=my_logger)
 
-       def copy_from(self, src, dest, compress=True, logger=""):
+       def copy_from(self, src, dest, compress=True, my_logger=""):
                 """Args: (src (list), dest) Returns: (exit_code, output).
 
                 Optional args: compress=0 or 1
@@ -69,6 +75,9 @@
                 #  I guess I could also use zip/unzip... oh well (What does 
this mean?)
                 # Note: tar mode appends src path of file to dest (just the 
way tar works)"""
 
+               if not my_logger:
+                       my_logger = self.logger
+
                 if compress:
                         # CompressionLevel only works for protocol 1... 
(bummer)
                         compress_option = "z"
@@ -84,30 +93,33 @@
                # Note: the -f - option to the remote tar is required for 
solaris tar, otherwise it tries to read from a tape
                command = "cd %s; ssh %s [EMAIL PROTECTED] ' %s %s -%spcf - %s 
' | %s -%spvxf - " % (dest, self.options, self.username, self.hostname, 
self.remote_tar_path, tar_dir_options, compress_option, files, 
self.local_tar_path, compress_tar_option )
 
-                return utils.launch_process(command, logger=logger)
+                return utils.launch_process(command, my_logger=my_logger)
 
        def make_path(self, dir):
-                """Args: (dir) Returns: (mkdir exit code)."""
+               """Args: (dir) Returns: (mkdir exit code and output)."""
 
                return self.execute_command("mkdir -p -m777 " + dir)
 
 
 class scp:
 
-       def __init__(self, username, hostname, env="", logger=""):
+       def __init__(self, username, hostname, env="", my_logger=""):
                # Ignore host checks, and blowfish might be faster and less cpu 
intensive
                self.options = '-o "StrictHostKeyChecking no" -o "Cipher 
blowfish" '
 
                self.username = username
                self.hostname = hostname
-               self.logger = logger
+               self.logger = my_logger
 
-       def copy_to(self, src, dest, compress=True, logger=""):
+       def copy_to(self, src, dest, compress=True, my_logger=""):
                """Args: src (list of strings), dest, Returns: (exit_code, 
output).
 
                Optional args: compress=0 or 1 (but doesn't actually do 
anything in ssh protocol 2... oh well
                Note, in scp mode, recursive is on by default"""
 
+               if not my_logger:
+                       my_logger = self.logger
+
                src = " ".join(src)
 
                if compress:
@@ -118,14 +130,17 @@
 
                command = "scp -r %s %s [EMAIL PROTECTED]:%s" % (self.options + 
compress_option, src, self.username, self.hostname, dest)
 
-               return utils.launch_process(command, logger=logger)
+               return utils.launch_process(command, my_logger=my_logger)
 
-       def copy_from(self, src, dest, compress=True, logger=""):
+       def copy_from(self, src, dest, compress=True, my_logger=""):
                """Args: (src (list), dest) Returns: (exit_code, output).
 
                Optional args: compress=0 or 1, mode=tar or scp.
                #  I guess I could also use zip/unzip... oh well (??)"""
 
+               if not my_logger:
+                       my_logger = self.logger
+
                if compress:
                        # CompressionLevel only works for protocol 1... (bummer)
                        compress_option = ' -o "Compression yes" -o 
"CompressionLevel 9" '
@@ -138,23 +153,26 @@
                        files += " [EMAIL PROTECTED]:%s " % (self.username, 
self.hostname, file)
                command = "scp -r %s %s %s" % (self.options + compress_option, 
files, dest)
 
-               return utils.launch_process(command, logger=logger)
+               return utils.launch_process(command, my_logger=my_logger)
 
 # TODO: implement code for this
 # Needs to be tested
 class smbclient:
        
-       def __init__(self, username, hostname, env="", logger=""):
+       def __init__(self, username, hostname, env="", my_logger=""):
 
                self.username = username
                self.hostname = hostname
-               self.logger = logger
+               self.logger = my_logger
 
 
        # Copy to jail
-       def copy_to(self, src, dest, compress=True, logger=""):
+       def copy_to(self, src, dest, compress=True, my_logger=""):
                """Args: src (list), dest, Returns: (exit_code, output)."""
 
+               if not my_logger:
+                       my_logger = self.logger
+
                # this is the original 'scp' mode
                # fixed: You can't mput files outside the current local dir 
(going to have to chdir to each dir and issue separate smbclient commands)
                # Kinda ugly, but it works!
@@ -175,11 +193,13 @@
                #command = "%s -spc %s | smbclient //%s/%s -A %s -U %s -D %s 
-Trqx -" % (self.local_tar_path, src, self.host, self.SMB_SHARE, 
config.smb_passfile, self.user, dest)
 
 
-               return utils.launch_process(command, 
print_output=self.print_output)
+               return utils.launch_process(command, my_logger=my_logger)
                
-       def copy_from(self, src, dest, compress=True, logger=""):
+       def copy_from(self, src, dest, compress=True, my_logger=""):
                """Args: (src (list), dest) Returns: (exit_code, output)."""
 
+               if not my_logger:
+                       my_logger = self.logger
 
                # fixed: You can't mput files outside the current local dir 
(going to have to chdir to each dir and issue separate smbclient commands)
                # Kinda ugly, but it works!
@@ -193,6 +213,6 @@
                        command += "cd %s; smbclient //%s/%s -A %s -U %s -D %s 
-c 'prompt; recurse; mget %s' ; cd %s ;" % (dest, self.host, self.SMB_SHARE, 
config.smb_passfile, self.user, dir, filename, current_dir)
 
 
-               return utils.launch_process(command, 
print_output=self.print_output)
+               return utils.launch_process(command, my_logger=my_logger)
 
 

Modified: trunk/release/pyutils/packaging.py
===================================================================
--- trunk/release/pyutils/packaging.py  2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/packaging.py  2007-07-21 01:15:13 UTC (rev 82404)
@@ -17,7 +17,7 @@
 
 class buildconf:
 
-       def __init__(self, conf_file_name, logger="", skip_alternates=False):
+       def __init__(self, conf_file_name, my_logger="", skip_alternates=False):
                """skip_alternates is for times like jail-do, where I don't 
care if the jail is locked or not"""
 
                # Parse out name and counter (alternate)
@@ -42,7 +42,7 @@
 
                # Construct arguments
                buildenv_args = {}
-               buildenv_args['logger'] = logger
+               buildenv_args['my_logger'] = my_logger
 
                # Find a buildenv that isn't locked
                while True:
@@ -78,8 +78,7 @@
                        if not check_alternates:
                                break
 
-                       # TODO: also check for being offline
-                       if not self.buildenv.is_locked() or skip_alternates:
+                       if skip_alternates or self.buildenv.offline() or not 
self.buildenv.is_locked():
                                # we're done, use that buildenv
                                break
 

Modified: trunk/release/pyutils/utils.py
===================================================================
--- trunk/release/pyutils/utils.py      2007-07-21 00:26:17 UTC (rev 82403)
+++ trunk/release/pyutils/utils.py      2007-07-21 01:15:13 UTC (rev 82404)
@@ -49,6 +49,7 @@
 
 debug=0
 KILLED_EXIT_CODE=-42
+INTERRUPTED_EXIT_CODE=10 # Should be arbitrary... ?
 
 match_all = re.compile('')
 
@@ -337,7 +338,7 @@
                fd.close()
 
 
-def launch_process(command, capture_stderr=1, print_output=1, print_command=0, 
terminate_reg="", logger="", output_timeout=0, kill_process_group=0, 
max_output_size=0):
+def launch_process(command, capture_stderr=1, print_output=1, print_command=0, 
terminate_reg="", my_logger="", output_timeout=0, kill_process_group=0, 
max_output_size=0, interruptable=False):
        """Execute a command, return output (stdout and optionally stderr), and 
optionally print output the process is being run.
 
        Returns a tuple: exit code, output
@@ -367,10 +368,16 @@
                print_command=1
 
        # turn off output if a logger is used
-       if logger: print_output=0
+       if my_logger: print_output=0
 
        if print_command: print command
 
+       if interruptable:
+               interrupted_file = os.path.dirname(interruptable) + os.sep + 
"interrupted"
+               if os.path.exists(interrupted_file):
+                       print "** Failing to execute command '%s' because 
'interrupted' file is present: %s" % (command, interrupted_file )
+                       sys.exit(INTERRUPTED_EXIT_CODE)
+
        if capture_stderr:
                process = popen2.Popen4(command)
        else:
@@ -379,6 +386,12 @@
        # Close unnecessary handles
        process.tochild.close()
 
+
+       if interruptable:
+               fd = open(interruptable, 'w')
+               fd.write(str(os.getpgrp()))
+               fd.close()
+
        if output_timeout:
                flags = fcntl.fcntl(process.fromchild.fileno(), fcntl.F_GETFL)
                flags = flags | os.O_NONBLOCK
@@ -438,6 +451,10 @@
 
                                #  Kill the process
                                try:
+                                       # clean up interruptable file if it's 
there
+                                       if interruptable and 
os.path.exists(interruptable):
+                                               os.unlink(interruptable)
+
                                        # Kill the process group of the current 
process...
                                        if kill_process_group:
                                                print "** Killing process group 
(committing suicide) ... **"
@@ -470,8 +487,8 @@
                        sys.stdout.flush()
                        # This doesn't work on macosx... ?
                        #process.fromchild.flush()
-               if logger:
-                       logger.log(line)
+               if my_logger:
+                       my_logger.log(line)
 
                collected += [ line ]
 
@@ -485,6 +502,12 @@
        if killed:
                exit_code = KILLED_EXIT_CODE
 
+       # Clean up interruptable file? naw... shouldn't need to
+       #  actually we do, one to remove file if process isn't running,
+       #  also if we're root (install rpms), we need to remove this while we 
have rights
+       if interruptable and os.path.exists(interruptable):
+               os.unlink(interruptable)
+
        #  Concat array elements into a string 
        #  strip the whitespace off the end (makes it behave more like 
commands.getxxx)
        return exit_code, "".join(collected).rstrip()

_______________________________________________
Mono-patches maillist  -  [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches

Reply via email to