Author: wade
Date: 2006-10-31 23:29:29 -0500 (Tue, 31 Oct 2006)
New Revision: 67205
Added:
trunk/release/pyutils/tests/log_test.py
Modified:
trunk/release/packaging/build
trunk/release/pyutils/launch_process.py
trunk/release/pyutils/logger.py
trunk/release/pyutils/sshutils.py
trunk/release/pyutils/tests/test_sshutils.py
trunk/release/pyutils/utils.py
Log:
pyutils/logger.py:
-add max log size support. Was going to use it for packaging/build, but
this was the wrong place to implement it. Leaving for possible future
use, even though it's not currently being used
pyutils/utils.py:
-added support to kill process when max output size has been reached, so
we don't create 1GB log files as we have in the past
pyutils/launch_process.py, packaging/build, pyutils/sshutils.py:
-changes to implement above change in utils.py
-fix terminate reg (although still has problems)
Modified: trunk/release/packaging/build
===================================================================
--- trunk/release/packaging/build 2006-11-01 03:38:48 UTC (rev 67204)
+++ trunk/release/packaging/build 2006-11-01 04:29:29 UTC (rev 67205)
@@ -33,6 +33,9 @@
import pdb
timeout=1200
+# Set max output to 10 MB, so the remote process dies if it exceed this.
+# (prevents us with ending up with 1GB log files. 10 MB should be sufficient)
+max_output_size = 10 * 1024 * 1024
def get_state(code, test=0):
print "Return code: %d" % code
@@ -250,7 +253,7 @@
build_env.env_vars['BUILD_OS'] = build_env.info['os']
build_env.env_vars['BUILD_ARCH'] = build_env.info['arch']
-(code, output) = build_env.ssh.execute(build_command, env=build_env.env_vars,
logger=log_obj, output_timeout=timeout)
+(code, output) = build_env.ssh.execute(build_command, env=build_env.env_vars,
logger=log_obj, output_timeout=timeout, max_output_size=max_output_size)
#code = 0
if code:
build_env.unlock_env()
@@ -283,7 +286,7 @@
log_obj = logger.Logger(filename=LOGFILE,
print_screen=0)
build_info.update_step(name, state="inprogress",
log=os.path.basename(LOGFILE), start=utils.get_time(), finish="")
- (code, output) = build_env.ssh.execute("%s cd %s " %
(step_env_pre_cmd, remote_src_dir + package.info['POSTBUILD_STEP%d' % counter]
), env=build_env.env_vars, logger=log_obj, output_timeout=timeout)
+ (code, output) = build_env.ssh.execute("%s cd %s " %
(step_env_pre_cmd, remote_src_dir + package.info['POSTBUILD_STEP%d' % counter]
), env=build_env.env_vars, logger=log_obj, output_timeout=timeout,
max_output_size=max_output_size)
state = get_state(code, test=1)
if code:
print "Step %d (%s) failed..." % (counter,
package.info['POSTBUILD_STEP_NAME%d' % counter])
Modified: trunk/release/pyutils/launch_process.py
===================================================================
--- trunk/release/pyutils/launch_process.py 2006-11-01 03:38:48 UTC (rev
67204)
+++ trunk/release/pyutils/launch_process.py 2006-11-01 04:29:29 UTC (rev
67205)
@@ -6,7 +6,7 @@
import utils
def usage():
- print "Usage: ./launch_process.py [--terminate_reg=<reg>]
[--output_timeout=<secs>] [--kill_process_group] <command>"
+ print "Usage: ./launch_process.py [--terminate_reg=<reg string>]
[--output_timeout=<secs>] [--max_output_size=<num chars>]
[--kill_process_group] <command>"
def commandline():
@@ -17,7 +17,7 @@
# Options for launch process
try:
#opts, command = getopt.gnu_getopt(sys.argv[1:], "", [
"terminate_reg=", "output_timeout=" ])
- opts, command = getopt.getopt(sys.argv[1:], "", [
"terminate_reg=", "output_timeout=", "kill_process_group" ])
+ opts, command = getopt.getopt(sys.argv[1:], "", [
"terminate_reg=", "output_timeout=", "max_output_size=", "kill_process_group" ])
except getopt.GetoptError:
usage()
sys.exit(1)
@@ -32,6 +32,8 @@
args['output_timeout'] = int(value)
if option == "--kill_process_group":
args['kill_process_group'] = 1
+ if option == "--max_output_size":
+ args['max_output_size'] = float(value)
command = " ".join(command)
code, output = utils.launch_process(command, **args)
Modified: trunk/release/pyutils/logger.py
===================================================================
--- trunk/release/pyutils/logger.py 2006-11-01 03:38:48 UTC (rev 67204)
+++ trunk/release/pyutils/logger.py 2006-11-01 04:29:29 UTC (rev 67205)
@@ -7,11 +7,21 @@
import re
-line_reg = re.compile('%s$' % os.linesep, re.M)
+#line_reg = re.compile('%s$' % os.linesep, re.M)
+class MaxLogOverflowException(Exception):
+ def __init__(self, msg, max):
+ print "Trying to log '%s': but max_logsize reached: %s MB" %
(msg, str(max/1024/1024))
+
class Logger:
- def __init__(self, filename="log.log", print_screen=1):
+ def __init__(self, filename="log.log", print_screen=1, max_size=0):
+ """Simple logging class.
+ filename: custom filename, otherwise will be log.log
+ print_screen: whether or not to do screen printout
+ max_size: throw exception if log is over this size in MB
(0 for no limit, which is the default)
+ """
+
self.print_screen = print_screen
self.full_path = os.path.abspath(filename)
@@ -21,6 +31,11 @@
self.fd = open(self.full_path, 'a')
+ if not max_size:
+ self.max_size = 0
+ else:
+ self.max_size = max_size * (1024 * 1024)
+
def __del__(self):
self.fd.close()
@@ -31,9 +46,15 @@
#if not line_reg.search(msg):
# msg += os.linesep
+ # throw exception if there is a size limit and we exceed it
+ if self.max_size and self.fd.tell() + len(msg) > self.max_size:
+ raise MaxLogOverflowException(msg, self.max_size)
+
self.fd.write(msg)
self.fd.flush()
if self.print_screen:
sys.stdout.write(msg)
sys.stdout.flush()
+
+
Modified: trunk/release/pyutils/sshutils.py
===================================================================
--- trunk/release/pyutils/sshutils.py 2006-11-01 03:38:48 UTC (rev 67204)
+++ trunk/release/pyutils/sshutils.py 2006-11-01 04:29:29 UTC (rev 67205)
@@ -64,7 +64,7 @@
# Args: command to execute, and option to print_output
# Will always return output
# NOTE: In order to use backticks, you must escape them in the string
you pass in
- def execute(self, command, capture_stderr=1, terminate_reg="", env={},
logger="", output_timeout=0, exec_as_root=0):
+ def execute(self, command, capture_stderr=1, terminate_reg="", env={},
logger="", output_timeout=0, exec_as_root=0, max_output_size=0):
"""Args, command string to execute. Option args:
capture_stderr, and terminate_reg.
capture_stderr is 1 or 0 (Currently unimplemented)
@@ -81,20 +81,22 @@
else:
command = "sudo " + command
- if output_timeout or terminate_reg:
+ if output_timeout or terminate_reg or max_output_size:
# Copy some files over
self.copy_to([ config.packaging_dir +
"/../pyutils/utils.py", config.packaging_dir + "/../pyutils/launch_process.py"
], self.build_location)
- term_portion = ""
+ extra_options = ""
if terminate_reg:
- term_portion = "--terminate_reg=%s" %
terminate_reg
+ extra_options += " --terminate_reg=%s" %
terminate_reg
- output_portion = ""
if output_timeout:
- output_portion = "--output_timeout=%s" %
output_timeout
+ extra_options += " --output_timeout=%s" %
output_timeout
+ if max_output_size:
+ extra_options += " --max_output_size=%s" %
max_output_size
+
# Surround command with escaped double quotes in order
to run multiline commands
- command = "%s/launch_process.py --kill_process_group %s
%s \\\"%s\\\" " % (self.build_location, term_portion, output_portion, command)
+ command = "%s/launch_process.py --kill_process_group %s
\\\"%s\\\" " % (self.build_location, extra_options, command)
if self.target_command_prefix:
command = self.target_command_prefix + command
Added: trunk/release/pyutils/tests/log_test.py
===================================================================
--- trunk/release/pyutils/tests/log_test.py 2006-11-01 03:38:48 UTC (rev
67204)
+++ trunk/release/pyutils/tests/log_test.py 2006-11-01 04:29:29 UTC (rev
67205)
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+
+import sys
+sys.path.append('..')
+
+import logger
+
+log = logger.Logger(print_screen=1, max_size=1)
+
+while(1):
+ try:
+ log.log("Hey, are we here yet?\n")
+ except logger.MaxLogOverflowException:
+ print "Hit max logsize of...%d" % log.max_size
+ break
+
+
+while(1):
+ log.log("Nope...\n")
Property changes on: trunk/release/pyutils/tests/log_test.py
___________________________________________________________________
Name: svn:executable
+ *
Modified: trunk/release/pyutils/tests/test_sshutils.py
===================================================================
--- trunk/release/pyutils/tests/test_sshutils.py 2006-11-01 03:38:48 UTC
(rev 67204)
+++ trunk/release/pyutils/tests/test_sshutils.py 2006-11-01 04:29:29 UTC
(rev 67205)
@@ -30,4 +30,11 @@
host.copy_to('*.py', '/tmp')
host.copy_to(['test_sshutils.pyc', 'rem*' ], '/tmp')
+# try some execute options
+host = sshutils.init('[EMAIL PROTECTED]', print_output=1)
+host.execute('find /', max_output_size=10000)
+host.execute('ls /tmp/sdfl')
+host.execute('ls /tmp/sdfl; sleep 15', output_timeout=10)
+host.execute('find /', terminate_reg="h.me")
+host.execute('find /', terminate_reg="h.me", output_timeout=10,
max_output_size=10000)
Modified: trunk/release/pyutils/utils.py
===================================================================
--- trunk/release/pyutils/utils.py 2006-11-01 03:38:48 UTC (rev 67204)
+++ trunk/release/pyutils/utils.py 2006-11-01 04:29:29 UTC (rev 67205)
@@ -300,25 +300,29 @@
fd.close()
-def launch_process(command, capture_stderr=1, print_output=1, print_command=0,
terminate_reg="", logger="", output_timeout=0, kill_process_group=0):
+def launch_process(command, capture_stderr=1, print_output=1, print_command=0,
terminate_reg="", logger="", output_timeout=0, kill_process_group=0,
max_output_size=0):
"""Execute a command, return output (stdout and optionally stderr), and
optionally print output the process is being run.
Returns a tuple: exit code, output
- terminate_reg is a regular expression object where if it is matched
during the output,
+ terminate_reg is a regular expression string where if it is matched
during the output,
execution terminates, and None is returned for exit code, and
output thus far is returned
If a logger object (pyutils/logger) is passed in, output will be logged
to there as well.
output_timeout: kill process if it produces no output for x number of
seconds.
+ max_output_size: kill process if it outputs more than x chars. 0
(default) to disable.
+
kill_process_group: This determines how the process is killed when
output_timeout is specified.
If this is true, the process group of the current python
interpreter is killed, killing all subprocesses.
If not, only the child process id is killed.
The exit status of killed processes seems to vary (see
packaging/build for possibilities)."""
terminate_flag = 0
- if terminate_reg: terminate_flag = 1
+ if terminate_reg:
+ terminate_flag = 1
+ terminate_reg_obj = re.compile(terminate_reg)
# This is set by utils.debug
if debug:
@@ -351,6 +355,11 @@
collected = []
+ # Track number of output bytes so we can die if it goes above log max
+ output_size = 0
+ output_overflow = False
+ output_timeout_flag = False
+
# there's a bug with output_timeout... ex:
# adding: lib/mono/1.0/mono-service.exe.mdb (deflated 51%)
# (deflated 52%)
@@ -361,17 +370,25 @@
# Use this looping method insead of 'for line in process' so it doesn't
use the readahead buffer
# This smooths output greatly, instead of getting big chunks of output
with lots of lag
- # TODO: Track number of output bytes so we can die if it goes above log
max
while(1):
try:
#line = process.fromchild.readline()
# in non blocking mode, readline somehow misses
data... use read() instead
line = process.fromchild.read(128)
+ output_size += len(line)
output_received_timestamp = time.time()
+
+ if max_output_size and output_size > max_output_size:
+ print "** Terminating process because output
size max of %d exceeded: %d chars(s)**" % (max_output_size, output_size)
+ output_overflow = True
+ raise IOError
+
except IOError:
#print "No data..."
- if time.time() - output_received_timestamp >
output_timeout:
+ if output_timeout and time.time() -
output_received_timestamp > output_timeout:
+ output_timeout_flag = True
print "** Terminating process because no output
for %d second(s)**" % output_timeout
+ if output_overflow or output_timeout_flag:
sys.stdout.flush()
# Close I/O handles?
process.fromchild.close()
@@ -415,8 +432,8 @@
collected += [ line ]
- if terminate_flag and terminate_reg.search(line):
- print "** Terminating process because termination
string was found **"
+ if terminate_flag and terminate_reg_obj.search(line):
+ print "** Terminating process because termination
regular expression (%s) was found: '%s'" % (terminate_reg, line)
os.kill(process.pid, signal.SIGKILL)
break
_______________________________________________
Mono-patches maillist - [email protected]
http://lists.ximian.com/mailman/listinfo/mono-patches