[gentoo-commits] proj/R_overlay:master commit in: roverlay/tools/
commit: 91474197a0ad11ad8169e8dbf582c9a29ab67cf9 Author: André Erdmann mailerd de> AuthorDate: Wed Feb 18 02:16:07 2015 + Commit: André Erdmann mailerd de> CommitDate: Wed Feb 18 02:16:07 2015 + URL: http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=91474197 fix 8647dac1ec5ad63cd44a8dfc121b8b0942b688ad add missing module / kwargs=>kill_kwargs typo --- roverlay/tools/subproc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roverlay/tools/subproc.py b/roverlay/tools/subproc.py index a62601f..b5fa476 100644 --- a/roverlay/tools/subproc.py +++ b/roverlay/tools/subproc.py @@ -7,6 +7,7 @@ import os import subprocess import sys +import time __all__ = [ 'get_subproc_devnull', @@ -98,7 +99,7 @@ def gracefully_stop_subprocess ( proc, **kill_kwargs ): if subproc_send_term ( proc ): proc.communicate() except: - stop_subprocess ( proc, **kwargs ) + stop_subprocess ( proc, **kill_kwargs ) raise def create_subprocess ( cmdv, **kwargs ):
[gentoo-commits] proj/R_overlay:master commit in: roverlay/tools/
commit: 60f3c871f6320e3b637d61f319c5960824567c82
Author: André Erdmann mailerd de>
AuthorDate: Tue Dec 16 00:17:10 2014 +
Commit: André Erdmann mailerd de>
CommitDate: Tue Dec 16 00:17:10 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=60f3c871
run non-user hooks with /dev/null as stdin
---
roverlay/tools/shenv.py | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/roverlay/tools/shenv.py b/roverlay/tools/shenv.py
index 8f2cea2..5670354 100644
--- a/roverlay/tools/shenv.py
+++ b/roverlay/tools/shenv.py
@@ -29,6 +29,8 @@ LOGGER = logging.getLogger ( 'shenv' )
NULL_PHASE = 'null'
+SHENV_PHASES_WITH_STDIN = frozenset ({ 'user', })
+
SHENV_VARS_TO_KEEP = frozenset ({
( 'PATH', '/usr/local/bin:/usr/bin:/bin' ),
'PWD',
@@ -348,7 +350,7 @@ def run_script_exec (
def run_script (
script, phase, argv=(), return_success=False, logger=None,
- log_output=True, initial_dir=None, allow_stdin=True
+ log_output=True, initial_dir=None
):
# global _SHELL_INTPR
# if _SHELL_INTPR is None:
@@ -360,7 +362,11 @@ def run_script (
script_call, output = _run_subprocess (
# ( _SHELL_INTPR, script, ),
( script, ) + argv,
- stdin = None if allow_stdin else False,
+ stdin = (
+ None if (
+my_env ['ROVERLAY_PHASE'] in SHENV_PHASES_WITH_STDIN
+ ) else False
+ ),
stdout = subprocess.PIPE if log_output else None,
stderr = subprocess.PIPE if log_output else None,
cwd= my_env ['S'] if initial_dir is None else initial_dir,
[gentoo-commits] proj/R_overlay:master commit in: roverlay/tools/
commit: f195588254e4f1516ec024973ebba3660622c3fa
Author: André Erdmann mailerd de>
AuthorDate: Mon Dec 15 23:00:17 2014 +
Commit: André Erdmann mailerd de>
CommitDate: Mon Dec 15 23:00:17 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f1955882
add subprocess helper module
---
roverlay/tools/subproc.py | 157 ++
1 file changed, 157 insertions(+)
diff --git a/roverlay/tools/subproc.py b/roverlay/tools/subproc.py
new file mode 100644
index 000..a62601f
--- /dev/null
+++ b/roverlay/tools/subproc.py
@@ -0,0 +1,157 @@
+# R overlay -- tools, subprocess helpers
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014 André Erdmann
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import os
+import subprocess
+import sys
+
+__all__ = [
+ 'get_subproc_devnull',
+ 'subproc_send_term', 'subproc_send_kill',
+ 'stop_subprocess', 'gracefully_stop_subprocess',
+ 'create_subprocess', 'run_subprocess'
+]
+
+# python >= 3.3 has ProcessLookupError, use more generic exception otherwise
+if sys.hexversion >= 0x303:
+ _ProcessLookupError = ProcessLookupError
+else:
+ _ProcessLookupError = OSError
+
+# python >= 3.3:
+if hasattr ( subprocess, 'DEVNULL' ):
+ def get_subproc_devnull(mode=None):
+ """Returns a devnull object suitable
+ for passing it as stdin/stdout/stderr to subprocess.Popen().
+
+ Python 3.3 and later variant: uses subprocess.DEVNULL
+
+ arguments:
+ * mode -- ignored
+ """
+ return subprocess.DEVNULL
+else:
+ def get_subproc_devnull(mode='a+'):
+ """Returns a devnull object suitable
+ for passing it as stdin/stdout/stderr to subprocess.Popen().
+
+ Python 3.2 and earlier variant: opens os.devnull
+
+ arguments:
+ * mode -- mode for open(). Defaults to read/append
+ """
+ return open ( os.devnull, mode )
+# --
+
+def _proc_catch_lookup_err ( func ):
+ def wrapped ( proc ):
+ try:
+ func ( proc )
+ except _ProcessLookupError:
+ return False
+ return True
+
+ return wrapped
+
+@_proc_catch_lookup_err
+def subproc_send_term ( proc ):
+ proc.terminate()
+
+@_proc_catch_lookup_err
+def subproc_send_kill ( proc ):
+ proc.kill()
+
+
+def stop_subprocess ( proc, kill_timeout_cs=10 ):
+ """Terminates or kills a subprocess created by subprocess.Popen().
+
+ Sends SIGTERM first and sends SIGKILL if the process is still alive after
+ the given timeout.
+
+ Returns: None
+
+ arguments:
+ * proc-- subprocess
+ * kill_timeout_cs -- max time to wait after terminate() before sending a
+kill signal (in centiseconds). Should be an int.
+Defaults to 10 (= 1s).
+ """
+ if not subproc_send_term ( proc ):
+ return
+
+ try:
+ for k in range ( kill_timeout_cs ):
+ if proc.poll() is not None:
+return
+ time.sleep ( 0.1 )
+ except:
+ subproc_send_kill ( proc )
+ else:
+ subproc_send_kill ( proc )
+# --- end of stop_subprocess (...) ---
+
+def gracefully_stop_subprocess ( proc, **kill_kwargs ):
+ try:
+ if subproc_send_term ( proc ):
+ proc.communicate()
+ except:
+ stop_subprocess ( proc, **kwargs )
+ raise
+
+def create_subprocess ( cmdv, **kwargs ):
+ """subprocess.Popen() wrapper that redirects stdin/stdout/stderr to
+ devnull or to a pipe if set to False/True.
+
+ Returns: subprocess
+
+ arguments:
+ * cmdv --
+ * **kwargs --
+ """
+ devnull_obj = None
+
+ for key in { 'stdin', 'stdout', 'stderr' }:
+ if key not in kwargs:
+ pass
+ elif kwargs [key] is True:
+ kwargs [key] = subprocess.PIPE
+
+ elif kwargs [key] is False:
+ if devnull_obj is None:
+devnull_obj = get_subproc_devnull()
+assert devnull_obj is not None
+
+ kwargs [key] = devnull_obj
+ # else don't care
+ # --
+
+ return subprocess.Popen ( cmdv, **kwargs )
+# --- end of create_subprocess (...) ---
+
+def run_subprocess ( cmdv, kill_timeout_cs=10, **kwargs ):
+ """Calls create_subprocess() and waits for the process to exit.
+ Catches exceptions and terminates/kills the process in that case
+
+ Returns: 2-tuple ( subprocess, output )
+
+ arguments:
+ * cmdv--
+ * kill_timeout_cs -- time to wait after SIGTERM before sending SIGKILL
+(in centiseconds), see stop_subprocess() for details
+Defaults to 10.
+ * **kwargs--
+ """
+ proc = None
+ try:
+ proc = create_subprocess ( cmdv, **kwargs )
+ output = proc.communicate()
+ except:
+ if proc is not None:
+ stop_subprocess ( proc, kill_timeout_cs=kill_timeout_cs )
+ raise
+
+ return ( proc, output )
+# --- end of run_subprocess (...) -
[gentoo-commits] proj/R_overlay:master commit in: roverlay/tools/
commit: cff8c8dd63384b8b38375168a4859fd0a78f3858 Author: André Erdmann mailerd de> AuthorDate: Mon Dec 15 23:07:40 2014 + Commit: André Erdmann mailerd de> CommitDate: Mon Dec 15 23:07:40 2014 + URL: http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=cff8c8dd run commands without stdin by default roverlay.tools->run_command(): let stdin default to /dev/null --- roverlay/tools/runcmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roverlay/tools/runcmd.py b/roverlay/tools/runcmd.py index 7561669..08e1bda 100644 --- a/roverlay/tools/runcmd.py +++ b/roverlay/tools/runcmd.py @@ -16,7 +16,7 @@ DEBUG_TO_CONSOLE = False def run_command_get_output ( cmdv, env, debug_to_console=False, use_filter=True, filter_func=None, - binary_stdout=False, stdin=None + binary_stdout=False, stdin=False ): # note that debug_to_console breaks calls that want to parse stdout pipe_target = None if debug_to_console else subprocess.PIPE @@ -46,9 +46,9 @@ def run_command_get_output ( return ( cmd_call, output ) # --- end of run_command_get_output (...) --- -def run_command ( cmdv, env, logger, return_success=False ): +def run_command ( cmdv, env, logger, return_success=False, **kwargs ): cmd_call, output = run_command_get_output ( - cmdv, env, DEBUG_TO_CONSOLE, use_filter=True + cmdv, env, DEBUG_TO_CONSOLE, use_filter=True, **kwargs ) # log stderr
[gentoo-commits] proj/R_overlay:master commit in: roverlay/tools/, roverlay/remote/
commit: 8647dac1ec5ad63cd44a8dfc121b8b0942b688ad
Author: André Erdmann mailerd de>
AuthorDate: Mon Dec 15 23:01:46 2014 +
Commit: André Erdmann mailerd de>
CommitDate: Mon Dec 15 23:02:28 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8647dac1
unify subprocess on-exception code
make sure to terminate/kill subprocesses on error.
---
roverlay/remote/rsync.py | 114 ++-
roverlay/tools/runcmd.py | 10 ++---
roverlay/tools/shenv.py | 41 +++--
3 files changed, 84 insertions(+), 81 deletions(-)
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 670e714..24e25f8 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -10,10 +10,15 @@ __all__ = [ 'RsyncRepo', ]
import os
import sys
-import subprocess
from roverlay import config, util
+import roverlay.tools.subproc
+from roverlay.tools.subproc import create_subprocess as _create_subprocess
+from roverlay.tools.subproc import stop_subprocess as _stop_subprocess
+from roverlay.tools.subproc import \
+ gracefully_stop_subprocess as _gracefully_stop_subprocess
+
from roverlay.remote.basicrepo import BasicRepo
RSYNC_ENV = util.keepenv (
@@ -55,6 +60,44 @@ DEFAULT_RSYNC_OPTS = (
'--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files
)
+def run_rsync ( cmdv, env=RSYNC_ENV ):
+ """Runs an rsync command and terminates/kills the process on error.
+
+ Returns: the command's returncode
+
+ Raises: Passes all exceptions
+
+ arguments:
+ * cmdv -- rsync command to (including the rsync executable!)
+ * env -- environment dict, defaults to RSYNC_ENV
+ """
+ proc = _create_subprocess ( cmdv, env=env )
+
+ try:
+ proc.communicate()
+
+ except KeyboardInterrupt:
+ sys.stderr.write (
+ "\nKeyboard interrupt - waiting for rsync to exit...\n"
+ )
+ # send SIGTERM and wait,
+ # fall back to _stop_subprocess() if another exception is hit
+ _gracefully_stop_subprocess ( proc, kill_timeout_cs=40 )
+ raise
+
+ except Exception:
+ # send SIGTERM, wait up to 4 seconds before sending SIGKILL
+ _stop_subprocess ( proc, kill_timeout_cs=40 )
+ raise
+ # --
+
+ if proc.returncode == RSYNC_SIGINT:
+ raise KeyboardInterrupt ( "propagated from rsync" )
+
+ return proc.returncode
+# --- end of run_rsync (...)
+
+
class RsyncRepo ( BasicRepo ):
def __init__ ( self,
@@ -113,9 +156,8 @@ class RsyncRepo ( BasicRepo ):
argv.extend ( ( self.remote_uri, self.distdir ) )
- # removing emty args from argv
- return tuple ( filter ( None, argv ) )
-
+ # remove empty args from argv
+ return [ arg for arg in argv if arg ]
# --- end of _rsync_argv (...) ---
def _dosync ( self ):
@@ -124,66 +166,38 @@ class RsyncRepo ( BasicRepo ):
"""
assert os.EX_OK not in RETRY_ON_RETCODE
- def waitfor ( p ):
- if p.communicate() != ( None, None ):
-raise AssertionError ( "expected None,None from communicate!" )
- if p.returncode == RSYNC_SIGINT:
-raise KeyboardInterrupt ( "propagated from rsync" )
-
- return p.returncode
- # --- end of waitfor (...) ---
-
- retcode = None
- proc= None
-
+ rsync_cmd = self._rsync_argv()
+ retcode = None
try:
- rsync_cmd = self._rsync_argv()
util.dodir ( self.distdir, mkdir_p=True )
self.logger.debug ( 'running rsync cmd: ' + ' '.join ( rsync_cmd ) )
- retry_count = 0
-
- proc= subprocess.Popen ( rsync_cmd, env=RSYNC_ENV )
- retcode = waitfor ( proc )
- proc= None
+ retcode = run_rsync ( rsync_cmd )
- while retcode in RETRY_ON_RETCODE and retry_count < MAX_RSYNC_RETRY:
-# this handles retcodes like
-# * 24: "Partial transfer due to vanished source files"
+ if retcode in RETRY_ON_RETCODE:
+for retry_count in range ( MAX_RSYNC_RETRY ):
+ # this handles retcodes like
+ # * 24: "Partial transfer due to vanished source files"
-retry_count += 1
-
-self.logger.warning (
- "rsync returned {ret!r}, retrying ({now}/{_max})".format (
- ret=retcode, now=retry_count, _max=MAX_RSYNC_RETRY
+ self.logger.warning (
+ "rsync returned {ret!r}, retrying ({now}/{_max})".format (
+ ret=retcode, now=retry_count, _max=MAX_RSYNC_RETRY
+ )
)
-)
-proc= subprocess.Popen ( rsync_cmd, env=RSYNC_ENV )
-retcode = waitfor ( proc )
-proc= None
- # -- end while
+ retcode = run_rsync ( rsync_cmd )
+ if retcode not in RETRY_ON_RETCODE: break
+ # -- end if
except Key
