[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug 29 19:56:50 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 30 23:50:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-   ('glsa', 'metadata/glsa'),
-   ('news', 'metadata/news'),
-   ('profiles', 'profiles'),
+   ('glsa', ('metadata/glsa',)),
+   ('news', ('metadata/news',)),
+   ('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
emerge_config = self.options.get('emerge_config')
if emerge_config is not None:
for name in emerge_config.opts.get('--sync-submodule', 
[]):
-   paths.append(_SUBMODULE_PATH_MAP[name])
+   paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec 12 22:24:04 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Dec 12 23:40:41 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
self.repo = repo
self.exitcode = 1
self.updatecache_flg = False
+   hooks_enabled = master_hooks or not 
repo.sync_hooks_only_on_change
if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \
% (bad("ERROR"), repo.sync_type)
-   return self.exitcode, msg
+   return self.exitcode, msg, self.updatecache_flg, 
hooks_enabled
 
rval = self.pre_sync(repo)
if rval != os.EX_OK:
-   return rval, None
+   return rval, None, self.updatecache_flg, hooks_enabled
 
# need to pass the kwargs dict to the modules
# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-   hooks_enabled = False
if (master_hooks or self.updatecache_flg or
not repo.sync_hooks_only_on_change):
hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: baeff1e7a7306081123d9a31b24c62d59ae73abb
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Nov 22 20:05:37 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Nov 24 16:15:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/sync/controller.py | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
self.usersync_uid = None
spawn_kwargs = {}
+   # Redirect command stderr to stdout, in order to prevent
+   # spurious cron job emails (bug 566132).
+   spawn_kwargs["fd_pipes"] = {
+   0: sys.__stdin__.fileno(),
+   1: sys.__stdout__.fileno(),
+   2: sys.__stdout__.fileno()
+   }
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4205ec912aebf2e3b0bd673fcacb576b1f344329
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct  5 23:01:49 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct  6 16:25:37 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 39 +++
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
-   proc = AsyncFunction(target=self.sync,
-   kwargs=dict(emerge_config=emerge_config, repo=repo))
-   proc.addExitListener(self._sync_callback)
-   return proc
+   return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+   kwargs=dict(emerge_config=emerge_config, repo=repo)),
+   sync_callback=self._sync_callback)
 
def sync(self, emerge_config=None, repo=None):
self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
action_metadata(self.settings, self.portdb, 
self.emerge_config.opts,
porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+   """
+   Encapsulates a sync operation and the callback which executes 
afterwards,
+   so both can be considered as a single composite task. This is useful
+   since we don't want to consider a particular repo's sync operation as
+   complete until after the callback has executed (bug 562264).
+
+   The kwargs and result properties expose attributes that are accessed
+   by SyncScheduler.
+   """
+
+   __slots__ = ('sync_task', 'sync_callback')
+
+   @property
+   def kwargs(self):
+   return self.sync_task.kwargs
+
+   @property
+   def result(self):
+   return self.sync_task.result
+
+   def _start(self):
+   self._start_task(self.sync_task, self._sync_task_exit)
+
+   def _sync_task_exit(self, sync_task):
+   self._current_task = None
+   self.returncode = sync_task.returncode
+   self.sync_callback(self.sync_task)
+   self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 200d876b01dd11521cdcd9bfa07abdca165d24e8
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Sep 23 16:16:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Sep 23 22:31:36 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
return []
 
def async(self, emerge_config=None, repo=None):
+   self.emerge_config = emerge_config
+   self.settings, self.trees, self.mtimedb = emerge_config
+   self.xterm_titles = "notitles" not in self.settings.features
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
proc = AsyncFunction(target=self.sync,
kwargs=dict(emerge_config=emerge_config, repo=repo))
proc.addExitListener(self._sync_callback)
return proc
 
def sync(self, emerge_config=None, repo=None):
-   self.emerge_config = emerge_config
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
def pre_sync(self, repo):
-   self.settings, self.trees, self.mtimedb = self.emerge_config
-   self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
-   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: c0e1d514c1c8159aa732d23a527640432a9c076d
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Jul 14 20:40:23 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Jul 14 21:28:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing 
sync modules.

 pym/portage/sync/__init__.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
global module_controller
 
if repo.sync_type:
-   opts = frozenset([opt for opt in
-   
module_controller.modules[repo.sync_type]['module_specific_options']])
-   return opts
+   try:
+   return frozenset(
+   
module_controller.modules[repo.sync_type]['module_specific_options'])
+   except KeyError:
+   pass
return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:43:47 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny  gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:51:01 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's 
parameters.
But it was not defined when it was moved into task_opts dictionary to be passed 
to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
task_opts = {
'emerge_config': emerge_config,
'logger': self.logger,
-   'portdb': 
self.trees[self.settings['EROOT']]['porttree'].dbapi,
+   'portdb': self.portdb,
'repo': repo,
'settings': self.settings,
'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 7a16988f4765963896170b92253ebbd4344216ea
Author: Brian Dolbec  gentoo  org>
AuthorDate: Wed Apr 22 16:58:38 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Wed Apr 22 16:58:38 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the 
native portage one
causes problems.  This is due to the fact the portage-utils supplied script did 
not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
writemsg_level("Spawning post_sync hook: %s\n"
% (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
-   retval = portage.process.spawn([filepath,
-   reponame, dosyncuri, repolocation], 
env=self.settings.environ())
+   if reponame:
+   retval = portage.process.spawn(
+   [filepath, reponame, dosyncuri, 
repolocation],
+   env=self.settings.environ())
+   else:
+   retval = portage.process.spawn([filepath],
+   env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
_unicode_decode(_hooks[filepath]), 
filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 768b9e920fe0919d87537665ae3ce88007630cc1
Author: Michał Górny  gentoo  org>
AuthorDate: Sat Jan 17 13:03:53 2015 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Jan 18 18:04:25 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
self.callback = callback or self._sync_callback
self.repo = repo
self.exitcode = 1
-   if repo.sync_type in self.module_names[1:]:
+   if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sat Dec  6 22:54:36 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 05:22:51 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d 
directory.
This also maintains compatibility with existing scripts in the postsync.d dir 
or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
self.module_controller = portage.sync.module_controller
self.module_names = self.module_controller.module_names
-   postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-   portage.USER_CONFIG_PATH, "postsync.d")
-   hooks = []
-   for root, dirs, names in os.walk(postsync_dir, topdown=True):
-   #print("root:", root, "dirs:", dirs, "names:", names)
-   for name in names:
-   filepath = os.path.join(root, name)
+   self.hooks = {}
+   for _dir in ["repo.postsync.d", "postsync.d"]:
+   postsync_dir = 
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+   portage.USER_CONFIG_PATH, _dir)
+   hooks = OrderedDict()
+   for filepath in util._recursive_file_list(postsync_dir):
+   name = 
filepath.split(postsync_dir)[1].lstrip(os.sep)
if os.access(filepath, os.X_OK):
-   hooks.append((filepath, name))
+   hooks[filepath] = name
else:
-   writemsg_level(" %s postsync.d hook: 
'%s' is not executable\n"
-   % (warn("*"), 
_unicode_decode(name),), level=logging.WARN, noiselevel=2)
-   self.hooks = hooks
+   writemsg_level(" %s %s hook: '%s' is 
not executable\n"
+   % (warn("*"), _dir, 
_unicode_decode(name),),
+   level=logging.WARN, 
noiselevel=2)
+   self.hooks[_dir] = hooks
+   print(self.hooks)
 
 
def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
def perform_post_sync_hook(self, reponame, dosyncuri='', 
repolocation=''):
succeeded = os.EX_OK
-   for filepath, hook in self.hooks:
+   if reponame:
+   _hooks = self.hooks["repo.postsync.d"]
+   else:
+   _hooks = self.hooks["postsync.d"]
+   for filepath in _hooks:
writemsg_level("Spawning post_sync hook: %s\n"
-   % (_unicode_decode(hook)),
+   % (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
retval = portage.process.spawn([filepath,
reponame, dosyncuri, repolocation], 
env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
-   _unicode_decode(hook), filepath),
+   _unicode_decode(_hooks[filepath]), 
filepath),
level=logging.ERROR, noiselevel=-1)
succeeded = retval
return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 8a5ee0a68764a272c554ed534fcb252e81f89118
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sun Dec  7 09:12:13 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 09:12:49 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
% (warn("*"), _dir, 
_unicode_decode(name),),
level=logging.WARN, 
noiselevel=2)
self.hooks[_dir] = hooks
-   print(self.hooks)
 
 
def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 9a2b1e6096e35508e9f101fa0478101493ef0335
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Dec  5 22:54:27 2014 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Dec  7 22:57:07 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++--
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
st = os.stat(repo.location)
except OSError:
st = None
-   if st is None:
-   writemsg_level(">>> '%s' not found, creating it."
-   % _unicode_decode(repo.location))
-   portage.util.ensure_dirs(repo.location, mode=0o755)
-   st = os.stat(repo.location)
 
self.usersync_uid = None
spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
-   elif ('usersync' in self.settings.features and
+
+   if st is None:
+   perms = {'mode': 0o755}
+   # respect sync-user if set
+   if 'umask' in spawn_kwargs:
+   perms['mode'] &= ~spawn_kwargs['umask']
+   if 'uid' in spawn_kwargs:
+   perms['uid'] = spawn_kwargs['uid']
+   if 'gid' in spawn_kwargs:
+   perms['gid'] = spawn_kwargs['gid']
+
+   writemsg_level(">>> '%s' not found, creating it."
+   % _unicode_decode(repo.location))
+   portage.util.ensure_dirs(repo.location, **perms)
+   st = os.stat(repo.location)
+
+   if (repo.sync_user is None and
+   'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:43:47 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny  gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:51:01 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's 
parameters.
But it was not defined when it was moved into task_opts dictionary to be passed 
to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
task_opts = {
'emerge_config': emerge_config,
'logger': self.logger,
-   'portdb': 
self.trees[self.settings['EROOT']]['porttree'].dbapi,
+   'portdb': self.portdb,
'repo': repo,
'settings': self.settings,
'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sat Dec  6 22:54:36 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 05:22:51 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d 
directory.
This also maintains compatibility with existing scripts in the postsync.d dir 
or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
self.module_controller = portage.sync.module_controller
self.module_names = self.module_controller.module_names
-   postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-   portage.USER_CONFIG_PATH, "postsync.d")
-   hooks = []
-   for root, dirs, names in os.walk(postsync_dir, topdown=True):
-   #print("root:", root, "dirs:", dirs, "names:", names)
-   for name in names:
-   filepath = os.path.join(root, name)
+   self.hooks = {}
+   for _dir in ["repo.postsync.d", "postsync.d"]:
+   postsync_dir = 
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+   portage.USER_CONFIG_PATH, _dir)
+   hooks = OrderedDict()
+   for filepath in util._recursive_file_list(postsync_dir):
+   name = 
filepath.split(postsync_dir)[1].lstrip(os.sep)
if os.access(filepath, os.X_OK):
-   hooks.append((filepath, name))
+   hooks[filepath] = name
else:
-   writemsg_level(" %s postsync.d hook: 
'%s' is not executable\n"
-   % (warn("*"), 
_unicode_decode(name),), level=logging.WARN, noiselevel=2)
-   self.hooks = hooks
+   writemsg_level(" %s %s hook: '%s' is 
not executable\n"
+   % (warn("*"), _dir, 
_unicode_decode(name),),
+   level=logging.WARN, 
noiselevel=2)
+   self.hooks[_dir] = hooks
+   print(self.hooks)
 
 
def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
def perform_post_sync_hook(self, reponame, dosyncuri='', 
repolocation=''):
succeeded = os.EX_OK
-   for filepath, hook in self.hooks:
+   if reponame:
+   _hooks = self.hooks["repo.postsync.d"]
+   else:
+   _hooks = self.hooks["postsync.d"]
+   for filepath in _hooks:
writemsg_level("Spawning post_sync hook: %s\n"
-   % (_unicode_decode(hook)),
+   % (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
retval = portage.process.spawn([filepath,
reponame, dosyncuri, repolocation], 
env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
-   _unicode_decode(hook), filepath),
+   _unicode_decode(_hooks[filepath]), 
filepath),
level=logging.ERROR, noiselevel=-1)
succeeded = retval
return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 8a5ee0a68764a272c554ed534fcb252e81f89118
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sun Dec  7 09:12:13 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 09:12:49 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
% (warn("*"), _dir, 
_unicode_decode(name),),
level=logging.WARN, 
noiselevel=2)
self.hooks[_dir] = hooks
-   print(self.hooks)
 
 
def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 9a2b1e6096e35508e9f101fa0478101493ef0335
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Dec  5 22:54:27 2014 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Dec  7 22:57:07 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++--
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
st = os.stat(repo.location)
except OSError:
st = None
-   if st is None:
-   writemsg_level(">>> '%s' not found, creating it."
-   % _unicode_decode(repo.location))
-   portage.util.ensure_dirs(repo.location, mode=0o755)
-   st = os.stat(repo.location)
 
self.usersync_uid = None
spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
-   elif ('usersync' in self.settings.features and
+
+   if st is None:
+   perms = {'mode': 0o755}
+   # respect sync-user if set
+   if 'umask' in spawn_kwargs:
+   perms['mode'] &= ~spawn_kwargs['umask']
+   if 'uid' in spawn_kwargs:
+   perms['uid'] = spawn_kwargs['uid']
+   if 'gid' in spawn_kwargs:
+   perms['gid'] = spawn_kwargs['gid']
+
+   writemsg_level(">>> '%s' not found, creating it."
+   % _unicode_decode(repo.location))
+   portage.util.ensure_dirs(repo.location, **perms)
+   st = os.stat(repo.location)
+
+   if (repo.sync_user is None and
+   'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 768b9e920fe0919d87537665ae3ce88007630cc1
Author: Michał Górny  gentoo  org>
AuthorDate: Sat Jan 17 13:03:53 2015 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Jan 18 18:04:25 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
self.callback = callback or self._sync_callback
self.repo = repo
self.exitcode = 1
-   if repo.sync_type in self.module_names[1:]:
+   if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 7a16988f4765963896170b92253ebbd4344216ea
Author: Brian Dolbec  gentoo  org>
AuthorDate: Wed Apr 22 16:58:38 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Wed Apr 22 16:58:38 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the 
native portage one
causes problems.  This is due to the fact the portage-utils supplied script did 
not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
writemsg_level("Spawning post_sync hook: %s\n"
% (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
-   retval = portage.process.spawn([filepath,
-   reponame, dosyncuri, repolocation], 
env=self.settings.environ())
+   if reponame:
+   retval = portage.process.spawn(
+   [filepath, reponame, dosyncuri, 
repolocation],
+   env=self.settings.environ())
+   else:
+   retval = portage.process.spawn([filepath],
+   env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
_unicode_decode(_hooks[filepath]), 
filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: c0e1d514c1c8159aa732d23a527640432a9c076d
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Jul 14 20:40:23 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Jul 14 21:28:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing 
sync modules.

 pym/portage/sync/__init__.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
global module_controller
 
if repo.sync_type:
-   opts = frozenset([opt for opt in
-   
module_controller.modules[repo.sync_type]['module_specific_options']])
-   return opts
+   try:
+   return frozenset(
+   
module_controller.modules[repo.sync_type]['module_specific_options'])
+   except KeyError:
+   pass
return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug 29 19:56:50 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 30 23:50:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-   ('glsa', 'metadata/glsa'),
-   ('news', 'metadata/news'),
-   ('profiles', 'profiles'),
+   ('glsa', ('metadata/glsa',)),
+   ('news', ('metadata/news',)),
+   ('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
emerge_config = self.options.get('emerge_config')
if emerge_config is not None:
for name in emerge_config.opts.get('--sync-submodule', 
[]):
-   paths.append(_SUBMODULE_PATH_MAP[name])
+   paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec 12 22:24:04 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Dec 12 23:40:41 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
self.repo = repo
self.exitcode = 1
self.updatecache_flg = False
+   hooks_enabled = master_hooks or not 
repo.sync_hooks_only_on_change
if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \
% (bad("ERROR"), repo.sync_type)
-   return self.exitcode, msg
+   return self.exitcode, msg, self.updatecache_flg, 
hooks_enabled
 
rval = self.pre_sync(repo)
if rval != os.EX_OK:
-   return rval, None
+   return rval, None, self.updatecache_flg, hooks_enabled
 
# need to pass the kwargs dict to the modules
# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-   hooks_enabled = False
if (master_hooks or self.updatecache_flg or
not repo.sync_hooks_only_on_change):
hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: baeff1e7a7306081123d9a31b24c62d59ae73abb
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Nov 22 20:05:37 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Nov 24 16:15:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/sync/controller.py | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
self.usersync_uid = None
spawn_kwargs = {}
+   # Redirect command stderr to stdout, in order to prevent
+   # spurious cron job emails (bug 566132).
+   spawn_kwargs["fd_pipes"] = {
+   0: sys.__stdin__.fileno(),
+   1: sys.__stdout__.fileno(),
+   2: sys.__stdout__.fileno()
+   }
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 200d876b01dd11521cdcd9bfa07abdca165d24e8
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Sep 23 16:16:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Sep 23 22:31:36 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
return []
 
def async(self, emerge_config=None, repo=None):
+   self.emerge_config = emerge_config
+   self.settings, self.trees, self.mtimedb = emerge_config
+   self.xterm_titles = "notitles" not in self.settings.features
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
proc = AsyncFunction(target=self.sync,
kwargs=dict(emerge_config=emerge_config, repo=repo))
proc.addExitListener(self._sync_callback)
return proc
 
def sync(self, emerge_config=None, repo=None):
-   self.emerge_config = emerge_config
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
def pre_sync(self, repo):
-   self.settings, self.trees, self.mtimedb = self.emerge_config
-   self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
-   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4205ec912aebf2e3b0bd673fcacb576b1f344329
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct  5 23:01:49 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct  6 16:25:37 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 39 +++
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
-   proc = AsyncFunction(target=self.sync,
-   kwargs=dict(emerge_config=emerge_config, repo=repo))
-   proc.addExitListener(self._sync_callback)
-   return proc
+   return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+   kwargs=dict(emerge_config=emerge_config, repo=repo)),
+   sync_callback=self._sync_callback)
 
def sync(self, emerge_config=None, repo=None):
self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
action_metadata(self.settings, self.portdb, 
self.emerge_config.opts,
porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+   """
+   Encapsulates a sync operation and the callback which executes 
afterwards,
+   so both can be considered as a single composite task. This is useful
+   since we don't want to consider a particular repo's sync operation as
+   complete until after the callback has executed (bug 562264).
+
+   The kwargs and result properties expose attributes that are accessed
+   by SyncScheduler.
+   """
+
+   __slots__ = ('sync_task', 'sync_callback')
+
+   @property
+   def kwargs(self):
+   return self.sync_task.kwargs
+
+   @property
+   def result(self):
+   return self.sync_task.result
+
+   def _start(self):
+   self._start_task(self.sync_task, self._sync_task_exit)
+
+   def _sync_task_exit(self, sync_task):
+   self._current_task = None
+   self.returncode = sync_task.returncode
+   self.sync_callback(self.sync_task)
+   self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sat Dec  6 22:54:36 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 05:22:51 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d 
directory.
This also maintains compatibility with existing scripts in the postsync.d dir 
or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
self.module_controller = portage.sync.module_controller
self.module_names = self.module_controller.module_names
-   postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-   portage.USER_CONFIG_PATH, "postsync.d")
-   hooks = []
-   for root, dirs, names in os.walk(postsync_dir, topdown=True):
-   #print("root:", root, "dirs:", dirs, "names:", names)
-   for name in names:
-   filepath = os.path.join(root, name)
+   self.hooks = {}
+   for _dir in ["repo.postsync.d", "postsync.d"]:
+   postsync_dir = 
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+   portage.USER_CONFIG_PATH, _dir)
+   hooks = OrderedDict()
+   for filepath in util._recursive_file_list(postsync_dir):
+   name = 
filepath.split(postsync_dir)[1].lstrip(os.sep)
if os.access(filepath, os.X_OK):
-   hooks.append((filepath, name))
+   hooks[filepath] = name
else:
-   writemsg_level(" %s postsync.d hook: 
'%s' is not executable\n"
-   % (warn("*"), 
_unicode_decode(name),), level=logging.WARN, noiselevel=2)
-   self.hooks = hooks
+   writemsg_level(" %s %s hook: '%s' is 
not executable\n"
+   % (warn("*"), _dir, 
_unicode_decode(name),),
+   level=logging.WARN, 
noiselevel=2)
+   self.hooks[_dir] = hooks
+   print(self.hooks)
 
 
def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
def perform_post_sync_hook(self, reponame, dosyncuri='', 
repolocation=''):
succeeded = os.EX_OK
-   for filepath, hook in self.hooks:
+   if reponame:
+   _hooks = self.hooks["repo.postsync.d"]
+   else:
+   _hooks = self.hooks["postsync.d"]
+   for filepath in _hooks:
writemsg_level("Spawning post_sync hook: %s\n"
-   % (_unicode_decode(hook)),
+   % (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
retval = portage.process.spawn([filepath,
reponame, dosyncuri, repolocation], 
env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
-   _unicode_decode(hook), filepath),
+   _unicode_decode(_hooks[filepath]), 
filepath),
level=logging.ERROR, noiselevel=-1)
succeeded = retval
return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 8a5ee0a68764a272c554ed534fcb252e81f89118
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sun Dec  7 09:12:13 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 09:12:49 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
% (warn("*"), _dir, 
_unicode_decode(name),),
level=logging.WARN, 
noiselevel=2)
self.hooks[_dir] = hooks
-   print(self.hooks)
 
 
def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 9a2b1e6096e35508e9f101fa0478101493ef0335
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Dec  5 22:54:27 2014 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Dec  7 22:57:07 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++--
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
st = os.stat(repo.location)
except OSError:
st = None
-   if st is None:
-   writemsg_level(">>> '%s' not found, creating it."
-   % _unicode_decode(repo.location))
-   portage.util.ensure_dirs(repo.location, mode=0o755)
-   st = os.stat(repo.location)
 
self.usersync_uid = None
spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
-   elif ('usersync' in self.settings.features and
+
+   if st is None:
+   perms = {'mode': 0o755}
+   # respect sync-user if set
+   if 'umask' in spawn_kwargs:
+   perms['mode'] &= ~spawn_kwargs['umask']
+   if 'uid' in spawn_kwargs:
+   perms['uid'] = spawn_kwargs['uid']
+   if 'gid' in spawn_kwargs:
+   perms['gid'] = spawn_kwargs['gid']
+
+   writemsg_level(">>> '%s' not found, creating it."
+   % _unicode_decode(repo.location))
+   portage.util.ensure_dirs(repo.location, **perms)
+   st = os.stat(repo.location)
+
+   if (repo.sync_user is None and
+   'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:43:47 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny  gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:51:01 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's 
parameters.
But it was not defined when it was moved into task_opts dictionary to be passed 
to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
task_opts = {
'emerge_config': emerge_config,
'logger': self.logger,
-   'portdb': 
self.trees[self.settings['EROOT']]['porttree'].dbapi,
+   'portdb': self.portdb,
'repo': repo,
'settings': self.settings,
'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 7a16988f4765963896170b92253ebbd4344216ea
Author: Brian Dolbec  gentoo  org>
AuthorDate: Wed Apr 22 16:58:38 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Wed Apr 22 16:58:38 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the 
native portage one
causes problems.  This is due to the fact the portage-utils supplied script did 
not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
writemsg_level("Spawning post_sync hook: %s\n"
% (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
-   retval = portage.process.spawn([filepath,
-   reponame, dosyncuri, repolocation], 
env=self.settings.environ())
+   if reponame:
+   retval = portage.process.spawn(
+   [filepath, reponame, dosyncuri, 
repolocation],
+   env=self.settings.environ())
+   else:
+   retval = portage.process.spawn([filepath],
+   env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
_unicode_decode(_hooks[filepath]), 
filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 768b9e920fe0919d87537665ae3ce88007630cc1
Author: Michał Górny  gentoo  org>
AuthorDate: Sat Jan 17 13:03:53 2015 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Jan 18 18:04:25 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
self.callback = callback or self._sync_callback
self.repo = repo
self.exitcode = 1
-   if repo.sync_type in self.module_names[1:]:
+   if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: c0e1d514c1c8159aa732d23a527640432a9c076d
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Jul 14 20:40:23 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Jul 14 21:28:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing 
sync modules.

 pym/portage/sync/__init__.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
global module_controller
 
if repo.sync_type:
-   opts = frozenset([opt for opt in
-   
module_controller.modules[repo.sync_type]['module_specific_options']])
-   return opts
+   try:
+   return frozenset(
+   
module_controller.modules[repo.sync_type]['module_specific_options'])
+   except KeyError:
+   pass
return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug 29 19:56:50 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 30 23:50:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-   ('glsa', 'metadata/glsa'),
-   ('news', 'metadata/news'),
-   ('profiles', 'profiles'),
+   ('glsa', ('metadata/glsa',)),
+   ('news', ('metadata/news',)),
+   ('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
emerge_config = self.options.get('emerge_config')
if emerge_config is not None:
for name in emerge_config.opts.get('--sync-submodule', 
[]):
-   paths.append(_SUBMODULE_PATH_MAP[name])
+   paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 200d876b01dd11521cdcd9bfa07abdca165d24e8
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Sep 23 16:16:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Sep 23 22:31:36 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
return []
 
def async(self, emerge_config=None, repo=None):
+   self.emerge_config = emerge_config
+   self.settings, self.trees, self.mtimedb = emerge_config
+   self.xterm_titles = "notitles" not in self.settings.features
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
proc = AsyncFunction(target=self.sync,
kwargs=dict(emerge_config=emerge_config, repo=repo))
proc.addExitListener(self._sync_callback)
return proc
 
def sync(self, emerge_config=None, repo=None):
-   self.emerge_config = emerge_config
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
def pre_sync(self, repo):
-   self.settings, self.trees, self.mtimedb = self.emerge_config
-   self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
-   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec 12 22:24:04 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Dec 12 23:40:41 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
self.repo = repo
self.exitcode = 1
self.updatecache_flg = False
+   hooks_enabled = master_hooks or not 
repo.sync_hooks_only_on_change
if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \
% (bad("ERROR"), repo.sync_type)
-   return self.exitcode, msg
+   return self.exitcode, msg, self.updatecache_flg, 
hooks_enabled
 
rval = self.pre_sync(repo)
if rval != os.EX_OK:
-   return rval, None
+   return rval, None, self.updatecache_flg, hooks_enabled
 
# need to pass the kwargs dict to the modules
# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-   hooks_enabled = False
if (master_hooks or self.updatecache_flg or
not repo.sync_hooks_only_on_change):
hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: baeff1e7a7306081123d9a31b24c62d59ae73abb
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Nov 22 20:05:37 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Nov 24 16:15:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/sync/controller.py | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
self.usersync_uid = None
spawn_kwargs = {}
+   # Redirect command stderr to stdout, in order to prevent
+   # spurious cron job emails (bug 566132).
+   spawn_kwargs["fd_pipes"] = {
+   0: sys.__stdin__.fileno(),
+   1: sys.__stdout__.fileno(),
+   2: sys.__stdout__.fileno()
+   }
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4205ec912aebf2e3b0bd673fcacb576b1f344329
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct  5 23:01:49 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct  6 16:25:37 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 39 +++
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
-   proc = AsyncFunction(target=self.sync,
-   kwargs=dict(emerge_config=emerge_config, repo=repo))
-   proc.addExitListener(self._sync_callback)
-   return proc
+   return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+   kwargs=dict(emerge_config=emerge_config, repo=repo)),
+   sync_callback=self._sync_callback)
 
def sync(self, emerge_config=None, repo=None):
self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
action_metadata(self.settings, self.portdb, 
self.emerge_config.opts,
porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+   """
+   Encapsulates a sync operation and the callback which executes 
afterwards,
+   so both can be considered as a single composite task. This is useful
+   since we don't want to consider a particular repo's sync operation as
+   complete until after the callback has executed (bug 562264).
+
+   The kwargs and result properties expose attributes that are accessed
+   by SyncScheduler.
+   """
+
+   __slots__ = ('sync_task', 'sync_callback')
+
+   @property
+   def kwargs(self):
+   return self.sync_task.kwargs
+
+   @property
+   def result(self):
+   return self.sync_task.result
+
+   def _start(self):
+   self._start_task(self.sync_task, self._sync_task_exit)
+
+   def _sync_task_exit(self, sync_task):
+   self._current_task = None
+   self.returncode = sync_task.returncode
+   self.sync_callback(self.sync_task)
+   self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec 12 22:24:04 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Dec 12 23:40:41 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
self.repo = repo
self.exitcode = 1
self.updatecache_flg = False
+   hooks_enabled = master_hooks or not 
repo.sync_hooks_only_on_change
if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \
% (bad("ERROR"), repo.sync_type)
-   return self.exitcode, msg
+   return self.exitcode, msg, self.updatecache_flg, 
hooks_enabled
 
rval = self.pre_sync(repo)
if rval != os.EX_OK:
-   return rval, None
+   return rval, None, self.updatecache_flg, hooks_enabled
 
# need to pass the kwargs dict to the modules
# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-   hooks_enabled = False
if (master_hooks or self.updatecache_flg or
not repo.sync_hooks_only_on_change):
hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: baeff1e7a7306081123d9a31b24c62d59ae73abb
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Nov 22 20:05:37 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Nov 24 16:15:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/sync/controller.py | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
self.usersync_uid = None
spawn_kwargs = {}
+   # Redirect command stderr to stdout, in order to prevent
+   # spurious cron job emails (bug 566132).
+   spawn_kwargs["fd_pipes"] = {
+   0: sys.__stdin__.fileno(),
+   1: sys.__stdout__.fileno(),
+   2: sys.__stdout__.fileno()
+   }
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4205ec912aebf2e3b0bd673fcacb576b1f344329
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct  5 23:01:49 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct  6 16:25:37 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 39 +++
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
-   proc = AsyncFunction(target=self.sync,
-   kwargs=dict(emerge_config=emerge_config, repo=repo))
-   proc.addExitListener(self._sync_callback)
-   return proc
+   return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+   kwargs=dict(emerge_config=emerge_config, repo=repo)),
+   sync_callback=self._sync_callback)
 
def sync(self, emerge_config=None, repo=None):
self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
action_metadata(self.settings, self.portdb, 
self.emerge_config.opts,
porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+   """
+   Encapsulates a sync operation and the callback which executes 
afterwards,
+   so both can be considered as a single composite task. This is useful
+   since we don't want to consider a particular repo's sync operation as
+   complete until after the callback has executed (bug 562264).
+
+   The kwargs and result properties expose attributes that are accessed
+   by SyncScheduler.
+   """
+
+   __slots__ = ('sync_task', 'sync_callback')
+
+   @property
+   def kwargs(self):
+   return self.sync_task.kwargs
+
+   @property
+   def result(self):
+   return self.sync_task.result
+
+   def _start(self):
+   self._start_task(self.sync_task, self._sync_task_exit)
+
+   def _sync_task_exit(self, sync_task):
+   self._current_task = None
+   self.returncode = sync_task.returncode
+   self.sync_callback(self.sync_task)
+   self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 200d876b01dd11521cdcd9bfa07abdca165d24e8
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Sep 23 16:16:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Sep 23 22:31:36 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
return []
 
def async(self, emerge_config=None, repo=None):
+   self.emerge_config = emerge_config
+   self.settings, self.trees, self.mtimedb = emerge_config
+   self.xterm_titles = "notitles" not in self.settings.features
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
proc = AsyncFunction(target=self.sync,
kwargs=dict(emerge_config=emerge_config, repo=repo))
proc.addExitListener(self._sync_callback)
return proc
 
def sync(self, emerge_config=None, repo=None):
-   self.emerge_config = emerge_config
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
def pre_sync(self, repo):
-   self.settings, self.trees, self.mtimedb = self.emerge_config
-   self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
-   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:43:47 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny  gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:51:01 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's 
parameters.
But it was not defined when it was moved into task_opts dictionary to be passed 
to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
task_opts = {
'emerge_config': emerge_config,
'logger': self.logger,
-   'portdb': 
self.trees[self.settings['EROOT']]['porttree'].dbapi,
+   'portdb': self.portdb,
'repo': repo,
'settings': self.settings,
'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 7a16988f4765963896170b92253ebbd4344216ea
Author: Brian Dolbec  gentoo  org>
AuthorDate: Wed Apr 22 16:58:38 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Wed Apr 22 16:58:38 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the 
native portage one
causes problems.  This is due to the fact the portage-utils supplied script did 
not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
writemsg_level("Spawning post_sync hook: %s\n"
% (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
-   retval = portage.process.spawn([filepath,
-   reponame, dosyncuri, repolocation], 
env=self.settings.environ())
+   if reponame:
+   retval = portage.process.spawn(
+   [filepath, reponame, dosyncuri, 
repolocation],
+   env=self.settings.environ())
+   else:
+   retval = portage.process.spawn([filepath],
+   env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
_unicode_decode(_hooks[filepath]), 
filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 768b9e920fe0919d87537665ae3ce88007630cc1
Author: Michał Górny  gentoo  org>
AuthorDate: Sat Jan 17 13:03:53 2015 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Jan 18 18:04:25 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
self.callback = callback or self._sync_callback
self.repo = repo
self.exitcode = 1
-   if repo.sync_type in self.module_names[1:]:
+   if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: c0e1d514c1c8159aa732d23a527640432a9c076d
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Jul 14 20:40:23 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Jul 14 21:28:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing 
sync modules.

 pym/portage/sync/__init__.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
global module_controller
 
if repo.sync_type:
-   opts = frozenset([opt for opt in
-   
module_controller.modules[repo.sync_type]['module_specific_options']])
-   return opts
+   try:
+   return frozenset(
+   
module_controller.modules[repo.sync_type]['module_specific_options'])
+   except KeyError:
+   pass
return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug 29 19:56:50 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 30 23:50:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-   ('glsa', 'metadata/glsa'),
-   ('news', 'metadata/news'),
-   ('profiles', 'profiles'),
+   ('glsa', ('metadata/glsa',)),
+   ('news', ('metadata/news',)),
+   ('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
emerge_config = self.options.get('emerge_config')
if emerge_config is not None:
for name in emerge_config.opts.get('--sync-submodule', 
[]):
-   paths.append(_SUBMODULE_PATH_MAP[name])
+   paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sat Dec  6 22:54:36 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 05:22:51 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d 
directory.
This also maintains compatibility with existing scripts in the postsync.d dir 
or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
self.module_controller = portage.sync.module_controller
self.module_names = self.module_controller.module_names
-   postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-   portage.USER_CONFIG_PATH, "postsync.d")
-   hooks = []
-   for root, dirs, names in os.walk(postsync_dir, topdown=True):
-   #print("root:", root, "dirs:", dirs, "names:", names)
-   for name in names:
-   filepath = os.path.join(root, name)
+   self.hooks = {}
+   for _dir in ["repo.postsync.d", "postsync.d"]:
+   postsync_dir = 
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+   portage.USER_CONFIG_PATH, _dir)
+   hooks = OrderedDict()
+   for filepath in util._recursive_file_list(postsync_dir):
+   name = 
filepath.split(postsync_dir)[1].lstrip(os.sep)
if os.access(filepath, os.X_OK):
-   hooks.append((filepath, name))
+   hooks[filepath] = name
else:
-   writemsg_level(" %s postsync.d hook: 
'%s' is not executable\n"
-   % (warn("*"), 
_unicode_decode(name),), level=logging.WARN, noiselevel=2)
-   self.hooks = hooks
+   writemsg_level(" %s %s hook: '%s' is 
not executable\n"
+   % (warn("*"), _dir, 
_unicode_decode(name),),
+   level=logging.WARN, 
noiselevel=2)
+   self.hooks[_dir] = hooks
+   print(self.hooks)
 
 
def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
def perform_post_sync_hook(self, reponame, dosyncuri='', 
repolocation=''):
succeeded = os.EX_OK
-   for filepath, hook in self.hooks:
+   if reponame:
+   _hooks = self.hooks["repo.postsync.d"]
+   else:
+   _hooks = self.hooks["postsync.d"]
+   for filepath in _hooks:
writemsg_level("Spawning post_sync hook: %s\n"
-   % (_unicode_decode(hook)),
+   % (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
retval = portage.process.spawn([filepath,
reponame, dosyncuri, repolocation], 
env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
-   _unicode_decode(hook), filepath),
+   _unicode_decode(_hooks[filepath]), 
filepath),
level=logging.ERROR, noiselevel=-1)
succeeded = retval
return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 8a5ee0a68764a272c554ed534fcb252e81f89118
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sun Dec  7 09:12:13 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 09:12:49 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
% (warn("*"), _dir, 
_unicode_decode(name),),
level=logging.WARN, 
noiselevel=2)
self.hooks[_dir] = hooks
-   print(self.hooks)
 
 
def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 9a2b1e6096e35508e9f101fa0478101493ef0335
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Dec  5 22:54:27 2014 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Dec  7 22:57:07 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++--
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
st = os.stat(repo.location)
except OSError:
st = None
-   if st is None:
-   writemsg_level(">>> '%s' not found, creating it."
-   % _unicode_decode(repo.location))
-   portage.util.ensure_dirs(repo.location, mode=0o755)
-   st = os.stat(repo.location)
 
self.usersync_uid = None
spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
-   elif ('usersync' in self.settings.features and
+
+   if st is None:
+   perms = {'mode': 0o755}
+   # respect sync-user if set
+   if 'umask' in spawn_kwargs:
+   perms['mode'] &= ~spawn_kwargs['umask']
+   if 'uid' in spawn_kwargs:
+   perms['uid'] = spawn_kwargs['uid']
+   if 'gid' in spawn_kwargs:
+   perms['gid'] = spawn_kwargs['gid']
+
+   writemsg_level(">>> '%s' not found, creating it."
+   % _unicode_decode(repo.location))
+   portage.util.ensure_dirs(repo.location, **perms)
+   st = os.stat(repo.location)
+
+   if (repo.sync_user is None and
+   'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sat Dec  6 22:54:36 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 05:22:51 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d 
directory.
This also maintains compatibility with existing scripts in the postsync.d dir 
or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
self.module_controller = portage.sync.module_controller
self.module_names = self.module_controller.module_names
-   postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-   portage.USER_CONFIG_PATH, "postsync.d")
-   hooks = []
-   for root, dirs, names in os.walk(postsync_dir, topdown=True):
-   #print("root:", root, "dirs:", dirs, "names:", names)
-   for name in names:
-   filepath = os.path.join(root, name)
+   self.hooks = {}
+   for _dir in ["repo.postsync.d", "postsync.d"]:
+   postsync_dir = 
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+   portage.USER_CONFIG_PATH, _dir)
+   hooks = OrderedDict()
+   for filepath in util._recursive_file_list(postsync_dir):
+   name = 
filepath.split(postsync_dir)[1].lstrip(os.sep)
if os.access(filepath, os.X_OK):
-   hooks.append((filepath, name))
+   hooks[filepath] = name
else:
-   writemsg_level(" %s postsync.d hook: 
'%s' is not executable\n"
-   % (warn("*"), 
_unicode_decode(name),), level=logging.WARN, noiselevel=2)
-   self.hooks = hooks
+   writemsg_level(" %s %s hook: '%s' is 
not executable\n"
+   % (warn("*"), _dir, 
_unicode_decode(name),),
+   level=logging.WARN, 
noiselevel=2)
+   self.hooks[_dir] = hooks
+   print(self.hooks)
 
 
def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
def perform_post_sync_hook(self, reponame, dosyncuri='', 
repolocation=''):
succeeded = os.EX_OK
-   for filepath, hook in self.hooks:
+   if reponame:
+   _hooks = self.hooks["repo.postsync.d"]
+   else:
+   _hooks = self.hooks["postsync.d"]
+   for filepath in _hooks:
writemsg_level("Spawning post_sync hook: %s\n"
-   % (_unicode_decode(hook)),
+   % (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
retval = portage.process.spawn([filepath,
reponame, dosyncuri, repolocation], 
env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
-   _unicode_decode(hook), filepath),
+   _unicode_decode(_hooks[filepath]), 
filepath),
level=logging.ERROR, noiselevel=-1)
succeeded = retval
return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 8a5ee0a68764a272c554ed534fcb252e81f89118
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sun Dec  7 09:12:13 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 09:12:49 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
% (warn("*"), _dir, 
_unicode_decode(name),),
level=logging.WARN, 
noiselevel=2)
self.hooks[_dir] = hooks
-   print(self.hooks)
 
 
def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 9a2b1e6096e35508e9f101fa0478101493ef0335
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Dec  5 22:54:27 2014 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Dec  7 22:57:07 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++--
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
st = os.stat(repo.location)
except OSError:
st = None
-   if st is None:
-   writemsg_level(">>> '%s' not found, creating it."
-   % _unicode_decode(repo.location))
-   portage.util.ensure_dirs(repo.location, mode=0o755)
-   st = os.stat(repo.location)
 
self.usersync_uid = None
spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
-   elif ('usersync' in self.settings.features and
+
+   if st is None:
+   perms = {'mode': 0o755}
+   # respect sync-user if set
+   if 'umask' in spawn_kwargs:
+   perms['mode'] &= ~spawn_kwargs['umask']
+   if 'uid' in spawn_kwargs:
+   perms['uid'] = spawn_kwargs['uid']
+   if 'gid' in spawn_kwargs:
+   perms['gid'] = spawn_kwargs['gid']
+
+   writemsg_level(">>> '%s' not found, creating it."
+   % _unicode_decode(repo.location))
+   portage.util.ensure_dirs(repo.location, **perms)
+   st = os.stat(repo.location)
+
+   if (repo.sync_user is None and
+   'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:43:47 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny  gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:51:01 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's 
parameters.
But it was not defined when it was moved into task_opts dictionary to be passed 
to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
task_opts = {
'emerge_config': emerge_config,
'logger': self.logger,
-   'portdb': 
self.trees[self.settings['EROOT']]['porttree'].dbapi,
+   'portdb': self.portdb,
'repo': repo,
'settings': self.settings,
'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 7a16988f4765963896170b92253ebbd4344216ea
Author: Brian Dolbec  gentoo  org>
AuthorDate: Wed Apr 22 16:58:38 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Wed Apr 22 16:58:38 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the 
native portage one
causes problems.  This is due to the fact the portage-utils supplied script did 
not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
writemsg_level("Spawning post_sync hook: %s\n"
% (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
-   retval = portage.process.spawn([filepath,
-   reponame, dosyncuri, repolocation], 
env=self.settings.environ())
+   if reponame:
+   retval = portage.process.spawn(
+   [filepath, reponame, dosyncuri, 
repolocation],
+   env=self.settings.environ())
+   else:
+   retval = portage.process.spawn([filepath],
+   env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
_unicode_decode(_hooks[filepath]), 
filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 768b9e920fe0919d87537665ae3ce88007630cc1
Author: Michał Górny  gentoo  org>
AuthorDate: Sat Jan 17 13:03:53 2015 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Jan 18 18:04:25 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
self.callback = callback or self._sync_callback
self.repo = repo
self.exitcode = 1
-   if repo.sync_type in self.module_names[1:]:
+   if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: c0e1d514c1c8159aa732d23a527640432a9c076d
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Jul 14 20:40:23 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Jul 14 21:28:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing 
sync modules.

 pym/portage/sync/__init__.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
global module_controller
 
if repo.sync_type:
-   opts = frozenset([opt for opt in
-   
module_controller.modules[repo.sync_type]['module_specific_options']])
-   return opts
+   try:
+   return frozenset(
+   
module_controller.modules[repo.sync_type]['module_specific_options'])
+   except KeyError:
+   pass
return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec 12 22:24:04 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Dec 12 23:40:41 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
self.repo = repo
self.exitcode = 1
self.updatecache_flg = False
+   hooks_enabled = master_hooks or not 
repo.sync_hooks_only_on_change
if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \
% (bad("ERROR"), repo.sync_type)
-   return self.exitcode, msg
+   return self.exitcode, msg, self.updatecache_flg, 
hooks_enabled
 
rval = self.pre_sync(repo)
if rval != os.EX_OK:
-   return rval, None
+   return rval, None, self.updatecache_flg, hooks_enabled
 
# need to pass the kwargs dict to the modules
# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-   hooks_enabled = False
if (master_hooks or self.updatecache_flg or
not repo.sync_hooks_only_on_change):
hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug 29 19:56:50 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 30 23:50:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-   ('glsa', 'metadata/glsa'),
-   ('news', 'metadata/news'),
-   ('profiles', 'profiles'),
+   ('glsa', ('metadata/glsa',)),
+   ('news', ('metadata/news',)),
+   ('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
emerge_config = self.options.get('emerge_config')
if emerge_config is not None:
for name in emerge_config.opts.get('--sync-submodule', 
[]):
-   paths.append(_SUBMODULE_PATH_MAP[name])
+   paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: baeff1e7a7306081123d9a31b24c62d59ae73abb
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Nov 22 20:05:37 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Nov 24 16:15:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/sync/controller.py | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
self.usersync_uid = None
spawn_kwargs = {}
+   # Redirect command stderr to stdout, in order to prevent
+   # spurious cron job emails (bug 566132).
+   spawn_kwargs["fd_pipes"] = {
+   0: sys.__stdin__.fileno(),
+   1: sys.__stdout__.fileno(),
+   2: sys.__stdout__.fileno()
+   }
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4205ec912aebf2e3b0bd673fcacb576b1f344329
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct  5 23:01:49 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct  6 16:25:37 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 39 +++
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
-   proc = AsyncFunction(target=self.sync,
-   kwargs=dict(emerge_config=emerge_config, repo=repo))
-   proc.addExitListener(self._sync_callback)
-   return proc
+   return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+   kwargs=dict(emerge_config=emerge_config, repo=repo)),
+   sync_callback=self._sync_callback)
 
def sync(self, emerge_config=None, repo=None):
self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
action_metadata(self.settings, self.portdb, 
self.emerge_config.opts,
porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+   """
+   Encapsulates a sync operation and the callback which executes 
afterwards,
+   so both can be considered as a single composite task. This is useful
+   since we don't want to consider a particular repo's sync operation as
+   complete until after the callback has executed (bug 562264).
+
+   The kwargs and result properties expose attributes that are accessed
+   by SyncScheduler.
+   """
+
+   __slots__ = ('sync_task', 'sync_callback')
+
+   @property
+   def kwargs(self):
+   return self.sync_task.kwargs
+
+   @property
+   def result(self):
+   return self.sync_task.result
+
+   def _start(self):
+   self._start_task(self.sync_task, self._sync_task_exit)
+
+   def _sync_task_exit(self, sync_task):
+   self._current_task = None
+   self.returncode = sync_task.returncode
+   self.sync_callback(self.sync_task)
+   self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 200d876b01dd11521cdcd9bfa07abdca165d24e8
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Sep 23 16:16:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Sep 23 22:31:36 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
return []
 
def async(self, emerge_config=None, repo=None):
+   self.emerge_config = emerge_config
+   self.settings, self.trees, self.mtimedb = emerge_config
+   self.xterm_titles = "notitles" not in self.settings.features
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
proc = AsyncFunction(target=self.sync,
kwargs=dict(emerge_config=emerge_config, repo=repo))
proc.addExitListener(self._sync_callback)
return proc
 
def sync(self, emerge_config=None, repo=None):
-   self.emerge_config = emerge_config
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
def pre_sync(self, repo):
-   self.settings, self.trees, self.mtimedb = self.emerge_config
-   self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
-   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sat Dec  6 22:54:36 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 05:22:51 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d 
directory.
This also maintains compatibility with existing scripts in the postsync.d dir 
or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++--
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
self.module_controller = portage.sync.module_controller
self.module_names = self.module_controller.module_names
-   postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-   portage.USER_CONFIG_PATH, "postsync.d")
-   hooks = []
-   for root, dirs, names in os.walk(postsync_dir, topdown=True):
-   #print("root:", root, "dirs:", dirs, "names:", names)
-   for name in names:
-   filepath = os.path.join(root, name)
+   self.hooks = {}
+   for _dir in ["repo.postsync.d", "postsync.d"]:
+   postsync_dir = 
os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+   portage.USER_CONFIG_PATH, _dir)
+   hooks = OrderedDict()
+   for filepath in util._recursive_file_list(postsync_dir):
+   name = 
filepath.split(postsync_dir)[1].lstrip(os.sep)
if os.access(filepath, os.X_OK):
-   hooks.append((filepath, name))
+   hooks[filepath] = name
else:
-   writemsg_level(" %s postsync.d hook: 
'%s' is not executable\n"
-   % (warn("*"), 
_unicode_decode(name),), level=logging.WARN, noiselevel=2)
-   self.hooks = hooks
+   writemsg_level(" %s %s hook: '%s' is 
not executable\n"
+   % (warn("*"), _dir, 
_unicode_decode(name),),
+   level=logging.WARN, 
noiselevel=2)
+   self.hooks[_dir] = hooks
+   print(self.hooks)
 
 
def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
def perform_post_sync_hook(self, reponame, dosyncuri='', 
repolocation=''):
succeeded = os.EX_OK
-   for filepath, hook in self.hooks:
+   if reponame:
+   _hooks = self.hooks["repo.postsync.d"]
+   else:
+   _hooks = self.hooks["postsync.d"]
+   for filepath in _hooks:
writemsg_level("Spawning post_sync hook: %s\n"
-   % (_unicode_decode(hook)),
+   % (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
retval = portage.process.spawn([filepath,
reponame, dosyncuri, repolocation], 
env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
-   _unicode_decode(hook), filepath),
+   _unicode_decode(_hooks[filepath]), 
filepath),
level=logging.ERROR, noiselevel=-1)
succeeded = retval
return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 8a5ee0a68764a272c554ed534fcb252e81f89118
Author: Brian Dolbec  gentoo  org>
AuthorDate: Sun Dec  7 09:12:13 2014 +
Commit: Brian Dolbec  gmail  com>
CommitDate: Sun Dec  7 09:12:49 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec  gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
% (warn("*"), _dir, 
_unicode_decode(name),),
level=logging.WARN, 
noiselevel=2)
self.hooks[_dir] = hooks
-   print(self.hooks)
 
 
def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 9a2b1e6096e35508e9f101fa0478101493ef0335
Author: Michał Górny  gentoo  org>
AuthorDate: Fri Dec  5 22:54:27 2014 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Dec  7 22:57:07 2014 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++--
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
st = os.stat(repo.location)
except OSError:
st = None
-   if st is None:
-   writemsg_level(">>> '%s' not found, creating it."
-   % _unicode_decode(repo.location))
-   portage.util.ensure_dirs(repo.location, mode=0o755)
-   st = os.stat(repo.location)
 
self.usersync_uid = None
spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
spawn_kwargs["groups"] = [gid]
if home is not None:
spawn_kwargs["env"]["HOME"] = home
-   elif ('usersync' in self.settings.features and
+
+   if st is None:
+   perms = {'mode': 0o755}
+   # respect sync-user if set
+   if 'umask' in spawn_kwargs:
+   perms['mode'] &= ~spawn_kwargs['umask']
+   if 'uid' in spawn_kwargs:
+   perms['uid'] = spawn_kwargs['uid']
+   if 'gid' in spawn_kwargs:
+   perms['gid'] = spawn_kwargs['gid']
+
+   writemsg_level(">>> '%s' not found, creating it."
+   % _unicode_decode(repo.location))
+   portage.util.ensure_dirs(repo.location, **perms)
+   st = os.stat(repo.location)
+
+   if (repo.sync_user is None and
+   'usersync' in self.settings.features and
portage.data.secpass >= 2 and
(st.st_uid != os.getuid() and st.st_mode & 0o700 or
st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:43:47 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny  gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author: Brian Dolbec  gentoo  org>
AuthorDate: Mon Feb  9 19:51:01 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Mon Feb  9 20:11:44 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's 
parameters.
But it was not defined when it was moved into task_opts dictionary to be passed 
to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
task_opts = {
'emerge_config': emerge_config,
'logger': self.logger,
-   'portdb': 
self.trees[self.settings['EROOT']]['porttree'].dbapi,
+   'portdb': self.portdb,
'repo': repo,
'settings': self.settings,
'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit: 768b9e920fe0919d87537665ae3ce88007630cc1
Author: Michał Górny  gentoo  org>
AuthorDate: Sat Jan 17 13:03:53 2015 +
Commit: Michał Górny  gentoo  org>
CommitDate: Sun Jan 18 18:04:25 2015 +
URL:
http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
self.callback = callback or self._sync_callback
self.repo = repo
self.exitcode = 1
-   if repo.sync_type in self.module_names[1:]:
+   if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: 7a16988f4765963896170b92253ebbd4344216ea
Author: Brian Dolbec  gentoo  org>
AuthorDate: Wed Apr 22 16:58:38 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Wed Apr 22 16:58:38 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the 
native portage one
causes problems.  This is due to the fact the portage-utils supplied script did 
not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
writemsg_level("Spawning post_sync hook: %s\n"
% (_unicode_decode(_hooks[filepath])),
level=logging.ERROR, noiselevel=4)
-   retval = portage.process.spawn([filepath,
-   reponame, dosyncuri, repolocation], 
env=self.settings.environ())
+   if reponame:
+   retval = portage.process.spawn(
+   [filepath, reponame, dosyncuri, 
repolocation],
+   env=self.settings.environ())
+   else:
+   retval = portage.process.spawn([filepath],
+   env=self.settings.environ())
if retval != os.EX_OK:
writemsg_level(" %s Spawn failed for: %s, %s\n" 
% (bad("*"),
_unicode_decode(_hooks[filepath]), 
filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Dec 12 22:24:04 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Dec 12 23:40:41 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
self.repo = repo
self.exitcode = 1
self.updatecache_flg = False
+   hooks_enabled = master_hooks or not 
repo.sync_hooks_only_on_change
if repo.sync_type in self.module_names:
tasks = 
[self.module_controller.get_class(repo.sync_type)]
else:
msg = "\n%s: Sync module '%s' is not an installed/known 
type'\n" \
% (bad("ERROR"), repo.sync_type)
-   return self.exitcode, msg
+   return self.exitcode, msg, self.updatecache_flg, 
hooks_enabled
 
rval = self.pre_sync(repo)
if rval != os.EX_OK:
-   return rval, None
+   return rval, None, self.updatecache_flg, hooks_enabled
 
# need to pass the kwargs dict to the modules
# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
taskmaster = TaskHandler(callback=self.do_callback)
taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-   hooks_enabled = False
if (master_hooks or self.updatecache_flg or
not repo.sync_hooks_only_on_change):
hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: baeff1e7a7306081123d9a31b24c62d59ae73abb
Author: Zac Medico  gentoo  org>
AuthorDate: Sun Nov 22 20:05:37 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Nov 24 16:15:04 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen  gentoo.org>

 pym/portage/sync/controller.py | 7 +++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
self.usersync_uid = None
spawn_kwargs = {}
+   # Redirect command stderr to stdout, in order to prevent
+   # spurious cron job emails (bug 566132).
+   spawn_kwargs["fd_pipes"] = {
+   0: sys.__stdin__.fileno(),
+   1: sys.__stdout__.fileno(),
+   2: sys.__stdout__.fileno()
+   }
spawn_kwargs["env"] = self.settings.environ()
if repo.sync_user is not None:
def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit: c0e1d514c1c8159aa732d23a527640432a9c076d
Author: Brian Dolbec  gentoo  org>
AuthorDate: Tue Jul 14 20:40:23 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Tue Jul 14 21:28:32 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing 
sync modules.

 pym/portage/sync/__init__.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
global module_controller
 
if repo.sync_type:
-   opts = frozenset([opt for opt in
-   
module_controller.modules[repo.sync_type]['module_specific_options']])
-   return opts
+   try:
+   return frozenset(
+   
module_controller.modules[repo.sync_type]['module_specific_options'])
+   except KeyError:
+   pass
return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Aug 29 19:56:50 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Aug 30 23:50:58 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-   ('glsa', 'metadata/glsa'),
-   ('news', 'metadata/news'),
-   ('profiles', 'profiles'),
+   ('glsa', ('metadata/glsa',)),
+   ('news', ('metadata/news',)),
+   ('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
emerge_config = self.options.get('emerge_config')
if emerge_config is not None:
for name in emerge_config.opts.get('--sync-submodule', 
[]):
-   paths.append(_SUBMODULE_PATH_MAP[name])
+   paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 4205ec912aebf2e3b0bd673fcacb576b1f344329
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Oct  5 23:01:49 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Tue Oct  6 16:25:37 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 39 +++
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
self.settings, self.trees, self.mtimedb = emerge_config
self.xterm_titles = "notitles" not in self.settings.features
self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
-   proc = AsyncFunction(target=self.sync,
-   kwargs=dict(emerge_config=emerge_config, repo=repo))
-   proc.addExitListener(self._sync_callback)
-   return proc
+   return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+   kwargs=dict(emerge_config=emerge_config, repo=repo)),
+   sync_callback=self._sync_callback)
 
def sync(self, emerge_config=None, repo=None):
self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
action_metadata(self.settings, self.portdb, 
self.emerge_config.opts,
porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+   """
+   Encapsulates a sync operation and the callback which executes 
afterwards,
+   so both can be considered as a single composite task. This is useful
+   since we don't want to consider a particular repo's sync operation as
+   complete until after the callback has executed (bug 562264).
+
+   The kwargs and result properties expose attributes that are accessed
+   by SyncScheduler.
+   """
+
+   __slots__ = ('sync_task', 'sync_callback')
+
+   @property
+   def kwargs(self):
+   return self.sync_task.kwargs
+
+   @property
+   def result(self):
+   return self.sync_task.result
+
+   def _start(self):
+   self._start_task(self.sync_task, self._sync_task_exit)
+
+   def _sync_task_exit(self, sync_task):
+   self._current_task = None
+   self.returncode = sync_task.returncode
+   self.sync_callback(self.sync_task)
+   self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 200d876b01dd11521cdcd9bfa07abdca165d24e8
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Sep 23 16:16:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Sep 23 22:31:36 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/controller.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
return []
 
def async(self, emerge_config=None, repo=None):
+   self.emerge_config = emerge_config
+   self.settings, self.trees, self.mtimedb = emerge_config
+   self.xterm_titles = "notitles" not in self.settings.features
+   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
proc = AsyncFunction(target=self.sync,
kwargs=dict(emerge_config=emerge_config, repo=repo))
proc.addExitListener(self._sync_callback)
return proc
 
def sync(self, emerge_config=None, repo=None):
-   self.emerge_config = emerge_config
self.callback = None
self.repo = repo
self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
def pre_sync(self, repo):
-   self.settings, self.trees, self.mtimedb = self.emerge_config
-   self.xterm_titles = "notitles" not in self.settings.features
msg = ">>> Syncing repository '%s' into '%s'..." \
% (repo.name, repo.location)
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n")
-   self.portdb = 
self.trees[self.settings['EROOT']]['porttree'].dbapi
try:
st = os.stat(repo.location)
except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit: 0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author: Nicolas Porcel  gmail  com>
AuthorDate: Sun Sep 17 23:29:27 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sun Sep 17 23:59:11 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
paths.extend(_SUBMODULE_PATH_MAP[name])
return tuple(paths)
 
+   def retrieve_head(self, **kwargs):
+   '''Get information about the head commit'''
+   raise NotImplementedError
+
 
 class NewBase(SyncBase):
'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
'''Update existing repository
'''
raise NotImplementedError
-
-   def retrieve_head(self, **kwargs):
-   '''Get information about the head commit'''
-   raise NotImplementedError

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit: c7c04b4f4f4f5e6f18ad76366b535dcbad72989e
Author: Zac Medico  gentoo  org>
AuthorDate: Fri Oct 27 19:40:01 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Fri Oct 27 19:42:44 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c7c04b4f

RsyncSync: fix PORTAGE_RSYNC_RETRIES (bug 497596)

When PORTAGE_RSYNC_RETRIES is set to a positive integer, recycle
the uris until the specified number of retries has been exhausted.

Bug: https://bugs.gentoo.org/497596

 pym/portage/sync/modules/rsync/rsync.py | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 45a70e7dd..01e4e5924 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -202,6 +202,7 @@ class RsyncSync(NewBase):
 
# reverse, for use with pop()
uris.reverse()
+   uris_orig = uris[:]
 
effective_maxretries = maxretries
if effective_maxretries < 0:
@@ -210,10 +211,13 @@ class RsyncSync(NewBase):
while (1):
if uris:
dosyncuri = uris.pop()
-   else:
+   elif maxretries < 0 or retries > maxretries:
writemsg("!!! Exhausted addresses for %s\n"
% _unicode_decode(hostname), 
noiselevel=-1)
return (1, False)
+   else:
+   uris.extend(uris_orig)
+   dosyncuri = uris.pop()
 
if (retries==0):
if "--ask" in opts:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Zac Medico]

commit: 540b97e416371c03e681625dd02a8f0eb371e9ab
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Aug  9 20:38:08 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Aug  9 20:39:26 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=540b97e4

GitSync: fix subprocess.CalledProcessError reference (bug 627416)

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/modules/git/git.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 8df9ca612..19c167485 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -139,6 +139,6 @@ class GitSync(NewBase):
try:
ret = (os.EX_OK, subprocess.check_output(rev_cmd,

cwd=portage._unicode_encode(self.repo.location)))
-   except CalledProcessError:
+   except subprocess.CalledProcessError:
ret = (1, False)
return ret

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Zac Medico]

commit: 2d0cefbcbf9612538171c581485c290f140b57ff
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Aug  9 20:54:48 2017 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Aug  9 20:56:24 2017 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d0cefbc

GitSync.retrieve_head: return str, not bytes (bug 625888)

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/modules/git/git.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 19c167485..8068149c7 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -137,8 +137,9 @@ class GitSync(NewBase):
self._kwargs(kwargs)
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", 
"HEAD"]
try:
-   ret = (os.EX_OK, subprocess.check_output(rev_cmd,
-   
cwd=portage._unicode_encode(self.repo.location)))
+   ret = (os.EX_OK,
+   
portage._unicode_decode(subprocess.check_output(rev_cmd,
+   
cwd=portage._unicode_encode(self.repo.location
except subprocess.CalledProcessError:
ret = (1, False)
return ret

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Brian Dolbec]

commit: 3346d3c1c1f6c2702d7d0d347370bee816a82c2f
Author: Consus  gmx  com>
AuthorDate: Fri Sep  4 10:20:27 2015 +
Commit: Brian Dolbec  gentoo  org>
CommitDate: Fri Sep  4 17:25:24 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3346d3c1

git sync: Respect PORTAGE_QUIET

Execute `git clone --quiet' and `git pull --quiet' when appropriate.

 pym/portage/sync/modules/git/git.py | 20 +---
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 7a710ef..c14782c 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -1,4 +1,4 @@
-# Copyright 2005-2014 Gentoo Foundation
+# Copyright 2005-2015 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
@@ -43,15 +43,18 @@ class GitSync(NewBase):
'Created new directory %s' % 
self.repo.location)
except IOError:
return (1, False)
+
sync_uri = self.repo.sync_uri
if sync_uri.startswith("file://"):
sync_uri = sync_uri[6:]
-   depth_arg = ''
-   if self.repo.sync_depth is not None:
-   depth_arg = '--depth %d ' % self.repo.sync_depth
 
-   git_cmd = "%s clone %s%s ." % (self.bin_command, depth_arg,
-   portage._shell_quote(sync_uri))
+   git_cmd_opts = ""
+   if self.settings.get("PORTAGE_QUIET") == "1":
+   git_cmd_opts += " --quiet"
+   if self.repo.sync_depth is not None:
+   git_cmd_opts += " --depth %d" % self.repo.sync_depth
+   git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
+   portage._shell_quote(sync_uri))
writemsg_level(git_cmd + "\n")
 
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
@@ -72,7 +75,10 @@ class GitSync(NewBase):
git directly.
'''
 
-   git_cmd = "%s pull" % self.bin_command
+   git_cmd_opts = ""
+   if self.settings.get("PORTAGE_QUIET") == "1":
+   git_cmd_opts += " --quiet"
+   git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
 
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/cvs/

[Arfrever Frehtes Taifersar Arahesis]

commit: 628aa100a480dfcb5fe0edeb1fcd6ea6f4b933a2
Author: Arfrever Frehtes Taifersar Arahesis  Apache  Org>
AuthorDate: Tue Nov 24 02:43:28 2015 +
Commit: Arfrever Frehtes Taifersar Arahesis  apache  org>
CommitDate: Tue Nov 24 02:43:28 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=628aa100

portage.sync.modules.cvs.CheckCVSConfig.check_cvs_repo(): Fix "KeyError: 
'sync-cvs-repo'".

 pym/portage/sync/modules/cvs/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/cvs/__init__.py 
b/pym/portage/sync/modules/cvs/__init__.py
index 952053a..0cf6067 100644
--- a/pym/portage/sync/modules/cvs/__init__.py
+++ b/pym/portage/sync/modules/cvs/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2014 Gentoo Foundation
+# Copyright 2014-2015 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 doc = """CVS plug-in module for portage.
@@ -18,7 +18,7 @@ class CheckCVSConfig(CheckSyncConfig):
 
 
def check_cvs_repo(self):
-   if self.repo.module_specific_options['sync-cvs-repo'] is None:
+   if self.repo.module_specific_options.get('sync-cvs-repo') is 
None:
writemsg_level("!!! %s\n" %
_("Repository '%s' has sync-type=cvs, but is 
missing sync-cvs-repo attribute")
% self.repo.name, level=self.logger.ERROR, 
noiselevel=-1)

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit: d984cea6ee08799f003b189e934f696b483e101a
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Nov  7 04:49:45 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Nov  7 18:14:02 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d984cea6

RsyncSync: skip metadata-transfer when appropriate (bug 564988)

Fix flaws in logic involving the updatecache_flg variable, in order
to skip metadata-transfer when sync fails (or the server timestamp
has not changed).

X-Gentoo-Bug: 564988
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=564988
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 17 ++---
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 8ae8a5c..e0f76b3 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -112,10 +112,10 @@ class RsyncSync(NewBase):
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[6:]
-   is_synced, exitcode = self._do_rsync(
+   is_synced, exitcode, updatecache_flg = self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
-   return (exitcode, exitcode == os.EX_OK)
+   return (exitcode, updatecache_flg)
 
retries=0
try:
@@ -138,7 +138,7 @@ class RsyncSync(NewBase):
else:
# getaddrinfo needs the brackets stripped
getaddrinfo_host = hostname[1:-1]
-   updatecache_flg=True
+   updatecache_flg = False
all_rsync_opts = set(self.rsync_opts)
all_rsync_opts.update(self.extra_rsync_opts)
 
@@ -240,7 +240,8 @@ class RsyncSync(NewBase):
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-   is_synced, exitcode = self._do_rsync(dosyncuri, 
timestamp, opts)
+   is_synced, exitcode, updatecache_flg = self._do_rsync(
+   dosyncuri, timestamp, opts)
if is_synced:
break
 
@@ -251,7 +252,6 @@ class RsyncSync(NewBase):
else:
# over retries
# exit loop
-   updatecache_flg=False
exitcode = EXCEEDED_MAX_RETRIES
break
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
@@ -382,6 +382,7 @@ class RsyncSync(NewBase):
 
 
def _do_rsync(self, syncuri, timestamp, opts):
+   updatecache_flg = False
is_synced = False
if timestamp != 0 and "--quiet" not in opts:
print(">>> Checking server timestamp ...")
@@ -489,7 +490,7 @@ class RsyncSync(NewBase):
print(">>> In order to force sync, remove 
'%s'." % self.servertimestampfile)
print(">>>")
print()
-   return is_synced, exitcode
+   return is_synced, exitcode, updatecache_flg
elif (servertimestamp != 0) and (servertimestamp < 
timestamp):
self.logger(self.xterm_titles,
">>> Server out of date: %s" % syncuri)
@@ -543,6 +544,8 @@ class RsyncSync(NewBase):

os.unlink(self.servertimestampfile)
except OSError:
pass
+   else:
+   updatecache_flg = True
 
if exitcode in [0,1,3,4,11,14,20,21]:
is_synced = True
@@ -554,4 +557,4 @@ class RsyncSync(NewBase):
# --prune-empty-directories.  Retry for a server that 
supports
# at least rsync protocol version 29 (>=rsync-2.6.4).
pass
-   return is_synced, exitcode
+   return is_synced, exitcode, updatecache_flg

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Zac Medico]

commit: 9921cfeb51ab8d9dc128aa5e927d29fc675b28b4
Author: Zac Medico  gentoo  org>
AuthorDate: Sat Nov  7 20:43:59 2015 +
Commit: Zac Medico  gentoo  org>
CommitDate: Sat Nov  7 21:27:01 2015 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9921cfeb

GitSync: skip metadata-transfer when appropriate (bug 564988)

Set updatecache_flg to False if the git revision is unchanged.

X-Gentoo-Bug: 564988
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=564988
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/modules/git/git.py | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index c14782c..179c0de 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
+import subprocess
 
 import portage
 from portage import os
@@ -81,6 +82,10 @@ class GitSync(NewBase):
git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
writemsg_level(git_cmd + "\n")
 
+   rev_cmd = [self.bin_command, "rev-list", "--max-count=1", 
"HEAD"]
+   previous_rev = subprocess.check_output(rev_cmd,
+   cwd=portage._unicode_encode(self.repo.location))
+
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), 
git_cmd),
**portage._native_kwargs(self.spawn_kwargs))
@@ -89,4 +94,8 @@ class GitSync(NewBase):
self.logger(self.xterm_titles, msg)
writemsg_level(msg + "\n", level=logging.ERROR, 
noiselevel=-1)
return (exitcode, False)
-   return (os.EX_OK, True)
+
+   current_rev = subprocess.check_output(rev_cmd,
+   cwd=portage._unicode_encode(self.repo.location))
+
+   return (os.EX_OK, current_rev != previous_rev)

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Zac Medico]

commit: 84413bb1dd9df322568ce25efc5b7854a43d03c7
Author: Zac Medico  gentoo  org>
AuthorDate: Thu Jul 14 08:23:46 2016 +
Commit: Zac Medico  gentoo  org>
CommitDate: Thu Jul 14 18:31:34 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=84413bb1

GitSync.update: respect sync-depth (bug 552814)

Fix updates to respect sync-depth (previously it was only respected
for clone operations). Since the default merge strategy typically
fails when the the depth is not unlimited, use `git fetch` followed
by `git reset --hard`.

X-Gentoo-Bug: 552814
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=552814
Acked-by: Brian Dolbec  gentoo.org>

 pym/portage/sync/modules/git/git.py | 27 +--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 02eeb16..09257f3 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -79,11 +79,26 @@ class GitSync(NewBase):
'''
 
git_cmd_opts = ""
-   if self.settings.get("PORTAGE_QUIET") == "1":
+   quiet = self.settings.get("PORTAGE_QUIET") == "1"
+   if quiet:
git_cmd_opts += " --quiet"
if 
self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
git_cmd_opts += " %s" % 
self.repo.module_specific_options['sync-git-pull-extra-opts']
-   git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
+   if self.repo.sync_depth is None:
+   git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
+   else:
+   # Since the default merge strategy typically fails when
+   # the depth is not unlimited, use `git fetch` followed 
by
+   # `git reset --hard`.
+   remote_branch = portage._unicode_decode(
+   subprocess.check_output([self.bin_command, 
'rev-parse',
+   '--abbrev-ref', '--symbolic-full-name', 
'@{upstream}'],
+   
cwd=portage._unicode_encode(self.repo.location))).rstrip('\n')
+
+   git_cmd_opts += " --depth %d" % self.repo.sync_depth
+   git_cmd = "%s fetch %s%s" % (self.bin_command,
+   remote_branch.partition('/')[0], git_cmd_opts)
+
writemsg_level(git_cmd + "\n")
 
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", 
"HEAD"]
@@ -93,6 +108,14 @@ class GitSync(NewBase):
exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
portage._shell_quote(self.repo.location), 
git_cmd),
**self.spawn_kwargs)
+
+   if exitcode == os.EX_OK and self.repo.sync_depth is not None:
+   reset_cmd = [self.bin_command, 'reset', '--hard', 
remote_branch]
+   if quiet:
+   reset_cmd.append('--quiet')
+   exitcode = subprocess.call(reset_cmd,
+   cwd=portage._unicode_encode(self.repo.location))
+
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Zac Medico]

commit: 55aef9bf297ef8cbf29921acb454449d01313818
Author: Zac Medico  gentoo  org>
AuthorDate: Mon Jul 18 15:48:35 2016 +
Commit: Zac Medico  gentoo  org>
CommitDate: Mon Jul 18 15:52:42 2016 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=55aef9bf

GitSync.update: use git reset --merge instead of --hard (bug 552814)

Since `git reset --merge` gives the closest behavior to `git pull`,
use it instead of `-git reset --hard`. This will provide the following
advantages:

* git will not have to stat files that have not changed since the
  previous sync, which will allow for optimal performance

* git will abort if there are unstaged local changes to any files that
  have changed since the previous sync

Suggested-by: Michał Górny  gentoo.org>
Fixes 84413bb1dd9d ("GitSync.update: respect sync-depth (bug 552814)")

 pym/portage/sync/modules/git/git.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 09257f3..c1028ab 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -89,7 +89,7 @@ class GitSync(NewBase):
else:
# Since the default merge strategy typically fails when
# the depth is not unlimited, use `git fetch` followed 
by
-   # `git reset --hard`.
+   # `git reset --merge`.
remote_branch = portage._unicode_decode(
subprocess.check_output([self.bin_command, 
'rev-parse',
'--abbrev-ref', '--symbolic-full-name', 
'@{upstream}'],
@@ -110,7 +110,7 @@ class GitSync(NewBase):
**self.spawn_kwargs)
 
if exitcode == os.EX_OK and self.repo.sync_depth is not None:
-   reset_cmd = [self.bin_command, 'reset', '--hard', 
remote_branch]
+   reset_cmd = [self.bin_command, 'reset', '--merge', 
remote_branch]
if quiet:
reset_cmd.append('--quiet')
exitcode = subprocess.call(reset_cmd,

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit: 8b773727f009e63fbaecf937701d7f9f1a97f112
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Jan 31 22:20:55 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jan 31 22:20:55 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8b773727

rsync: handle CommandNotFound for gemato (bug 646184)

Bug: https://bugs.gentoo.org/646184

 pym/portage/sync/modules/rsync/rsync.py | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 7c020a563..69e4be734 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -13,6 +13,7 @@ import tempfile
 import portage
 from portage import os
 from portage import _unicode_decode
+from portage.exception import CommandNotFound
 from portage.util import writemsg_level
 from portage.output import create_color_func, yellow, blue, bold
 good = create_color_func("GOOD")
@@ -277,7 +278,12 @@ class RsyncSync(NewBase):
command += ['-K', self.repo.openpgp_key_path]
if self.verify_jobs is not None:
command += ['-j', self.verify_jobs]
-   exitcode = portage.process.spawn(command, 
**self.spawn_kwargs)
+   try:
+   exitcode = portage.process.spawn(command, 
**self.spawn_kwargs)
+   except CommandNotFound as e:
+   writemsg_level("!!! Command not found: %s\n" % 
(command[0],),
+   level=logging.ERROR, noiselevel=-1)
+   exitcode = 127
 
return (exitcode, updatecache_flg)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Zac Medico]

commit: 5aa69880ca3665e633cb042d3e26f3a72536fc10
Author: Zac Medico  gentoo  org>
AuthorDate: Wed Jan 31 23:37:43 2018 +
Commit: Zac Medico  gentoo  org>
CommitDate: Wed Jan 31 23:37:43 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5aa69880

rsync: if local state is unchanged then skip verification

 pym/portage/sync/modules/rsync/rsync.py | 15 ++-
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 69e4be734..e2e6d0658 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -123,7 +123,7 @@ class RsyncSync(NewBase):
if syncuri.startswith("file://"):
self.proto = "file"
dosyncuri = syncuri[7:]
-   is_synced, exitcode, updatecache_flg = self._do_rsync(
+   unchanged, is_synced, exitcode, updatecache_flg = 
self._do_rsync(
dosyncuri, timestamp, opts)
self._process_exitcode(exitcode, dosyncuri, out, 1)
return (exitcode, updatecache_flg)
@@ -219,6 +219,7 @@ class RsyncSync(NewBase):
if effective_maxretries < 0:
effective_maxretries = len(uris) - 1
 
+   local_state_unchanged = True
while (1):
if uris:
dosyncuri = uris.pop()
@@ -255,8 +256,10 @@ class RsyncSync(NewBase):
if dosyncuri.startswith('ssh://'):
dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-   is_synced, exitcode, updatecache_flg = self._do_rsync(
+   unchanged, is_synced, exitcode, updatecache_flg = 
self._do_rsync(
dosyncuri, timestamp, opts)
+   if not unchanged:
+   local_state_unchanged = False
if is_synced:
break
 
@@ -272,7 +275,7 @@ class RsyncSync(NewBase):
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
 
# if synced successfully, verify now
-   if exitcode == 0 and self.verify_metamanifest:
+   if exitcode == 0 and not local_state_unchanged and 
self.verify_metamanifest:
command = ['gemato', 'verify', '-s', self.repo.location]
if self.repo.openpgp_key_path is not None:
command += ['-K', self.repo.openpgp_key_path]
@@ -437,6 +440,7 @@ class RsyncSync(NewBase):
if "--debug" in opts:
print(rsynccommand)
 
+   local_state_unchanged = False
exitcode = os.EX_OK
servertimestamp = 0
# Even if there's no timestamp available locally, fetch the
@@ -521,6 +525,7 @@ class RsyncSync(NewBase):
 
if exitcode == os.EX_OK:
if (servertimestamp != 0) and (servertimestamp == 
timestamp):
+   local_state_unchanged = True
is_synced = True
self.logger(self.xterm_titles,
">>> Cancelling sync -- Already 
current.")
@@ -532,7 +537,6 @@ class RsyncSync(NewBase):
print(">>> In order to force sync, remove 
'%s'." % self.servertimestampfile)
print(">>>")
print()
-   return is_synced, exitcode, updatecache_flg
elif (servertimestamp != 0) and (servertimestamp < 
timestamp):
self.logger(self.xterm_titles,
">>> Server out of date: %s" % syncuri)
@@ -599,4 +603,5 @@ class RsyncSync(NewBase):
# --prune-empty-directories.  Retry for a server that 
supports
# at least rsync protocol version 29 (>=rsync-2.6.4).
pass
-   return is_synced, exitcode, updatecache_flg
+
+   return local_state_unchanged, is_synced, exitcode, 
updatecache_flg

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/git/

[Michał Górny]

commit: d30191b887bb3a3d896c2b8bbf57571e8821b413
Author: Michał Górny  gentoo  org>
AuthorDate: Thu Feb  1 13:30:46 2018 +
Commit: Michał Górny  gentoo  org>
CommitDate: Mon Feb  5 18:43:35 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d30191b8

git: Support running the verification against sync-openpgp-key-path

Closes: https://github.com/gentoo/portage/pull/252
Reviewed-by: Zac Medico  gentoo.org>

 pym/portage/sync/modules/git/git.py | 101 +---
 1 file changed, 70 insertions(+), 31 deletions(-)

diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index 7e5ddf3b5..cec760d00 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -1,6 +1,7 @@
 # Copyright 2005-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import io
 import logging
 import subprocess
 
@@ -13,6 +14,12 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.sync.syncbase import NewBase
 
+try:
+   from gemato.exceptions import GematoException
+   import gemato.openpgp
+except ImportError:
+   gemato = None
+
 
 class GitSync(NewBase):
'''Git sync class'''
@@ -141,39 +148,71 @@ class GitSync(NewBase):
'sync-git-verify-commit-signature', 'false') != 
'true'):
return True
 
-   rev_cmd = [self.bin_command, "log", "--pretty=format:%G?", "-1"]
-   try:
-   status = (portage._unicode_decode(
-   subprocess.check_output(rev_cmd,
-   
cwd=portage._unicode_encode(self.repo.location)))
-   .strip())
-   except subprocess.CalledProcessError:
-   return False
-
-   out = EOutput()
-   if status == 'G':  # good signature is good
-   out.einfo('Trusted signature found on top commit')
-   return True
-   elif status == 'U':  # untrusted
-   out.ewarn('Top commit signature is valid but not 
trusted')
-   return True
+   if self.repo.sync_openpgp_key_path is not None:
+   if gemato is None:
+   writemsg_level("!!! Verifying against specified 
key requires gemato-11.0+ installed\n",
+   level=logging.ERROR, noiselevel=-1)
+   return False
+   openpgp_env = gemato.openpgp.OpenPGPEnvironment()
else:
-   if status == 'B':
-   expl = 'bad signature'
-   elif status == 'X':
-   expl = 'expired signature'
-   elif status == 'Y':
-   expl = 'expired key'
-   elif status == 'R':
-   expl = 'revoked key'
-   elif status == 'E':
-   expl = 'unable to verify signature (missing 
key?)'
-   elif status == 'N':
-   expl = 'no signature'
+   openpgp_env = None
+
+   try:
+   out = EOutput()
+   env = None
+   if openpgp_env is not None:
+   try:
+   out.einfo('Using keys from %s' % 
(self.repo.sync_openpgp_key_path,))
+   with 
io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+   openpgp_env.import_key(f)
+   out.ebegin('Refreshing keys from 
keyserver')
+   openpgp_env.refresh_keys()
+   out.eend(0)
+   except GematoException as e:
+   writemsg_level("!!! Verification 
impossible due to keyring problem:\n%s\n"
+   % (e,),
+   level=logging.ERROR, 
noiselevel=-1)
+   return (1, False)
+
+   env = os.environ.copy()
+   env['GNUPGHOME'] = openpgp_env.home
+
+   rev_cmd = [self.bin_command, "log", 
"--pretty=format:%G?", "-1"]
+   try:
+   status = (portage._unicode_decode(
+   subprocess.check_output(rev_cmd,
+   
cwd=portage._unicode_encode(self.repo.location),
+   env=env))

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit: a02b5f9f0ae428eeb6cdc30bb3bcb7ac647358de
Author: Michał Górny  gentoo  org>
AuthorDate: Thu Feb  1 11:44:49 2018 +
Commit: Michał Górny  gentoo  org>
CommitDate: Mon Feb  5 18:41:35 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a02b5f9f

rsync: Verify the Manifest signature even if tree is unchanged

Always verify the Manifest signature if verification is enabled.
Skipping the deep tree verification for unchanged case is reasonable
but we need to make sure the Manifest signature stays valid to catch
the case of the signing key being revoked.

Reviewed-by: Zac Medico  gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 13 +
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 39c4066d8..e6e218868 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -292,7 +292,7 @@ class RsyncSync(NewBase):
self._process_exitcode(exitcode, dosyncuri, out, maxretries)
 
# if synced successfully, verify now
-   if exitcode == 0 and not local_state_unchanged and 
self.verify_metamanifest:
+   if exitcode == 0 and self.verify_metamanifest:
if gemato is None:
writemsg_level("!!! Unable to verify: 
gemato-11.0+ is required\n",
level=logging.ERROR, noiselevel=-1)
@@ -315,6 +315,8 @@ class RsyncSync(NewBase):

openpgp_env.refresh_keys()
out.eend(0)
 
+   # we always verify the Manifest 
signature, in case
+   # we had to deal with key 
revocation case
m = 
gemato.recursiveloader.ManifestRecursiveLoader(

os.path.join(self.repo.location, 'Manifest'),

verify_openpgp=True,
@@ -336,9 +338,12 @@ class RsyncSync(NewBase):
out.einfo('- timestamp: %s UTC' 
% (

m.openpgp_signature.timestamp))
 
-   out.ebegin('Verifying %s' % 
(self.repo.location,))
-   m.assert_directory_verifies()
-   out.eend(0)
+   # if nothing has changed, skip 
the actual Manifest
+   # verification
+   if not local_state_unchanged:
+   out.ebegin('Verifying 
%s' % (self.repo.location,))
+   
m.assert_directory_verifies()
+   out.eend(0)
except GematoException as e:
writemsg_level("!!! Manifest 
verification failed:\n%s\n"
% (e,),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/modules/rsync/

[Michał Górny]

commit: c49715693d6602895021ff66a917bb9edc736d31
Author: Michał Górny  gentoo  org>
AuthorDate: Thu Feb  1 11:56:25 2018 +
Commit: Michał Górny  gentoo  org>
CommitDate: Mon Feb  5 18:41:41 2018 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c4971569

rsync: Load and update keys early

Load and update keys early to avoid delaying failures post rsync. Any
failure will prevent verification from happening, and presumably most of
the users will prefer fixing it and trying to sync again. For that case,
it is better to perform the task before actual rsync to avoid
unnecessarily rsyncing twice.

Reviewed-by: Zac Medico  gentoo.org>

 pym/portage/sync/modules/rsync/rsync.py | 103 ++--
 1 file changed, 57 insertions(+), 46 deletions(-)

diff --git a/pym/portage/sync/modules/rsync/rsync.py 
b/pym/portage/sync/modules/rsync/rsync.py
index 5c0b53f9e..dc4674548 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -110,7 +110,33 @@ class RsyncSync(NewBase):
level=logging.WARNING, noiselevel=-1)
self.verify_jobs = None
 
+   openpgp_env = None
+   if self.verify_metamanifest and gemato is not None:
+   # Use isolated environment if key is specified,
+   # system environment otherwise
+   if self.repo.sync_openpgp_key_path is not None:
+   openpgp_env = 
gemato.openpgp.OpenPGPEnvironment()
+   else:
+   openpgp_env = 
gemato.openpgp.OpenPGPSystemEnvironment()
+
try:
+   # Load and update the keyring early. If it fails, then 
verification
+   # will not be performed and the user will have to fix 
it and try again,
+   # so we may as well bail out before actual rsync 
happens.
+   if openpgp_env is not None and 
self.repo.sync_openpgp_key_path is not None:
+   try:
+   out.einfo('Using keys from %s' % 
(self.repo.sync_openpgp_key_path,))
+   with 
io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
+   openpgp_env.import_key(f)
+   out.ebegin('Refreshing keys from 
keyserver')
+   openpgp_env.refresh_keys()
+   out.eend(0)
+   except GematoException as e:
+   writemsg_level("!!! Manifest 
verification impossible due to keyring problem:\n%s\n"
+   % (e,),
+   level=logging.ERROR, 
noiselevel=-1)
+   return (1, False)
+
# Real local timestamp file.
self.servertimestampfile = os.path.join(
self.repo.location, "metadata", "timestamp.chk")
@@ -299,52 +325,36 @@ class RsyncSync(NewBase):
level=logging.ERROR, 
noiselevel=-1)
exitcode = 127
else:
-   # Use isolated environment if key is 
specified,
-   # system environment otherwise
-   if self.repo.sync_openpgp_key_path is 
not None:
-   openpgp_env_cls = 
gemato.openpgp.OpenPGPEnvironment
-   else:
-   openpgp_env_cls = 
gemato.openpgp.OpenPGPSystemEnvironment
-
try:
-   with openpgp_env_cls() as 
openpgp_env:
-   if 
self.repo.sync_openpgp_key_path is not None:
-   
out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
-   with 
io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
-   
openpgp_env.import_key(f)
-   
out.ebegin('Refreshing keys from keyserver')
-   
openpgp_env.refresh_keys()
-   out.eend(0)
-
-   # we always verify the 
Manifest signature, in case
-