Re: [gentoo-portage-dev] emerge getting slower

2015-12-21 Thread Zac Medico 
On 12/21/2015 12:24 PM, Joakim Tjernlund wrote:
> Lately it feels like emerge has been getting unusual slow to calculate 
> dependencies.
> I did a strace while emerge was running and noticed a lot of:
> 
> stat64("/usr/portage/profiles/base/profile.bashrc", {st_mode=S_IFREG|0644, 
> st_size=183, ...}) = 0
> stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
> 0xbfbb02d0) = -1 ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfbb02d0) 
> = -1 ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 
> 0xbfbb02d0) = -1 ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/releases/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc",
>  0xbfbb02d0) = -1 ENOENT (No such file or directory)
> stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc",
>  {st_mode=S_IFREG|0644, st_size=133, ...}) = 0
> stat64("/etc/portage/profile/profile.bashrc", 0xbfbb02d0) = -1 ENOENT (No 
> such file or directory)
> access("/usr/local/portage/tmv3-target-overlay/dev-libs/apr/apr-1.5.1-r1.ebuild",
>  R_OK) = -1 ENOENT (No such file or directory)
> access("/usr/portage/dev-libs/apr/apr-1.5.1-r1.ebuild", R_OK) = 0
> stat64("/usr/portage/dev-libs/apr/apr-1.5.1-r1.ebuild", 
> {st_mode=S_IFREG|0644, st_size=4215, ...}) = 0
> open("/usr/portage/metadata/md5-cache/dev-libs/apr-1.5.1-r1", 
> O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5
> fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
> ioctl(5, TCGETS, 0xbfbb0d88)= -1 ENOTTY (Inappropriate ioctl for 
> device)
> fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
> _llseek(5, 0, [0], SEEK_CUR)= 0
> _llseek(5, 0, [0], SEEK_CUR)= 0
> _llseek(5, 0, [0], SEEK_CUR)= 0
> fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
> read(5, "DEFINED_PHASES=compile configure"..., 1351) = 1350
> read(5, "", 1)  = 0
> fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
> close(5)= 0
> open("/usr/portage/dev-libs/apr/apr-1.5.1-r1.ebuild", 
> O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5
> fstat64(5, {st_mode=S_IFREG|0644, st_size=4215, ...}) = 0
> ioctl(5, TCGETS, 0xbfbb0898)= -1 ENOTTY (Inappropriate ioctl for 
> device)
> fstat64(5, {st_mode=S_IFREG|0644, st_size=4215, ...}) = 0
> _llseek(5, 0, [0], SEEK_CUR)= 0
> read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 4215
> read(5, "", 24576)  = 0
> read(5, "", 32768)  = 0
> close(5)= 0
> stat64("/usr/portage/profiles/base/profile.bashrc", {st_mode=S_IFREG|0644, 
> st_size=183, ...}) = 0
> stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
> 0xbfbb02d0) = -1 ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfbb02d0) 
> = -1 ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 
> 0xbfbb02d0) = -1 ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/releases/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 0xbfbb02d0) = -1 
> ENOENT (No such file or directory)
> stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc",
>  0xbfbb02d0) = -1 ENOENT (No such file or directory)
> stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc",
>  {st_mode=S_IFREG|0644,

[gentoo-portage-dev] emerge getting slower

2015-12-21 Thread Joakim Tjernlund 
Lately it feels like emerge has been getting unusual slow to calculate 
dependencies.
I did a strace while emerge was running and noticed a lot of:

stat64("/usr/portage/profiles/base/profile.bashrc", {st_mode=S_IFREG|0644, 
st_size=183, ...}) = 0
stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
0xbfbb02d0) = -1 ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfbb02d0) = 
-1 ENOENT (No such file or directory)
stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 
0xbfbb02d0) = -1 ENOENT (No such file or directory)
stat64("/usr/portage/profiles/releases/profile.bashrc", 0xbfbb02d0) = -1 ENOENT 
(No such file or directory)
stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc", 
0xbfbb02d0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc",
 {st_mode=S_IFREG|0644, st_size=133, ...}) = 0
stat64("/etc/portage/profile/profile.bashrc", 0xbfbb02d0) = -1 ENOENT (No such 
file or directory)
access("/usr/local/portage/tmv3-target-overlay/dev-libs/apr/apr-1.5.1-r1.ebuild",
 R_OK) = -1 ENOENT (No such file or directory)
access("/usr/portage/dev-libs/apr/apr-1.5.1-r1.ebuild", R_OK) = 0
stat64("/usr/portage/dev-libs/apr/apr-1.5.1-r1.ebuild", {st_mode=S_IFREG|0644, 
st_size=4215, ...}) = 0
open("/usr/portage/metadata/md5-cache/dev-libs/apr-1.5.1-r1", 
O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
ioctl(5, TCGETS, 0xbfbb0d88)= -1 ENOTTY (Inappropriate ioctl for 
device)
fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
_llseek(5, 0, [0], SEEK_CUR)= 0
_llseek(5, 0, [0], SEEK_CUR)= 0
_llseek(5, 0, [0], SEEK_CUR)= 0
fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
read(5, "DEFINED_PHASES=compile configure"..., 1351) = 1350
read(5, "", 1)  = 0
fstat64(5, {st_mode=S_IFREG|0644, st_size=1350, ...}) = 0
close(5)= 0
open("/usr/portage/dev-libs/apr/apr-1.5.1-r1.ebuild", 
O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=4215, ...}) = 0
ioctl(5, TCGETS, 0xbfbb0898)= -1 ENOTTY (Inappropriate ioctl for 
device)
fstat64(5, {st_mode=S_IFREG|0644, st_size=4215, ...}) = 0
_llseek(5, 0, [0], SEEK_CUR)= 0
read(5, "# Copyright 1999-2015 Gentoo Fou"..., 32768) = 4215
read(5, "", 24576)  = 0
read(5, "", 32768)  = 0
close(5)= 0
stat64("/usr/portage/profiles/base/profile.bashrc", {st_mode=S_IFREG|0644, 
st_size=183, ...}) = 0
stat64("/usr/portage/profiles/default/linux/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/default/linux/powerpc/profile.bashrc", 
0xbfbb02d0) = -1 ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/base/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/powerpc/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/arch/powerpc/ppc32/profile.bashrc", 0xbfbb02d0) = 
-1 ENOENT (No such file or directory)
stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/profile.bashrc", 
0xbfbb02d0) = -1 ENOENT (No such file or directory)
stat64("/usr/portage/profiles/releases/profile.bashrc", 0xbfbb02d0) = -1 ENOENT 
(No such file or directory)
stat64("/usr/portage/profiles/releases/13.0/profile.bashrc", 0xbfbb02d0) = -1 
ENOENT (No such file or directory)
stat64("/usr/portage/profiles/default/linux/powerpc/ppc32/13.0/profile.bashrc", 
0xbfbb02d0) = -1 ENOENT (No such file or directory)
stat64("/usr/local/portage/tmv3-target-overlay/profiles/cusfpv3/profile.bashrc",
 {st_mode=S_IFREG|0644, st_size=133, ...}) = 0
stat64("/etc/portage/profile/profile.bashrc", 0xbfbb02d0) = -1 ENOENT (No such 
file or directory)
access("/usr/local/portage/tmv3-target-overlay/dev-libs/apr/apr-1.5.0-r2.ebuild",
 R_OK) = -1 ENOENT

Re: [gentoo-portage-dev] [PATCH v2] egencache: add --stable-mtime option

2015-12-21 Thread Zac Medico 
On 12/21/2015 07:04 AM, Alexander Berntsen wrote:
> The patch looks OK.
> 
> On 20/12/15 23:32, Zac Medico wrote:
>> Since the Manifest "stable mtime" behavior could be have
>> undiscovered bugs, disable it by default, and add a corresponding
>> egencache option.
> You have an English børk here -- "could be have".

Thanks, fixed.

>> [PATCH v2] fixes preserved_stats to contain stat results for
>> self.pkgdir even if the Manifest has been removed, and also
>> explains the reasoning for disabling the stable mtime behavior by
>> default.
> Maybe this could go into the commit message? I'd not mind a more
> verbose message for this.

In v3 I've added a comment block to Manifest.write in order to document
subtle issues involving the mtime of self.pkgdir that must be accounted for.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v2] egencache: add --stable-mtime option

2015-12-21 Thread Alexander Berntsen 
-BEGIN PGP SIGNED MESSAGE-
Hash: SHA512

The patch looks OK.

On 20/12/15 23:32, Zac Medico wrote:
> Since the Manifest "stable mtime" behavior could be have
> undiscovered bugs, disable it by default, and add a corresponding
> egencache option.
You have an English børk here -- "could be have".

> [PATCH v2] fixes preserved_stats to contain stat results for
> self.pkgdir even if the Manifest has been removed, and also
> explains the reasoning for disabling the stable mtime behavior by
> default.
Maybe this could go into the commit message? I'd not mind a more
verbose message for this.

- -- 
Alexander
berna...@gentoo.org
https://secure.plaimi.net/~alexander
-BEGIN PGP SIGNATURE-
Version: GnuPG v2

iQIcBAEBCgAGBQJWeBUaAAoJENQqWdRUGk8Bc8wQALHRzKQUdHzl8oGgzCvhvvCn
YSX3JT+R7OcDbIxy+JjDQ88Mh2DQf2JfEcAmUcL/lA0grAJ+xV5dYwOnRIEbqYZM
sTK+oIxIs0awOQa/qKvUCwpP8hBLB4CM95ECwFWMb1JvOWnP3+czuXz74DIrhmRC
3P6JVPOzF7oaNDx/WdHPYyp1un6jQRCAsnwWmSMA1DFzbpE9S94C9uFzWQDha/Kk
0pOZJmqYciCxUa0v7rcQCuJqdYmyHgN5wfaYN/bUC2YO+7IkTvM7EkI3avg95fK5
BsTGKZfse4x5VYBZ+urEl7w/rNLG6BvpWnpz3BELVE792It/+vazq1e/ceC582Zy
I69jidlP7xH5+0VpcUx0LVaPJ/x5qZ/18Jhwt4JAGgehKdneOp+JefhfhZ+im6n7
KTH/D6gtVmg+MkL51JTcVLXsWeCpwVbm1fPzi31+T3zqngiLKtNxUaIhlLVyJQNZ
JMl/GBGpXiyA7Ig9l7g2cBqUWAj3GcYk20XikEY3oTnxT3rQzIMP8GGpTFA4wYUf
88aSo+VFyr02nOHXtmyTugpvPlXxwjMxFhQ04+j57NR39Y4zfpteiNRb9o8yYxKu
3rOoexmqY3U7z49qPLaA+Gnf810v+Jw6Hwv0CNPjhCSbB705acTDpaAGLhj/hvXv
Dg7KrpzMqt5mYsPYIzLB
=iK0o
-END PGP SIGNATURE-

[gentoo-portage-dev] [PATCH v3] egencache: add --stable-mtime option

2015-12-21 Thread Zac Medico 
Since the Manifest "stable mtime" behavior could have undiscovered
bugs, disable it by default, and add a corresponding egencache option.

Suggested-by: Michał Górny 
---
[PATCH v3] fixes the commit message and adds a comment block to
Manifest.write in order to document subtle issues involving the mtime
of self.pkgdir that must be accounted for.

 bin/egencache  |  6 +-
 man/egencache.1|  3 +++
 pym/portage/manifest.py| 23 +-
 .../ebuild/_parallel_manifest/ManifestProcess.py   |  6 --
 .../ebuild/_parallel_manifest/ManifestScheduler.py |  7 +--
 .../ebuild/_parallel_manifest/ManifestTask.py  |  8 +---
 6 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/bin/egencache b/bin/egencache
index 7e3387e..07665e8 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -120,6 +120,9 @@ def parse_args(args):
choices=('y', 'n'),
metavar="",
help="manually override layout.conf sign-manifests setting")
+   common.add_argument("--stable-mtime",
+   action="store_true",
+   help="apply stable mtime to generated manifests (for rsync)")
common.add_argument("--strict-manifests",
choices=('y', 'n'),
metavar="",
@@ -1151,7 +1154,8 @@ def egencache_main(args):
force_sign_key=force_sign_key,
max_jobs=options.jobs,
max_load=options.load_average,
-   event_loop=event_loop)
+   event_loop=event_loop,
+   manifest_kwargs=dict(stable_mtime=options.stable_mtime))
 
signum = run_main_scheduler(scheduler)
if signum is not None:
diff --git a/man/egencache.1 b/man/egencache.1
index 7fd17c2..081e8c1 100644
--- a/man/egencache.1
+++ b/man/egencache.1
@@ -100,6 +100,9 @@ Manually override layout.conf sign-manifests setting.
 .BR "\-\-strict\-manifests< y | n >"
 Manually override "strict" FEATURES setting.
 .TP
+.BR "\-\-stable\-mtime"
+Apply stable mtime to generated manifests (for rsync).
+.TP
 .BR "\-\-thin\-manifests< y | n >"
 Manually override layout.conf thin-manifests setting.
 .TP
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index f696f84..6efd6e5 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -128,7 +128,7 @@ class Manifest(object):
def __init__(self, pkgdir, distdir=None, fetchlist_dict=None,
manifest1_compat=DeprecationWarning, from_scratch=False, 
thin=False,
allow_missing=False, allow_create=True, hashes=None,
-   find_invalid_path_char=None):
+   find_invalid_path_char=None, stable_mtime=False):
""" Create new Manifest instance for package in pkgdir.
Do not parse Manifest file if from_scratch == True (only 
for internal use)
The fetchlist_dict parameter is required only for 
generation of
@@ -145,6 +145,7 @@ class Manifest(object):
find_invalid_path_char = _find_invalid_path_char
self._find_invalid_path_char = find_invalid_path_char
self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
+   self.stable_mtime = stable_mtime
self.fhashdict = {}
self.hashes = set()
 
@@ -283,7 +284,16 @@ class Manifest(object):
myentries = list(self._createManifestEntries())
update_manifest = True
preserved_stats = {}
-   preserved_stats[self.pkgdir.rstrip(os.sep)] = 
os.stat(self.pkgdir)
+   if self.stable_mtime:
+   # The pre-existing mtime of self.pkgdir is 
included in the
+   # max mtime calculation in order to account for 
anything
+   # that may have been renamed or removed in this 
directory
+   # (including the Manifest itself). Note that 
the mtime of
+   # this directory will be always be bumped as a 
side-effect
+   # of writing the Manifest (since write_atomic 
uses a rename
+   # operation for atomicity), therefore it must 
be preserved
+   # before the Manifest is written.
+   preserved_stats[self.pkgdir.rstrip(os.sep)] = 
os.stat(self.pkgdir)
if myentries and not force:
try:
f = 
io.open(_unicode_encode(self.getFullname(),
@@ -291,7 +301,8 @@ class Manifest(object):
mode='r', 
encoding=_encodings['repo.content'],