[gentoo-portage-dev] [PATCH] collision_protect: use dynamic report interval
The reporting of files remaining can look somewhat odd since the report
interval is hardcoded to be per 1000 objects. Adjust this interval to
be regular towards the end. While at it, report percentage done.
Output before this patch:
* checking 6111 files for package collisions
5111 files remaining ...
4111 files remaining ...
3111 files remaining ...
2111 files remaining ...
files remaining ...
111 files remaining ...
After:
* checking 6158 files for package collisions
16% done, 5131 files remaining ...
33% done, 4104 files remaining ...
50% done, 3077 files remaining ...
66% done, 2050 files remaining ...
83% done, 1023 files remaining ...
100% done
Signed-off-by: Fabian Groffen
---
lib/portage/dbapi/vartree.py | 14 +++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 4b91caea8..78f2b37f2 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3475,13 +3475,19 @@ class dblink(object):
symlink_collisions = []
destroot = self.settings['ROOT']
totfiles = len(file_list) + len(symlink_list)
+ bucksize = 1000
+ buckcnt = int(totfiles / bucksize)
+ if buckcnt == 0 or totfiles % bucksize > int(bucksize /
2):
+ buckcnt = buckcnt + 1
+ bucksize = int(totfiles / buckcnt) + 1
showMessage(_(" %s checking %d files for package
collisions\n") % \
- (colorize("GOOD", "*"), totfiles))
+ (colorize("GOOD", "*"), totfiles))
for i, (f, f_type) in enumerate(chain(
((f, "reg") for f in file_list),
((f, "sym") for f in symlink_list))):
- if i % 1000 == 0 and i != 0:
- showMessage(_("%d files remaining
...\n") % (totfiles - i))
+ if i % bucksize == 0 and i != 0:
+ showMessage(_("%3d%% done, %d files
remaining ...\n") %
+ (i * 100 / totfiles,
totfiles - i))
dest_path = normalize_path(
os.path.join(destroot,
f.lstrip(os.path.sep)))
@@ -3570,6 +3576,8 @@ class dblink(object):
break
if stopmerge:
collisions.append(f)
+ if bucksize < totfiles:
+ showMessage(_("100% done\n"))
return collisions, dirs_ro, symlink_collisions,
plib_collisions
def _lstat_inode_map(self, path_iter):
--
2.20.1
Re: [gentoo-portage-dev] [PATCH] collision_protect: use dynamic report interval
On 08-01-2019 09:17:04 +0100, Ulrich Mueller wrote: > > On Tue, 08 Jan 2019, Fabian Groffen wrote: > > > Output before this patch: > > > * checking 6111 files for package collisions > > [...] > > > After: > > > * checking 6158 files for package collisions > > [...] > > Why does the total number of files change? First was python2, second python3. The result was indicative only, the number doesn't change of course. Sorry for the confusion. Fabian -- Fabian Groffen Gentoo on a different level signature.asc Description: PGP signature
Re: [gentoo-portage-dev] [PATCH] pid-sandbox: whitelist selected pkg_* phases (bug 673794)
Dnia January 8, 2019 4:53:11 AM UTC, Zac Medico napisał(a):
>Whitelist the same phases that are whitelisted for FEATURES=cgroup,
>since pid-sandbox is less valuable and is likely to have unintended
>consenquences during these phases.
>
>Bug: https://bugs.gentoo.org/673794
>Signed-off-by: Zac Medico
>---
> lib/_emerge/AbstractEbuildProcess.py | 9 ++---
> lib/portage/package/ebuild/doebuild.py | 8 ++--
> 2 files changed, 12 insertions(+), 5 deletions(-)
>
>diff --git a/lib/_emerge/AbstractEbuildProcess.py
>b/lib/_emerge/AbstractEbuildProcess.py
>index bda0bd83f..ddf04e9b3 100644
>--- a/lib/_emerge/AbstractEbuildProcess.py
>+++ b/lib/_emerge/AbstractEbuildProcess.py
>@@ -1,4 +1,4 @@
>-# Copyright 1999-2018 Gentoo Foundation
>+# Copyright 1999-2019 Gentoo Foundation
> # Distributed under the terms of the GNU General Public License v2
>
> import errno
>@@ -22,6 +22,10 @@ from portage.util.futures import asyncio
> from portage.util._pty import _create_pty_or_pipe
> from portage.util import apply_secpass_permissions
>
>+portage.proxy.lazyimport.lazyimport(globals(),
>+ 'portage.package.ebuild.doebuild:_global_pid_phases',
>+)
>+
> class AbstractEbuildProcess(SpawnProcess):
>
> __slots__ = ('phase', 'settings',) + \
>@@ -30,7 +34,6 @@ class AbstractEbuildProcess(SpawnProcess):
>
> _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)
> _phases_interactive_whitelist = ('config',)
>- _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm',
>'config')
>
> # Number of milliseconds to allow natural exit of the ebuild
> # process after it has called the exit command via IPC. It
>@@ -71,7 +74,7 @@ class AbstractEbuildProcess(SpawnProcess):
> # Check if the cgroup hierarchy is in place. If it's not, mount
> it.
> if (os.geteuid() == 0 and platform.system() == 'Linux'
> and 'cgroup' in self.settings.features
>- and self.phase not in
>self._phases_without_cgroup):
>+ and self.phase not in _global_pid_phases):
> cgroup_root = '/sys/fs/cgroup'
> cgroup_portage = os.path.join(cgroup_root, 'portage')
>
>diff --git a/lib/portage/package/ebuild/doebuild.py
>b/lib/portage/package/ebuild/doebuild.py
>index baebb9a27..f11923595 100644
>--- a/lib/portage/package/ebuild/doebuild.py
>+++ b/lib/portage/package/ebuild/doebuild.py
>@@ -1,4 +1,4 @@
>-# Copyright 2010-2018 Gentoo Authors
>+# Copyright 2010-2019 Gentoo Authors
> # Distributed under the terms of the GNU General Public License v2
>
> from __future__ import unicode_literals
>@@ -110,6 +110,9 @@ _ipc_phases = frozenset([
> "preinst", "postinst", "prerm", "postrm",
> ])
>
>+# phases which execute in the global PID namespace
>+_global_pid_phases = frozenset(['preinst', 'postinst', 'prerm',
>'postrm', 'config'])
>+
> # phases in which networking access is allowed
> _networked_phases = frozenset([
> # for VCS fetching
>@@ -153,7 +156,8 @@ def _doebuild_spawn(phase, settings,
>actionmap=None, **kwargs):
> kwargs['networked'] = 'network-sandbox' not in settings.features or \
> phase in _networked_phases or \
> 'network-sandbox' in settings['PORTAGE_RESTRICT'].split()
>- kwargs['pidns'] = 'pid-sandbox' in settings.features
>+ kwargs['pidns'] = ('pid-sandbox' in settings.features and
>+ phase not in _global_pid_phases)
>
> if phase == 'depend':
> kwargs['droppriv'] = 'userpriv' in settings.features
LGTM, thanks.
--
Best regards,
Michał Górny
Re: [gentoo-portage-dev] [PATCH] collision_protect: use dynamic report interval
> On Tue, 08 Jan 2019, Fabian Groffen wrote: > Output before this patch: > * checking 6111 files for package collisions > [...] > After: > * checking 6158 files for package collisions > [...] Why does the total number of files change? Ulrich signature.asc Description: PGP signature
[gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
The reporting of files remaining can look somewhat odd since the report
interval is hardcoded to be per 1000 objects. Adjust this interval to
be time based. This means that modern (fast) machines likely will never
see the countdown messages at all. On slow setups the message will be
informative that there is progress, albeit rather slowly. While at it,
report percentage done.
Output before this patch:
* checking 6158 files for package collisions
5158 files remaining ...
4158 files remaining ...
3158 files remaining ...
2158 files remaining ...
1158 files remaining ...
158 files remaining ...
Possible output after this patch on a slower machine:
* checking 6158 files for package collisions
48% done, 3145 files remaining ...
96% done, 192 files remaining ...
100% done
Signed-off-by: Fabian Groffen
---
lib/portage/dbapi/vartree.py | 11 +--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 4b91caea8..244195fad 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -3475,13 +3475,18 @@ class dblink(object):
symlink_collisions = []
destroot = self.settings['ROOT']
totfiles = len(file_list) + len(symlink_list)
+ tnow = time.time()
+ tinterv = 2 # seconds
+ ninterv = tnow + tinterv
showMessage(_(" %s checking %d files for package
collisions\n") % \
(colorize("GOOD", "*"), totfiles))
for i, (f, f_type) in enumerate(chain(
((f, "reg") for f in file_list),
((f, "sym") for f in symlink_list))):
- if i % 1000 == 0 and i != 0:
- showMessage(_("%d files remaining
...\n") % (totfiles - i))
+ if time.time() > ninterv:
+ showMessage(_("%3d%% done, %d files
remaining ...\n") %
+ (i * 100 / totfiles,
totfiles - i))
+ ninterv = time.time() + tinterv
dest_path = normalize_path(
os.path.join(destroot,
f.lstrip(os.path.sep)))
@@ -3570,6 +3575,8 @@ class dblink(object):
break
if stopmerge:
collisions.append(f)
+ if tnow + tinterv < ninterv:
+ showMessage(_("100% done\n"))
return collisions, dirs_ro, symlink_collisions,
plib_collisions
def _lstat_inode_map(self, path_iter):
--
2.20.1
Re: [gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
On 1/8/19 5:42 AM, Fabian Groffen wrote:
> The reporting of files remaining can look somewhat odd since the report
> interval is hardcoded to be per 1000 objects. Adjust this interval to
> be time based. This means that modern (fast) machines likely will never
> see the countdown messages at all. On slow setups the message will be
> informative that there is progress, albeit rather slowly. While at it,
> report percentage done.
>
> Output before this patch:
>
> * checking 6158 files for package collisions
> 5158 files remaining ...
> 4158 files remaining ...
> 3158 files remaining ...
> 2158 files remaining ...
> 1158 files remaining ...
> 158 files remaining ...
>
> Possible output after this patch on a slower machine:
>
> * checking 6158 files for package collisions
> 48% done, 3145 files remaining ...
> 96% done, 192 files remaining ...
> 100% done
>
> Signed-off-by: Fabian Groffen
> ---
> lib/portage/dbapi/vartree.py | 11 +--
> 1 file changed, 9 insertions(+), 2 deletions(-)
>
> diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
> index 4b91caea8..244195fad 100644
> --- a/lib/portage/dbapi/vartree.py
> +++ b/lib/portage/dbapi/vartree.py
> @@ -3475,13 +3475,18 @@ class dblink(object):
> symlink_collisions = []
> destroot = self.settings['ROOT']
> totfiles = len(file_list) + len(symlink_list)
> + tnow = time.time()
> + tinterv = 2 # seconds
> + ninterv = tnow + tinterv
> showMessage(_(" %s checking %d files for package
> collisions\n") % \
> (colorize("GOOD", "*"), totfiles))
> for i, (f, f_type) in enumerate(chain(
> ((f, "reg") for f in file_list),
> ((f, "sym") for f in symlink_list))):
> - if i % 1000 == 0 and i != 0:
> - showMessage(_("%d files remaining
> ...\n") % (totfiles - i))
> + if time.time() > ninterv:
> + showMessage(_("%3d%% done, %d files
> remaining ...\n") %
> + (i * 100 / totfiles,
> totfiles - i))
> + ninterv = time.time() + tinterv
>
> dest_path = normalize_path(
> os.path.join(destroot,
> f.lstrip(os.path.sep)))
> @@ -3570,6 +3575,8 @@ class dblink(object):
> break
> if stopmerge:
> collisions.append(f)
> + if tnow + tinterv < ninterv:
> + showMessage(_("100% done\n"))
> return collisions, dirs_ro, symlink_collisions,
> plib_collisions
>
> def _lstat_inode_map(self, path_iter):
>
Please replace time.time() with portage.util.monotonic.monotonic().
--
Thanks,
Zac
signature.asc
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH v3] collision_protect: use dynamic report interval
On 08/01/19 19:15, Zac Medico wrote:
> On 1/8/19 5:42 AM, Fabian Groffen wrote:
>> The reporting of files remaining can look somewhat odd since the report
>> interval is hardcoded to be per 1000 objects. Adjust this interval to
>> be time based. This means that modern (fast) machines likely will never
>> see the countdown messages at all. On slow setups the message will be
>> informative that there is progress, albeit rather slowly. While at it,
>> report percentage done.
>>
>> Output before this patch:
>>
>> * checking 6158 files for package collisions
>> 5158 files remaining ...
>> 4158 files remaining ...
>> 3158 files remaining ...
>> 2158 files remaining ...
>> 1158 files remaining ...
>> 158 files remaining ...
>>
>> Possible output after this patch on a slower machine:
>>
>> * checking 6158 files for package collisions
>> 48% done, 3145 files remaining ...
>> 96% done, 192 files remaining ...
>> 100% done
>>
>> Signed-off-by: Fabian Groffen
>> ---
>> lib/portage/dbapi/vartree.py | 11 +--
>> 1 file changed, 9 insertions(+), 2 deletions(-)
>>
>> diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
>> index 4b91caea8..244195fad 100644
>> --- a/lib/portage/dbapi/vartree.py
>> +++ b/lib/portage/dbapi/vartree.py
>> @@ -3475,13 +3475,18 @@ class dblink(object):
>> symlink_collisions = []
>> destroot = self.settings['ROOT']
>> totfiles = len(file_list) + len(symlink_list)
>> +tnow = time.time()
>> +tinterv = 2 # seconds
>> +ninterv = tnow + tinterv
>> showMessage(_(" %s checking %d files for package
>> collisions\n") % \
>> (colorize("GOOD", "*"), totfiles))
>> for i, (f, f_type) in enumerate(chain(
>> ((f, "reg") for f in file_list),
>> ((f, "sym") for f in symlink_list))):
>> -if i % 1000 == 0 and i != 0:
>> -showMessage(_("%d files remaining
>> ...\n") % (totfiles - i))
>> +if time.time() > ninterv:
>> +showMessage(_("%3d%% done, %d files
>> remaining ...\n") %
>> +(i * 100 / totfiles,
>> totfiles - i))
>> +ninterv = time.time() + tinterv
>>
>> dest_path = normalize_path(
>> os.path.join(destroot,
>> f.lstrip(os.path.sep)))
>> @@ -3570,6 +3575,8 @@ class dblink(object):
>> break
>> if stopmerge:
>> collisions.append(f)
>> +if tnow + tinterv < ninterv:
>> +showMessage(_("100% done\n"))
>> return collisions, dirs_ro, symlink_collisions,
>> plib_collisions
>>
>> def _lstat_inode_map(self, path_iter):
>>
> Please replace time.time() with portage.util.monotonic.monotonic().
It's a bit of a nit-pick, granted, but can we ensure that the count-down's
remain padded/justified such that the numbers line up for easy at-a-glance
inspection ? The optimal standard looks somewhat like the pre-merge Sizes:
* Final size of build directory: 2696 KiB (2.6 MiB)
* Final size of installed tree: 5372 KiB (5.2 MiB)
Otherwise, I think this will be quite helpful. Thanks.
signature.asc
Description: OpenPGP digital signature
