commit: 207d98bd60abfd6f389032b269c265601926c8ca Author: Florian Schmaus <flow <AT> gentoo <DOT> org> AuthorDate: Fri Feb 7 09:23:45 2025 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Thu Sep 11 00:07:07 2025 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=207d98bd
Show @lru_cache stats if PORTAGE_SHOW_LRU_CACHE_INFO env var is set This introduces the PORTAGE_SHOW_LRU_CACHE_INFO environment variable. Setting the PORTAGE_SHOW_LRU_CACHE_INFO environment variable allows to analyze the effectiveness and potential bottlenecks of the @lru_cache annotation used by portage. The cache information will be printed after dependency calculation and after emerge "finished". Example output: Portage @lru_cache information use_reduce_cached: hit ratio: 71.25% (total: 48575, hits: 34611, misses: 13964) util: 100.00% (1024 / 1024) encoded_length: hit ratio: 0.00% (total: 0, hits: 0, misses: 0) util: 0.00% (1024 / 0) vercmp: hit ratio: 69.50% (total: 1777, hits: 1235, misses: 542) util: 52.93% (1024 / 542) catpkgsplit: hit ratio: 66.90% (total: 191511, hits: 128129, misses: 63382) util: 100.00% (10240 / 10240) get_eapi_attrs: hit ratio: 100.00% (total: 267206, hits: 267200, misses: 6) util: 18.75% (32 / 6) Signed-off-by: Florian Schmaus <flow <AT> gentoo.org> Part-of: https://github.com/gentoo/portage/pull/1427 Closes: https://github.com/gentoo/portage/pull/1427 Signed-off-by: Sam James <sam <AT> gentoo.org> NEWS | 5 +++++ lib/_emerge/depgraph.py | 2 ++ lib/_emerge/post_emerge.py | 3 +++ lib/portage/process.py | 12 +++++----- lib/portage/util/meson.build | 1 + lib/portage/util/portage_lru_cache.py | 42 +++++++++++++++++++++++++++++++++++ man/emerge.1 | 10 +++++++++ 7 files changed, 70 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 212765e8fa..bcd09b16d3 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,11 @@ Security: achievable in the course of bash performing arithmetic operations on numbers representing unix time. +Features: + +* Portage will now show statistics about its internal LRU caches if the + PORTAGE_SHOW_LRU_CACHE_INFO environment variable is set. + Bug fixes: * The emerge-webrsync utility will now ensure that the temporary directory diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index ddef7bb0a1..a6e63c058c 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -62,6 +62,7 @@ from portage.util import writemsg_level, write_atomic from portage.util.digraph import digraph from portage.util.futures import asyncio from portage.util._async.TaskScheduler import TaskScheduler +from portage.util.portage_lru_cache import show_lru_cache_info from portage.versions import _pkg_str, catpkgsplit from portage.binpkg import get_binpkg_format @@ -11800,6 +11801,7 @@ def _spinner_stop(spinner, backtracked: int = -1, max_retries: int = -1): portage.writemsg_stdout( f"Dependency resolution took {darkgreen(time_fmt)} s{backtrack_info}.\n\n" ) + show_lru_cache_info() def backtrack_depgraph( diff --git a/lib/_emerge/post_emerge.py b/lib/_emerge/post_emerge.py index 6f1f1c243d..06fb8d1784 100644 --- a/lib/_emerge/post_emerge.py +++ b/lib/_emerge/post_emerge.py @@ -11,6 +11,7 @@ from portage.news import count_unread_news, display_news_notifications from portage.output import colorize from portage.util._dyn_libs.display_preserved_libs import display_preserved_libs from portage.util._info_files import chk_updated_info_files +from portage.util.portage_lru_cache import show_lru_cache_info from .chk_updated_cfg_files import chk_updated_cfg_files from .emergelog import emergelog @@ -169,3 +170,5 @@ def post_emerge(myaction, myopts, myfiles, target_root, trees, mtimedb, retval): if "--quiet" not in myopts and myaction is None and "@world" in myfiles: show_depclean_suggestion() + + show_lru_cache_info() diff --git a/lib/portage/process.py b/lib/portage/process.py index 502f49f81f..977fe53fe7 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -323,17 +323,19 @@ class EnvStats: env_largest_size: int +@lru_cache(1024) +def _encoded_length(s): + return len(os.fsencode(s)) + + def calc_env_stats(env) -> EnvStats: - @lru_cache(1024) - def encoded_length(s): - return len(os.fsencode(s)) env_size = 0 env_largest_name = None env_largest_size = 0 for env_name, env_value in env.items(): - env_name_size = encoded_length(env_name) - env_value_size = encoded_length(env_value) + env_name_size = _encoded_length(env_name) + env_value_size = _encoded_length(env_value) # Add two for '=' and the terminating null byte. total_size = env_name_size + env_value_size + 2 if total_size > env_largest_size: diff --git a/lib/portage/util/meson.build b/lib/portage/util/meson.build index 8a60617d6f..fed09e20df 100644 --- a/lib/portage/util/meson.build +++ b/lib/portage/util/meson.build @@ -21,6 +21,7 @@ py.install_sources( 'mtimedb.py', 'netlink.py', 'path.py', + 'portage_lru_cache.py', 'shelve.py', 'socks5.py', 'whirlpool.py', diff --git a/lib/portage/util/portage_lru_cache.py b/lib/portage/util/portage_lru_cache.py new file mode 100644 index 0000000000..6cf689ef23 --- /dev/null +++ b/lib/portage/util/portage_lru_cache.py @@ -0,0 +1,42 @@ +# Copyright 2025 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import os +import portage + + +def show_lru_cache_info(): + if not os.environ.get("PORTAGE_SHOW_LRU_CACHE_INFO"): + return + + portage_lru_caches = { + portage.dep._use_reduce_cached: "use_reduce_cached", + portage.eapi._get_eapi_attrs: "get_eapi_attrs", + portage.process._encoded_length: "encoded_length", + portage.versions.catpkgsplit: "catpkgsplit", + portage.versions.vercmp: "vercmp", + } + + print("Portage @lru_cache information") + for method, name in portage_lru_caches.items(): + cache_info = method.cache_info() + + hits = cache_info.hits + misses = cache_info.misses + maxsize = cache_info.maxsize + currsize = cache_info.currsize + + total = hits + misses + if total: + hitratio = hits / total + else: + hitratio = 0 + + if maxsize: + utilization = currsize / maxsize + else: + utilization = 0 + + pretty_cache_info = f"hit ratio: {hitratio:.2%} (total: {total}, hits: {hits}, misses: {misses}) util: {utilization:.2%} ({maxsize} / {currsize})" + + print(f"{name}: {pretty_cache_info}") diff --git a/man/emerge.1 b/man/emerge.1 index c629c68262..42fc53f95b 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1441,6 +1441,16 @@ add this to \fBmake.conf\fR(5): Tools such as dispatch\-conf, cfg\-update, and etc\-update are also available to aid in the merging of these files. They provide interactive merging and can auto\-merge trivial changes. +.SH "PORTAGE DEVELOPMENT FEATURES" +This section describes features that are of interest when developing +Portage. They allow for introspection, debugging, and performance +analysis. Portage provides no stability guarantee; any feature may +disappear in the future, and the output format may change without +prior notice. +.TP +.BR PORTAGE_SHOW_LRU_CACHE_INFO +If this environment variable is set, then Portage will display +statistics about its internal LRU caches. .SH "REPORTING BUGS" Please report any bugs you encounter through our website: .LP
