Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package MirrorCache for openSUSE:Factory checked in at 2025-11-13 18:17:31 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old) and /work/SRC/openSUSE:Factory/.MirrorCache.new.2061 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache" Thu Nov 13 18:17:31 2025 rev:58 rq:1317619 version:1.104 Changes: -------- --- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2025-10-23 18:32:32.777868201 +0200 +++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2061/MirrorCache.changes 2025-11-13 18:17:34.014469821 +0100 @@ -1,0 +2,13 @@ +Thu Nov 06 14:24:56 UTC 2025 - Andrii Nikitin <[email protected]> + +- Update to version 1.104: + * Prepare folder mirrorlist agg during mirror scan (#629) + +------------------------------------------------------------------- +Mon Nov 03 05:20:43 UTC 2025 - Andrii Nikitin <[email protected]> + +- Update to version 1.103: + * Detect vpn from http header variable (#628) + * Add parameter MIRRORCACHE_VPN_PREFIX_NEG to escape some vpn ranges (#627) + +------------------------------------------------------------------- Old: ---- MirrorCache-1.102.obscpio New: ---- MirrorCache-1.104.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ MirrorCache.spec ++++++ --- /var/tmp/diff_new_pack.PgbPJe/_old 2025-11-13 18:17:34.910507656 +0100 +++ /var/tmp/diff_new_pack.PgbPJe/_new 2025-11-13 18:17:34.910507656 +0100 @@ -23,7 +23,7 @@ %define main_requires %{assetpack_requires} perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) rubygem(sass) %define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow sysuser-tools Name: MirrorCache -Version: 1.102 +Version: 1.104 Release: 0 Summary: WebApp to redirect and manage mirrors License: GPL-2.0-or-later ++++++ MirrorCache-1.102.obscpio -> MirrorCache-1.104.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/lib/MirrorCache/Datamodule.pm new/MirrorCache-1.104/lib/MirrorCache/Datamodule.pm --- old/MirrorCache-1.102/lib/MirrorCache/Datamodule.pm 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/lib/MirrorCache/Datamodule.pm 2025-11-06 15:13:54.000000000 +0100 @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2024 SUSE LLC +# Copyright (C) 2021-2025 SUSE LLC # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -63,7 +63,10 @@ has root_subtree => ($ENV{MIRRORCACHE_SUBTREE} // ""); has _vpn_var => $ENV{MIRRORCACHE_VPN}; +has _vpn_header_variable => ($ENV{MIRRORCACHE_VPN_HEADER_VARIABLE} // ""); +has _vpn_header_value => ($ENV{MIRRORCACHE_VPN_HEADER_VALUE} // ""); has vpn_prefix => ($ENV{MIRRORCACHE_VPN_PREFIX} ? lc($ENV{MIRRORCACHE_VPN_PREFIX}) : "10."); +has vpn_prefix_neg => ($ENV{MIRRORCACHE_VPN_PREFIX_NEG} ? lc($ENV{MIRRORCACHE_VPN_PREFIX_NEG}) : ""); has 'at'; has '_mime'; @@ -113,6 +116,7 @@ $self->file_age(undef); $self->media_version(undef); $self->ext(undef); + $self->_vpn(undef); } sub ip_sha1($self) { @@ -128,6 +132,22 @@ sub vpn($self) { return $self->_vpn_var if defined $self->_vpn_var; + if (my $var = $self->_vpn_header_variable) { + unless (defined $self->_vpn) { + if (my $val = scalar($self->_vpn_header_value)) { + eval { + if (my $zone = $self->c->req->headers->header($var)) { + if (fc($zone) eq fc($val)) { + $self->_vpn(1); + } else { + $self->_vpn(0); + } + } + 1; + } or print STDERR "Error in detecting $var: $@"; + } + } + } unless (defined $self->_vpn) { unless ($self->vpn_prefix) { @@ -139,6 +159,11 @@ for my $pref (split /[\s]+/, $self->vpn_prefix) { $match = 1 if (rindex($ip, $pref, 0) == 0); } + if ($match && $self->vpn_prefix_neg) { + for my $pref (split /[\s]+/, $self->vpn_prefix_neg) { + $match = 0 if (rindex($ip, $pref, 0) == 0); + } + } $self->_vpn($match); } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/lib/MirrorCache/Schema/Result/FolderDiff.pm new/MirrorCache-1.104/lib/MirrorCache/Schema/Result/FolderDiff.pm --- old/MirrorCache-1.102/lib/MirrorCache/Schema/Result/FolderDiff.pm 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/lib/MirrorCache/Schema/Result/FolderDiff.pm 2025-11-06 15:13:54.000000000 +0100 @@ -61,6 +61,8 @@ data_type => 'timestamp', is_nullable => 0 }, + "mtime_latest", + { data_type => 'bigint' }, "realfolder_id", { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 }, ); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/lib/MirrorCache/Schema/ResultSet/Server.pm new/MirrorCache-1.104/lib/MirrorCache/Schema/ResultSet/Server.pm --- old/MirrorCache-1.102/lib/MirrorCache/Schema/ResultSet/Server.pm 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/lib/MirrorCache/Schema/ResultSet/Server.pm 2025-11-06 15:13:54.000000000 +0100 @@ -105,18 +105,17 @@ $condition_server_project = "and sp.server_id IS NULL"; } - my $join_file_cond = "fl.folder_id = fd.folder_id"; - my $folder_cond = "fd.folder_id in (coalesce((select id from folder where path = concat(?::text,'/repodata')),?),?) and (fdf.file_id is NULL and fl.folder_id in (coalesce((select id from folder where path = concat(?::text,'/repodata')),?),?))"; - my $where_recent = "where s.mtime > 0"; - # license.tar* and info.xml* might be kept with the same name through updates, so timestamp on them is unreliable in mirrorlist for folders - my $file_dt = ", max(case when fdf.file_id is null and fl.name ~ '[0-9]' and fl.name not like '%license.tar.%' and fl.name not like '%info.xml.%' and fl.name not like '%.asc' and fl.name not like '%.txt' and fl.name not like '%/' and fl.name not like 'yast2%' and fl.name not like '%.pf2' and fl.name not like '%patterns.xml.zst' and fl.name not like '%data%xml%' and fl.name not like '%appdata-icons%' then fl.mtime else null end) as mtime"; - my $group_by = "group by s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country, s.lat, s.lng, s.score, fd.folder_id"; + my $join_file = ""; + my $join_folder_diff_file = ""; + my $folder_cond = "fd.folder_id in (coalesce((select id from folder where path = concat(?::text,'/repodata')),?),?)"; + my $file_dt = ", fd.mtime_latest as mtime"; + my $fl_relevant = ""; if ($file_id) { - $join_file_cond = "fl.id = ?"; + $join_file = "join file fl on fl.id = ?"; + $join_folder_diff_file = "left join folder_diff_file fdf on fdf.file_id = fl.id and fdf.folder_diff_id = fd.id"; $file_dt = ", fl.mtime as mtime"; - $group_by = ""; - $where_recent = ""; + $fl_relevant = "and date_trunc('second', fl.dt) <= coalesce(fds.dt, fl.dt)"; $folder_cond = "fd.folder_id in (?,?) and coalesce(fd.realfolder_id, ?) = ? and (fdf.file_id is NULL and fl.folder_id = ?)"; } @@ -154,11 +153,11 @@ from ( select s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country, s.lat, s.lng, s.score $file_dt, fd.folder_id from folder_diff fd - join file fl on $join_file_cond - join folder_diff_server fds on fd.id = fds.folder_diff_id and date_trunc('second', fl.dt) <= coalesce(fds.dt, fl.dt) + $join_file + join folder_diff_server fds on fd.id = fds.folder_diff_id $fl_relevant join server s on fds.server_id = s.id and s.enabled $country_condition $condition_our_regions left join server_capability_declaration scd on s.id = scd.server_id and scd.capability = 'country' - left join folder_diff_file fdf on fdf.file_id = fl.id and fdf.folder_diff_id = fd.id + $join_folder_diff_file $join_server_project where $folder_cond $condition_server_project and ( -- here mirrors may be declared to handle only specific countries @@ -170,7 +169,6 @@ ( not scd.enabled and not ? ~ scd.extra) ) ) - $group_by ) s join folder f on f.id = s.folder_id left join server_capability_declaration scd on s.id = scd.server_id and scd.capability = '$capability' and NOT scd.enabled @@ -181,7 +179,6 @@ left join server_stability stability_schemex on s.id = stability_schemex.server_id and stability_schemex.capability = '$capabilityx' left join server_stability stability_ipv on s.id = stability_ipv.server_id and stability_ipv.capability = '$ipv' left join server_stability stability_ipvx on s.id = stability_ipvx.server_id and stability_ipvx.capability = '$ipvx' -$where_recent ) x WHERE not_disabled $extra order by rating_country desc, (dist/100)::int, support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv desc, score, realfolderscore, random() @@ -203,7 +200,7 @@ if ($file_id) { $prep->execute($file_id, @country_params, $realfolder_id, $folder_id, $realfolder_id, $realfolder_id, $realfolder_id, $country, $country, $country); } else { - $prep->execute(@country_params, $path, $realfolder_id, $folder_id, $path, $realfolder_id, $folder_id, $country, $country, $country); + $prep->execute(@country_params, $path, $realfolder_id, $folder_id, $country, $country, $country); } my $server_arrayref = $dbh->selectall_arrayref($prep, { Slice => {} }); return $server_arrayref; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/lib/MirrorCache/Task/MirrorScan.pm new/MirrorCache-1.104/lib/MirrorCache/Task/MirrorScan.pm --- old/MirrorCache-1.102/lib/MirrorCache/Task/MirrorScan.pm 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/lib/MirrorCache/Task/MirrorScan.pm 2025-11-06 15:13:54.000000000 +0100 @@ -47,17 +47,17 @@ unless my $guard = $minion->guard('mirror_scan' . $path, 20*60); $job->note($path => 1); - my ($folder_id, $realfolder_id, $anotherpath, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes); + my ($folder_id, $realfolder_id, $anotherpath, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes, $dbfilemtimes); { return $job->finish('folder sync job is still active') unless my $guard_r = $minion->guard('folder_sync' . $path, 360); - ($folder_id, $realfolder_id, $anotherpath, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes) + ($folder_id, $realfolder_id, $anotherpath, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes, $dbfilemtimes) = _dbfiles($app, $job, $path); } return undef unless $dbfiles; - my $count = _doscan($app, $job, $path, $realfolder_id, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes); + my $count = _doscan($app, $job, $path, $realfolder_id, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes, $dbfilemtimes); $job->note($count => 1); return $job->finish; } @@ -88,6 +88,7 @@ my @dbfiles = (); my %dbfileids = (); my %dbfileprefixes = (); + my %dbfilemtimes = (); my $max_dt = 0; for my $file ($schema->resultset('File')->search({folder_id => $folder_id})) { my $basename = $file->name; @@ -96,14 +97,15 @@ $dbfileprefixes{_reliable_prefix($basename)} = 1; push @dbfiles, $basename; $dbfileids{$basename} = $file->id; + $dbfilemtimes{$basename} = $file->mtime; $max_dt = $file->dt if !$max_dt || ( 0 > DateTime->compare($max_dt, $file->dt) ); } @dbfiles = sort @dbfiles; - return $folder->id, $folder_id, $realpath, $latestdt, $max_dt, \@dbfiles, \%dbfileids, \%dbfileprefixes; + return $folder->id, $folder_id, $realpath, $latestdt, $max_dt, \@dbfiles, \%dbfileids, \%dbfileprefixes, \%dbfilemtimes; } sub _doscan { - my ($app, $job, $path, $realfolder_id, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes) = @_; + my ($app, $job, $path, $realfolder_id, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes, $dbfilemtimes) = @_; my @dbfiles = @$dbfiles; my %dbfileids = %$dbfileids; my %dbfileprefixes = %$dbfileprefixes; @@ -216,8 +218,20 @@ } my $ctx = Digest::MD5->new; my @missing_files = (); + my $mtime_latest = 0; foreach my $file (@dbfiles) { - next if $mirrorfiles{$file} || substr($file,length($file)-1) eq '/' || $hasall; + next if substr($file,length($file)-1) eq '/'; + if ($mirrorfiles{$file} || $hasall) { + # track mtime only for versioned files + next unless $file =~ m/[0-9]/; + # exluding some versioned files that change timestamp + next if $file =~ m/(license\.tar\.|info\.xml|\.asc$|\.txt|^yast2|\.pf2$|.patterns.xml.zst$|data.*xml|appdata-icons|)/; + if (my $mtime = $dbfilemtimes->{$file}) { + $mtime_latest = $mtime if !$mtime_latest || $mtime_latest < $mtime; + } + next; + } + ; $ctx->add($file); push @missing_files, $dbfileids{$file}; } @@ -228,6 +242,7 @@ $folder_diff = $schema->resultset('FolderDiff')->find_or_new({folder_id => $folder_id, hash => $digest}); unless($folder_diff->in_storage) { $folder_diff->dt($latestdt); + $folder_diff->mtime_latest($mtime_latest); $folder_diff->realfolder_id($realfolder_id) if $realfolder_id && $realfolder_id != $folder_id; $folder_diff->insert; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/lib/MirrorCache/resources/migrations/Pg.sql new/MirrorCache-1.104/lib/MirrorCache/resources/migrations/Pg.sql --- old/MirrorCache-1.102/lib/MirrorCache/resources/migrations/Pg.sql 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/lib/MirrorCache/resources/migrations/Pg.sql 2025-11-06 15:13:54.000000000 +0100 @@ -465,3 +465,5 @@ alter table project add column if not exists shard varchar(32); -- 46 up -- noop +-- 47 up +alter table folder_diff add column if not exists mtime_latest bigint; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/lib/MirrorCache/resources/migrations/mysql.sql new/MirrorCache-1.104/lib/MirrorCache/resources/migrations/mysql.sql --- old/MirrorCache-1.102/lib/MirrorCache/resources/migrations/mysql.sql 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/lib/MirrorCache/resources/migrations/mysql.sql 2025-11-06 15:13:54.000000000 +0100 @@ -480,3 +480,5 @@ -- delete duplicate rows and add PR to server_project delete from server_project where (server_id, project_id, dt) in (select x.server_id, x.project_id, x.dt from server_project x join server_project y on (y.server_id, y.project_id) = (x.server_id, x.project_id) and x.dt < y.dt); alter table server_project add primary key if not exists (server_id, project_id); +-- 47 up +alter table folder_diff add column if not exists mtime_latest bigint; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/t/environ/01-smoke-multiroot.sh new/MirrorCache-1.104/t/environ/01-smoke-multiroot.sh --- old/MirrorCache-1.102/t/environ/01-smoke-multiroot.sh 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/t/environ/01-smoke-multiroot.sh 2025-11-06 15:13:54.000000000 +0100 @@ -5,6 +5,8 @@ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 $mc/gen_env MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \ + MIRRORCACHE_VPN_HEADER_VARIABLE='X-Company-Zone' \ + MIRRORCACHE_VPN_HEADER_VALUE='engineering' \ MIRRORCACHE_ROOT="'$mc/dt/root1:root1.com:root1.vpn|$mc/dt/root2:root2.com:root2.vpn|$mc/dt/root3:root3.com:root3.vpn'" mkdir -p $mc/dt/root1 @@ -44,6 +46,11 @@ $mc/curl -Is /download/folder1/file1.1.dat | grep -i location: | grep root1.com $mc/curl -H 'X-Forwarded-For: 10.0.0.1' -Is /download/folder1/file1.1.dat | grep -i location: | grep root1.vpn +$mc/curl -H 'X-Company-Zone: other' -Is /download/folder1/file1.1.dat | grep -i location: | grep root1.com +$mc/curl -H 'X-Company-Zone: engineering' -Is /download/folder1/file1.1.dat | grep -i location: | grep root1.vpn +$mc/curl -H 'X-Forwarded-For: 10.0.0.1; X-Company-Zone: other' -Is /download/folder1/file1.1.dat | grep -i location: | grep root1.com + + $mc/backstage/job folder_sync_schedule_from_misses $mc/backstage/job folder_sync_schedule $mc/backstage/shoot diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.102/t/environ/15-local-symlink-subtree-rootredirect.sh new/MirrorCache-1.104/t/environ/15-local-symlink-subtree-rootredirect.sh --- old/MirrorCache-1.102/t/environ/15-local-symlink-subtree-rootredirect.sh 2025-10-16 10:45:34.000000000 +0200 +++ new/MirrorCache-1.104/t/environ/15-local-symlink-subtree-rootredirect.sh 2025-11-06 15:13:54.000000000 +0100 @@ -45,6 +45,7 @@ MIRRORCACHE_SUBTREE=/updates \ MIRRORCACHE_TOP_FOLDERS=tool \ MIRRORCACHE_VPN_PREFIX='"11. 10.10"' \ + MIRRORCACHE_VPN_PREFIX_NEG='"10.10.99"' \ MIRRORCACHE_ROOT_COUNTRY=us $mcsub/start @@ -64,9 +65,9 @@ $mcsub/curl /tool/v1/file1.1.dat.mirrorlist | grep -C30 $($ap7/print_address)/folder1/file1.1.dat | grep -C30 $($ap8/print_address)/updates/tool/v1/file1.1.dat | grep testhost.com/folder1/file1.1.dat $mcsub/curl /tool/v2/file1.1.dat.mirrorlist | grep -C30 $($ap8/print_address)/updates/tool/v2/file1.1.dat | grep testhost.com/folder2/file1.1.dat -$mcsub/curl /tool/v1/file1.1.dat.metalink?IP=10.11.10.1 | grep testhost +$mcsub/curl /tool/v1/file1.1.dat.metalink?IP=10.10.99.1 | grep testhost.com $mcsub/curl /tool/v1/file1.1.dat.metalink?IP=10.11.10.1 | grep testhost.com -$mcsub/curl /tool/v1/file1.1.dat.metalink?IP=10.10.10.1 | grep testhost +$mcsub/curl /tool/v1/file1.1.dat.metalink?IP=10.10.10.1 | grep testhost.vpn rc=0 $mcsub/curl /tool/v1/file1.1.dat.metalink?IP=10.10.10.1 | grep testhost.com || rc=$? test $rc -gt 0 ++++++ MirrorCache.obsinfo ++++++ --- /var/tmp/diff_new_pack.PgbPJe/_old 2025-11-13 18:17:35.834546673 +0100 +++ /var/tmp/diff_new_pack.PgbPJe/_new 2025-11-13 18:17:35.842547010 +0100 @@ -1,5 +1,5 @@ name: MirrorCache -version: 1.102 -mtime: 1760604334 -commit: 446d1ea62d87818cb75e16abfbebe5ffc5f0a250 +version: 1.104 +mtime: 1762438434 +commit: 207d61237c0597f8f4ff9d7ad12c4f9cb5d5cd1f
