Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2023-04-20 15:15:00
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.2023 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Thu Apr 20 15:15:00 2023 rev:26 rq:1080817 version:1.058
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2023-04-13
14:11:02.404397354 +0200
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2023/MirrorCache.changes
2023-04-20 15:16:09.226545709 +0200
@@ -1,0 +2,8 @@
+Thu Apr 13 10:36:14 UTC 2023 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.058:
+ * Add Vary header into http responses (#373)
+ * Add experimental feature mirrorlist for folder (#372)
+ * Fix redirect on remote MirrorCache (#372)
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.057.obscpio
New:
----
MirrorCache-1.058.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.6je53u/_old 2023-04-20 15:16:09.850549942 +0200
+++ /var/tmp/diff_new_pack.6je53u/_new 2023-04-20 15:16:09.858549997 +0200
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
perl(Config::IniFiles)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.057
+Version: 1.058
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.057.obscpio -> MirrorCache-1.058.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.057/lib/MirrorCache/Schema/ResultSet/Server.pm
new/MirrorCache-1.058/lib/MirrorCache/Schema/ResultSet/Server.pm
--- old/MirrorCache-1.057/lib/MirrorCache/Schema/ResultSet/Server.pm
2023-04-05 11:42:32.000000000 +0200
+++ new/MirrorCache-1.058/lib/MirrorCache/Schema/ResultSet/Server.pm
2023-04-13 12:16:03.000000000 +0200
@@ -100,12 +100,23 @@
$condition_server_project = "and sp.server_id IS NULL";
}
+ my $join_file_cond = "fl.folder_id = fd.folder_id";
+ my $file_dt = ", max(case when fdf.file_id is null and fl.name ~ '[0-9]'
and fl.name not like '%license.tar.gz' and fl.name not like '%info.xml.gz' then
fl.mtime else null end) as mtime";
+ my $group_by = "group by s.id, s.hostname, s.hostname_vpn, s.urldir,
s.region, s.country, s.lat, s.lng, s.score";
+
+ if ($file_id) {
+ $join_file_cond = "fl.id = ?";
+ $file_dt = ", fl.mtime as mtime";
+ $group_by = "";
+ }
+
my $sql = <<"END_SQL";
select * from (
select x.id as mirror_id,
case when support_scheme > 0 then '$capability' else '$capabilityx' end as
scheme,
hostname,
-uri,
+urldir,
+mtime,
dist,
case $weight_country_case when region $avoid_region= '$region' then 1 else 0
end rating_country,
score, country, region, lat, lng,
@@ -115,7 +126,8 @@
rating_ipv
from (
select s.id, $hostname as hostname,
- left(concat(s.urldir,f.path,'/',s.name),$MIRRORCACHE_MAX_PATH) as uri,
+ left(concat(s.urldir,f.path),$MIRRORCACHE_MAX_PATH) as urldir,
+s.mtime,
s.lat as lat,
s.lng as lng,
case when $lat=0 and $lng=0 then 0
@@ -129,14 +141,15 @@
CASE WHEN COALESCE(stability_ipv.rating, 0) > 0 THEN 1 ELSE 0 END AS
support_ipv,
CASE WHEN COALESCE(stability_ipv.rating, 0) > 0 THEN stability_ipv.rating
WHEN COALESCE(stability_ipvx.rating, 0) > 0 THEN stability_ipvx.rating
ELSE 0 END AS rating_ipv
from (
- select s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country,
s.lat, s.lng, s.score, fl.name
+ select s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country,
s.lat, s.lng, s.score $file_dt
from folder_diff fd
- join file fl on fl.id = ?
+ join file fl on $join_file_cond
join folder_diff_server fds on fd.id = fds.folder_diff_id and
date_trunc('second', fl.dt) <= fds.dt
join server s on fds.server_id = s.id and s.enabled $country_condition
left join folder_diff_file fdf on fdf.file_id = fl.id and
fdf.folder_diff_id = fd.id
$join_server_project
where fd.folder_id = ? and fdf.file_id is NULL $condition_server_project
+ $group_by
) s
join folder f on f.id = ?
left join server_capability_declaration scd on s.id = scd.server_id and
scd.capability = '$capability' and NOT scd.enabled
@@ -159,10 +172,15 @@
$sql =~ s/::int//g unless
($dbh->{Driver}->{Name} eq 'Pg');
$sql =~ s/random/rand/g unless
($dbh->{Driver}->{Name} eq 'Pg');
$sql =~ s/date_trunc\('second', fl.dt\)/cast(fl.dt as DATETIME)/g unless
($dbh->{Driver}->{Name} eq 'Pg');
+ $sql =~ s/fl.name \~ /fl.name REGEXP /g unless
($dbh->{Driver}->{Name} eq 'Pg');
my $prep = $dbh->prepare($sql);
- $prep->execute($file_id, @country_params, $folder_id, $folder_id);
+ if ($file_id) {
+ $prep->execute($file_id, @country_params, $folder_id, $folder_id);
+ } else {
+ $prep->execute(@country_params, $folder_id, $folder_id);
+ }
my $server_arrayref = $dbh->selectall_arrayref($prep, { Slice => {} });
return $server_arrayref;
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.057/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.058/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.057/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2023-04-05
11:42:32.000000000 +0200
+++ new/MirrorCache-1.058/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2023-04-13
12:16:03.000000000 +0200
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2022 SUSE LLC
+# Copyright (C) 2020-2023 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -305,45 +305,51 @@
$c->log->error($c->dumper('$file_pattern_in_folder',
$file_pattern_in_folder)) if $MCDEBUG;
my $it_must_be_folder = ( $trailing_slash || $path eq '/');
my $folder_or_pattern = $it_must_be_folder || $file_pattern_in_folder;
- { # this bracked to simplify diff
+
+ { # this bracket is here to simplify diff
my $f = Mojo::File->new($path);
my $dirname = ($folder_or_pattern? $path : $f->dirname);
$dirname = $root->realpath($dirname);
$dirname = $dm->root_subtree . ($folder_or_pattern? $path :
$f->dirname) unless $dirname;
$c->log->error($c->dumper('dirname:', $dirname, 'path:', $path,
'trail:', $trailing_slash)) if $MCDEBUG;
- if (my $parent_folder = $rsFolder->find_folder_or_redirect($dirname)) {
- my $realpath_subtree;
- if ($root->is_remote && $parent_folder && $parent_folder->{path}
ne $dirname) {
- $realpath_subtree = $parent_folder->{path};
+ if (my $folder = $rsFolder->find_folder_or_redirect($dirname)) {
+ $c->log->error('found redirect : ', $folder->{pathto}) if $MCDEBUG
&& $folder->{pathto};
+ # return $dm->redirect($folder->{pathto} . $trailing_slash) if
$folder->{pathto};
+ my $folder_path = $folder->{pathto} ? $folder->{pathto} :
$folder->{path};
+ return $c->render(status => 404, text => "path {$path} not
found!!") unless $folder_path;
+ my $realpath_subtree = '';
+ if ($root->is_remote && $folder_path ne $dirname) {
+ $realpath_subtree = $folder_path;
} else {
$realpath_subtree = $root->realpath($dm->root_subtree .
($folder_or_pattern? $path : $f->dirname)) // $dirname;
}
if ($dirname eq $realpath_subtree) {
if ($dirname eq $f->dirname || $folder_or_pattern) {
- $dm->folder_id($parent_folder->{id});
- $dm->folder_sync_last($parent_folder->{sync_last});
- $dm->folder_scan_last($parent_folder->{scan_last});
+ $dm->folder_id($folder->{id});
+ $dm->folder_sync_last($folder->{sync_last});
+ $dm->folder_scan_last($folder->{scan_last});
}
} else {
- my $another_folder = $rsFolder->find({path =>
$dm->root_subtree . $f->dirname});
+ my $another_folder = $rsFolder->find({path =>
$realpath_subtree});
$c->log->error($c->dumper('another_folder:',
$another_folder->{id})) if $MCDEBUG;
return undef unless $another_folder || $it_must_be_folder; #
nothing found, proceed to _guess_what_to_render
- if ($parent_folder) {
- $dm->real_folder_id($parent_folder->{id});
- $dm->folder_id($another_folder->{id}) if $another_folder;
- $dm->folder_sync_last($parent_folder->{sync_last});
- $dm->folder_scan_last($parent_folder->{scan_last});
+ $dm->folder_id($another_folder->{id}) if $another_folder;
+ if ($folder->{id}) {
+ $dm->real_folder_id($folder->{id});
+ $dm->folder_sync_last($folder->{sync_last});
+ $dm->folder_scan_last($folder->{scan_last});
}
}
if ($it_must_be_folder && !$file_pattern_in_folder) {
- $dm->real_folder_id($parent_folder->{id});
+ $dm->real_folder_id($folder->{id}) if $folder->{id};
+ return $c->mirrorcache->render_dir_mirrorlist($path, $dm) if
$dm->mirrorlist;
return _render_dir($dm, $path, $rsFolder);
}
my $xtra = '';
$xtra = '.zsync' if $dm->zsync && !$dm->accept_zsync;
my $file;
- $c->log->error($c->dumper('parent_folder:',
$parent_folder->{path})) if $MCDEBUG && $parent_folder;
- $file =
$schema->resultset('File')->find_with_hash($parent_folder->{id}, $f->basename,
$xtra, $dm->regex, $dm->glob_regex) if $parent_folder;
+ $c->log->error($c->dumper('parent_folder:', $folder->{path})) if
$MCDEBUG && $folder && $folder->{path};
+ $file = $schema->resultset('File')->find_with_hash($folder->{id},
$f->basename, $xtra, $dm->regex, $dm->glob_regex) if $folder && $folder->{id};
$c->log->error($c->dumper('file:', $f->basename, $file)) if
$MCDEBUG;
# folders are stored with trailing slash in file table, so they
will not be selected here
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.057/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.058/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.057/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2023-04-05 11:42:32.000000000 +0200
+++ new/MirrorCache-1.058/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2023-04-13 12:16:03.000000000 +0200
@@ -37,6 +37,38 @@
$app->types->type(meta4 => 'application/metalink4+xml; charset=UTF-8');
$app->types->type(zsync => 'application/x-zsync');
+ $app->helper( 'mirrorcache.render_dir_mirrorlist' => sub {
+ my ($c, $path, $dm)= @_;
+ my $folder_id = $dm->folder_id;
+ unless ($folder_id) {
+ return $c->render(status => 404, text => "Folder not found");
+ return 1;
+ }
+ my (@mirrors_country, @mirrors_region, @mirrors_rest);
+ my $project_id = $c->mcproject->get_id($path);
+ my $cnt = _collect_mirrors($dm, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, undef, undef, $folder_id, $project_id);
+
+ return $c->render(status => 204, text => 'No mirrors found') unless
$cnt;
+ my @mirrors;
+ my $prio = 0;
+ for my $m (@mirrors_country, @mirrors_region, @mirrors_rest) {
+ my %h;
+ $h{url} = $m->{url};
+ $h{prio} = $prio++;
+ if (my $mtime = $m->{mtime}) {
+ $h{mtime} = $mtime;
+ eval {
+ my $dt = strftime("%Y-%m-%d %H:%M:%S", localtime($mtime));
+ $h{time} = $dt;
+ };
+ };
+
+ push @mirrors, \%h;
+ }
+
+ return $c->render(json => \@mirrors);
+ });
+
$app->helper( 'mirrorcache.render_file' => sub {
my ($c, $filepath, $dm, $file)= @_;
$c->log->error($c->dumper('RENDER START', $filepath)) if $MCDEBUG;
@@ -113,6 +145,7 @@
return $c->render(status => 404, text => "File not found");
}
$c->log->error($c->dumper('RENDER FILE_ID', $file->{id})) if $MCDEBUG;
+ $c->res->headers->vary('Accept, COUNTRY');
my $baseurl; # just hostname + eventual urldir (without folder and
file)
my $fullurl; # baseurl with path and filename
if ($dm->metalink || $dm->meta4 || $dm->torrent || $dm->zsync ||
$dm->magnet) {
@@ -145,12 +178,12 @@
my (@mirrors_country, @mirrors_region, @mirrors_rest);
my $project_id = $c->mcproject->get_id($dirname);
- _collect_mirrors($dm, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, $file->{id}, $folder_id, $project_id);
+ _collect_mirrors($dm, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, $file->{id}, $file->{name}, $folder_id, $project_id);
# add mirrors that have realpath
if ($realfolder_id && $realfolder_id != $folder_id) {
my $realproject_id = $c->mcproject->get_id($realdirname);
- _collect_mirrors($dm, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, $file->{id}, $realfolder_id, $realproject_id);
+ _collect_mirrors($dm, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, $file->{id}, $file->{name}, $realfolder_id, $realproject_id);
}
my $mirror;
$mirror = $mirrors_country[0] if @mirrors_country;
@@ -187,14 +220,14 @@
my $xml;
if ($dm->meta4) {
$xml = _build_meta4(
- $dm, ($folder? $folder->path : $realdirname), $file,
$country, $region, \@mirrors_country, \@mirrors_region,
+ $dm, (($folder && $folder->path)? $folder->path :
$realdirname), $file, $country, $region, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, $origin, 'MirrorCache', $baseurl);
$c->res->headers->content_disposition('attachment; filename="'
.$basename. '.meta4"');
$c->render(data => $xml, format => 'meta4');
return 1;
}
$xml = _build_metalink(
- $dm, ($folder? $folder->path : $realdirname), $file, $country,
$region, \@mirrors_country, \@mirrors_region,
+ $dm, (($folder && $folder->path)? $folder->path :
$realdirname), $file, $country, $region, \@mirrors_country, \@mirrors_region,
\@mirrors_rest, $origin, 'MirrorCache', $baseurl);
$c->res->headers->content_disposition('attachment; filename="'
.$basename. '.metalink"');
$c->render(data => $xml, format => 'metalink');
@@ -698,7 +731,7 @@
}
sub _collect_mirrors {
- my ($dm, $mirrors_country, $mirrors_region, $mirrors_rest, $file_id,
$folder_id, $project_id) = @_;
+ my ($dm, $mirrors_country, $mirrors_region, $mirrors_rest, $file_id,
$file_name, $folder_id, $project_id) = @_;
my $country = $dm->country;
my $region = $dm->region;
@@ -758,7 +791,7 @@
}
}
for $m (@$mirrors_country, @$mirrors_region, @$mirrors_rest) {
- $m->{url} = $m->{scheme} . '://' . $m->{hostname} .
Mojo::Util::url_escape($m->{uri}, '^A-Za-z0-9\-._~/');
+ $m->{url} = $m->{scheme} . '://' . $m->{hostname} .
Mojo::Util::url_escape($m->{urldir} . '/' . $file_name, '^A-Za-z0-9\-._~/');
}
return $found_count;
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.057/t/environ/03-geo.sh
new/MirrorCache-1.058/t/environ/03-geo.sh
--- old/MirrorCache-1.057/t/environ/03-geo.sh 2023-04-05 11:42:32.000000000
+0200
+++ new/MirrorCache-1.058/t/environ/03-geo.sh 2023-04-13 12:16:03.000000000
+0200
@@ -53,7 +53,7 @@
$mc/curl --interface 127.0.0.3 -Is /download/folder1/file1.1.dat?COUNTRY=ca |
grep 1304
$mc/curl --interface 127.0.0.2 -Is /download/folder1/file1.1.dat?COUNTRY=ca |
grep 1304
-# Further we test that servers are listed only once in metalink output
+echo Further we test that servers are listed only once in metalink output
$mc/curl -H "Accept: */*, application/metalink+xml" --interface 127.0.0.2
/download/folder1/file1.1.dat
duplicates=$($mc/curl -H "Accept: */*, application/metalink+xml" --interface
127.0.0.2 /download/folder1/file1.1.dat | grep location | grep -E -o
'https?[^"s][^\<]*' | sort | uniq -cd | wc -l)
@@ -61,12 +61,17 @@
$mc/curl -H "Accept: */*, application/metalink+xml" --interface 127.0.0.2 -s
/download/folder1/file1.1.dat | grep -B20 127.0.0.2 | grep -i 'this country
(us)'
-# test get parameter COUNTRY
+echo test get parameter COUNTRY
$mc/curl -H "Accept: */*, application/metalink+xml" --interface 127.0.0.2 -s
/download/folder1/file1.1.dat?COUNTRY=DE | grep -B20 127.0.0.3 | grep -i 'this
country (de)'
-# test get parameter AVOID_COUNTRY
+echo test get parameter AVOID_COUNTRY
$mc/curl -H "Accept: */*, application/metalink+xml" --interface 127.0.0.2 -s
/download/folder1/file1.1.dat?AVOID_COUNTRY=DE,US | grep 127.0.0.4
-# check continent
+echo check continent
$mc/curl -H "Accept: */*, application/metalink+xml" --interface 127.0.0.2 -s
/download/folder1/file1.1.dat?COUNTRY=fr | grep -B20 127.0.0.3
$mc/curl -H "Accept: */*, application/metalink4+xml" --interface 127.0.0.2 -s
/download/folder1/file1.1.dat?COUNTRY=fr | grep -B20 127.0.0.3
+
+echo check metalink for folder
+$mc/curl -i /download/folder1/?mirrorlist | grep -F
'"url":"http:\/\/127.0.0.2:1304\/folder1\/"' | grep -F
'"url":"http:\/\/127.0.0.3:1314\/folder1\/"'
+
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.057/t/environ/03-headquarter-subsidiaries-remote.sh
new/MirrorCache-1.058/t/environ/03-headquarter-subsidiaries-remote.sh
--- old/MirrorCache-1.057/t/environ/03-headquarter-subsidiaries-remote.sh
2023-04-05 11:42:32.000000000 +0200
+++ new/MirrorCache-1.058/t/environ/03-headquarter-subsidiaries-remote.sh
2023-04-13 12:16:03.000000000 +0200
@@ -82,6 +82,18 @@
echo filesmall is served right away
$mc9/curl -I --interface $eu_interface /folder1/repodata/filesmall1.1.dat |
grep '200 OK'
+echo check VARY header except in small files
+rc=?
+$mc9/curl -I --interface $eu_interface /folder1/repodata/filesmall1.1.dat |
grep -i vary: || rc=$?
+test $rc -gt 0
+rc=?
+$mc9/curl -I --interface $hq_interface /folder1/repodata/filesmall1.1.dat |
grep -i vary: || rc=$?
+test $rc -gt 0
+
+$mc9/curl -I --interface $hq_interface /folder1/repodata/filebig1.1.dat
| grep -i vary: | grep 'Accept, COUNTRY'
+$mc9/curl -I --interface $hq_interface
/folder1/repodata/filebig1.1.dat.metalink | grep -i vary: | grep 'Accept,
COUNTRY'
+
+
echo check huge files are redirected to FAKEURL, but we need to scan folder
first
$mc9/curl -I --interface $hq_interface
/download/folder1/repodata/filehuge1.1.dat | grep "Location:
http://$FAKEURL/folder1/repodata/filehuge1.1.dat";
$mc9/curl -I --interface $hq_interface
/download/folder1/repodata/filebig1.1.dat | grep "Location: " | grep
$($root/print_address) # we removed it from mirror
@@ -94,4 +106,3 @@
$mc9/curl -I --interface $hq_interface
/download/folder1/repodata/filehuge2.1.dat | grep
$($mirror/print_address)/folder1/repodata/filehuge2.1.dat # this is on mirror
echo success
-
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.6je53u/_old 2023-04-20 15:16:10.362553416 +0200
+++ /var/tmp/diff_new_pack.6je53u/_new 2023-04-20 15:16:10.366553443 +0200
@@ -1,5 +1,5 @@
name: MirrorCache
-version: 1.057
-mtime: 1680687752
-commit: ebc5c1979916df7a82a430f0766f706a3296282f
+version: 1.058
+mtime: 1681380963
+commit: e3518bbd8cde6c4fb950e1a8eb434b320b8f2799
