Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2021-12-23 17:53:49
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.2520 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Thu Dec 23 17:53:49 2021 rev:7 rq:942193 version:1.022
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2021-12-09
19:46:22.509155376 +0100
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2520/MirrorCache.changes
2021-12-23 17:54:05.431731800 +0100
@@ -1,0 +2,23 @@
+Thu Dec 16 07:13:04 UTC 2021 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.022:
+ * Add experimental route geoip to rewrite baseurl in .repo (#242)
+ * Set MirrorCache useragent for scan jobs (#240)
+ * Add RestartSec to backstage systemd files (#241)
+
+-------------------------------------------------------------------
+Sun Dec 12 13:09:46 UTC 2021 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.021:
+ * Handle timeouts in mirror_probe_projects (#238)
+ * Track projects on mirrors and do skip scans accordingly (#237)
+
+-------------------------------------------------------------------
+Thu Dec 09 06:22:21 UTC 2021 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.020:
+ * Do not try to find mirror outside region when ROOT_COUNTRY is set (#236)
+ * Special handling unversioned media symlinks (#235)
+ * Import hash create folder (#233)
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.019.obscpio
New:
----
MirrorCache-1.022.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.U82QS7/_old 2021-12-23 17:54:05.891732086 +0100
+++ /var/tmp/diff_new_pack.U82QS7/_new 2021-12-23 17:54:05.895732088 +0100
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.019
+Version: 1.022
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.019.obscpio -> MirrorCache-1.022.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
new/MirrorCache-1.022/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
---
old/MirrorCache-1.019/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
1970-01-01 01:00:00.000000000 +0100
+++
new/MirrorCache-1.022/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
2021-12-16 07:57:47.000000000 +0100
@@ -0,0 +1,26 @@
+insert into project(name,path) select 'repositories','/repositories';
+insert into project(name,path) select 'TW ISOs','/tumbleweed/iso';
+insert into project(name,path) select 'TW repo','/tumbleweed/repo';
+insert into project(name,path) select 'TW debug','/debug/tumbleweed/repo';
+insert into project(name,path) select 'TW source','/source/tumbleweed/repo';
+insert into project(name,path) select 'TW history','/history';
+
+insert into project(name,path) select '15.2 ISO','/distribution/leap/15.2/iso';
+insert into project(name,path) select '15.2
repo','/distribution/leap/15.2/repo';
+insert into project(name,path) select '15.2
debug','/debug/distribution/leap/15.2/repo';
+insert into project(name,path) select '15.2
source','/source/distribution/leap/15.2/repo';
+insert into project(name,path) select '15.2
update','/update/distribution/leap/15.2/repo';
+
+insert into project(name,path) select '15.3 ISO','/distribution/leap/15.3/iso';
+insert into project(name,path) select '15.3
repo','/distribution/leap/15.3/repo';
+insert into project(name,path) select '15.3
debug','/debug/distribution/leap/15.3/repo';
+insert into project(name,path) select '15.3
source','/source/distribution/leap/15.3/repo';
+insert into project(name,path) select '15.3
update','/update/distribution/leap/15.3/repo';
+insert into project(name,path) select '15.3
port','/port/aarch64/distribution/leap/15.3';
+
+insert into project(name,path) select '15.4 ISO','/distribution/leap/15.4/iso';
+insert into project(name,path) select '15.4
repo','/distribution/leap/15.4/repo';
+insert into project(name,path) select '15.4
debug','/debug/distribution/leap/15.4/repo';
+insert into project(name,path) select '15.4
source','/source/distribution/leap/15.4/repo';
+insert into project(name,path) select '15.4
update','/update/distribution/leap/15.4/repo';
+insert into project(name,path) select '15.4
port','/port/aarch64/distribution/leap/15.4';
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/dist/systemd/mirrorcache-backstage-hashes.service
new/MirrorCache-1.022/dist/systemd/mirrorcache-backstage-hashes.service
--- old/MirrorCache-1.019/dist/systemd/mirrorcache-backstage-hashes.service
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/dist/systemd/mirrorcache-backstage-hashes.service
2021-12-16 07:57:47.000000000 +0100
@@ -7,6 +7,7 @@
ExecStart=/usr/share/mirrorcache/script/mirrorcache-backstage-hashes
Nice=19
Restart=on-failure
+RestartSec=10
EnvironmentFile=/etc/mirrorcache/conf.env
EnvironmentFile=-/etc/mirrorcache/conf-hashes.env
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/dist/systemd/mirrorcache-backstage.service
new/MirrorCache-1.022/dist/systemd/mirrorcache-backstage.service
--- old/MirrorCache-1.019/dist/systemd/mirrorcache-backstage.service
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/dist/systemd/mirrorcache-backstage.service
2021-12-16 07:57:47.000000000 +0100
@@ -8,6 +8,7 @@
ExecStart=/usr/share/mirrorcache/script/mirrorcache-backstage
Nice=19
Restart=on-failure
+RestartSec=10
EnvironmentFile=/etc/mirrorcache/conf.env
[Install]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/lib/MirrorCache/Datamodule.pm
new/MirrorCache-1.022/lib/MirrorCache/Datamodule.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Datamodule.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Datamodule.pm 2021-12-16
07:57:47.000000000 +0100
@@ -230,7 +230,14 @@
}
sub redirect($self, $url) {
- return $self->c->redirect_to($url . $self->query1);
+ my $xtra = '';
+ if ($self->_original_path =~ m/.*\.metalink$/) {
+ $xtra = ".metalink";
+ } elsif ($self->_original_path =~ m/.*\.mirrorlist$/) {
+ $xtra = ".mirrorlist";
+ }
+
+ return $self->c->redirect_to($url . $xtra . $self->query1);
}
sub _init_headers($self) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Schema/Result/File.pm
new/MirrorCache-1.022/lib/MirrorCache/Schema/Result/File.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Schema/Result/File.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Schema/Result/File.pm 2021-12-16
07:57:47.000000000 +0100
@@ -77,6 +77,8 @@
{ data_type => "bigint", is_nullable => 1 },
"mtime",
{ data_type => "bigint", is_nullable => 1 },
+ "target",
+ { data_type => "varchar", is_nullable => 1, size => 512 },
"dt",
{
data_type => 'timestamp',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Schema/Result/Hash.pm
new/MirrorCache-1.022/lib/MirrorCache/Schema/Result/Hash.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Schema/Result/Hash.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Schema/Result/Hash.pm 2021-12-16
07:57:47.000000000 +0100
@@ -30,6 +30,8 @@
{ data_type => "int" },
"pieces",
{ data_type => "text" },
+ "target",
+ { data_type => "varchar", size => 512 },
"dt",
{
data_type => 'timestamp',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/File.pm
new/MirrorCache-1.022/lib/MirrorCache/Schema/ResultSet/File.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/File.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Schema/ResultSet/File.pm
2021-12-16 07:57:47.000000000 +0100
@@ -30,9 +30,10 @@
# html parser may loose seconds from file.mtime, so we allow hash.mtime
differ for up to 1 min for now
my $sql = <<'END_SQL';
select file.id, file.folder_id, file.name,
-case when coalesce(file.size, 0::bigint) = 0::bigint and coalesce(hash.size,
0::bigint) != 0::bigint then hash.size else file.size end,
-case when coalesce(file.mtime, 0::bigint) = 0::bigint and coalesce(hash.mtime,
0::bigint) != 0::bigint then hash.mtime else file.mtime end,
-file.mtime, file.dt, hash.mtime as hash_mtime, hash.md5, hash.sha1,
hash.sha256, hash.piece_size, hash.pieces,
+case when coalesce(file.size, 0::bigint) = 0::bigint and coalesce(hash.size,
0::bigint) != 0::bigint then hash.size else file.size end size,
+case when coalesce(file.mtime, 0::bigint) = 0::bigint and coalesce(hash.mtime,
0::bigint) != 0::bigint then hash.mtime else file.mtime end mtime,
+coalesce(hash.target, file.target) target,
+file.dt, hash.md5, hash.sha1, hash.sha256, hash.piece_size, hash.pieces,
(DATE_PART('day', now() - file.dt) * 24 * 3600 +
DATE_PART('hour', now() - file.dt) * 3600 +
DATE_PART('minute', now() - file.dt) * 60 +
@@ -44,8 +45,12 @@
or
(coalesce(file.size, 0::bigint) = 0::bigint and coalesce(hash.size,
0::bigint) != 0::bigint and file.dt <= hash.dt)
)
-where file.folder_id = ? and name = ?
+where file.folder_id = ?
END_SQL
+
+ return $dbh->selectall_hashref($sql, 'id', {}, $folder_id) unless $name;
+
+ $sql = $sql . " and file.name = ?";
my $prep = $dbh->prepare($sql);
$prep->execute($folder_id, $name);
return $dbh->selectrow_hashref($prep);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Hash.pm
new/MirrorCache-1.022/lib/MirrorCache/Schema/ResultSet/Hash.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Hash.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Schema/ResultSet/Hash.pm
2021-12-16 07:57:47.000000000 +0100
@@ -21,15 +21,15 @@
use base 'DBIx::Class::ResultSet';
sub store {
- my ($self, $file_id, $mtime, $size, $md5hex, $sha1hex, $sha256hex,
$block_size, $pieceshex) = @_;
+ my ($self, $file_id, $mtime, $size, $md5hex, $sha1hex, $sha256hex,
$block_size, $pieceshex, $target) = @_;
my $rsource = $self->result_source;
my $schema = $rsource->schema;
my $dbh = $schema->storage->dbh;
my $sql = <<'END_SQL';
-insert into hash(file_id, mtime, size, md5, sha1, sha256, piece_size, pieces,
dt)
-values (?, ?, ?, ?, ?, ?, ?, ?, now())
+insert into hash(file_id, mtime, size, md5, sha1, sha256, piece_size, pieces,
target, dt)
+values (?, ?, ?, ?, ?, ?, ?, ?, ?, now())
ON CONFLICT (file_id) DO UPDATE
SET size = excluded.size,
mtime = excluded.mtime,
@@ -38,10 +38,11 @@
sha256 = excluded.sha256,
piece_size = excluded.piece_size,
pieces = excluded.pieces,
+ target = excluded.target,
dt = now()
END_SQL
my $prep = $dbh->prepare($sql);
- $prep->execute($file_id, $mtime, $size, $md5hex, $sha1hex, $sha256hex,
$block_size, $pieceshex);
+ $prep->execute($file_id, $mtime, $size, $md5hex, $sha1hex, $sha256hex,
$block_size, $pieceshex, $target);
}
sub hashes_since {
@@ -59,7 +60,7 @@
}
my $sql = <<"END_SQL";
-select file.name, hash.mtime, hash.size, md5, sha1, sha256, piece_size,
pieces, hash.dt
+select file.name, hash.mtime, hash.size, md5, sha1, sha256, piece_size,
pieces, hash.target, hash.dt
from hash left join file on file_id = id
where file_id in ( select id from file where folder_id = ? )
$time_constraint_condition limit 100000
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Server.pm
new/MirrorCache-1.022/lib/MirrorCache/Schema/ResultSet/Server.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Server.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Schema/ResultSet/Server.pm
2021-12-16 07:57:47.000000000 +0100
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 SUSE LLC
+# Copyright (C) 2020,2021 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -174,9 +174,12 @@
left join folder_diff fd on fd.folder_id = f.id
left join folder_diff_server fds on fd.id = fds.folder_diff_id and
fds.server_id=s.id
left join server_capability_declaration cap_hasall on cap_hasall.server_id =
s.id and cap_hasall.capability = 'hasall' and cap_hasall.enabled
+left join project p on f.path like concat(p.path, '%')
+left join server_project sp on (sp.server_id, sp.project_id) = (s.id, p.id)
and sp.state < 1
where
(fds.folder_diff_id IS NOT DISTINCT FROM fd.id OR fds.server_id is null)
AND (cap_fhttp.server_id IS NULL or cap_fhttps.server_id IS NULL)
+AND (sp.server_id IS NULL)
group by s.id, s.hostname, s.urldir, f.path, cap_http.server_id,
cap_fhttp.server_id, cap_hasall.capability
order by s.id
END_SQL
@@ -188,4 +191,49 @@
return $server_arrayref;
}
+sub server_projects {
+ my ($self) = @_;
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql = <<'END_SQL';
+select concat(s.id, '::', p.id) as key,
+ s.id as server_id,
+ p.id as project_id,
+ concat(CASE WHEN length(s.hostname_vpn)>0 THEN s.hostname_vpn ELSE
s.hostname END,s.urldir, p.path) as uri,
+ sp.server_id as mirror_id,
+ coalesce(sp.state, -2) oldstate
+from project p
+ join server s on s.enabled
+ left join server_project sp on sp.server_id = s.id and sp.project_id = p.id
+where 't'
+ AND coalesce(sp.state,0) > -1
+END_SQL
+ return $dbh->selectall_hashref($sql, 'key', {});
+}
+
+sub log_project_probe_outcome {
+ my ($self, $server_id, $project_id, $mirror_id, $state, $extra) = @_;
+
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql = <<'END_SQL';
+insert into server_project(state, extra, dt, server_id, project_id)
+values (?, ?, now(), ?, ?);
+END_SQL
+
+ if ($mirror_id) {
+ $sql = <<'END_SQL';
+update server_project set state = ?, extra = ?, dt = now()
+where server_id = ? and project_id = ?;
+END_SQL
+ }
+
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($state, $extra, $server_id, $project_id);
+}
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/lib/MirrorCache/Task/Cleanup.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/Cleanup.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/Cleanup.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/Cleanup.pm 2021-12-16
07:57:47.000000000 +0100
@@ -31,6 +31,7 @@
unless my $guard = $minion->guard('cleanup', 120);
my $schema = $app->schema;
+ $minion->enqueue('mirror_probe_projects');
# purge unreferenced folder_diff
my $sql = <<'END_SQL';
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesCreate.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/FolderHashesCreate.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesCreate.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/FolderHashesCreate.pm
2021-12-16 07:57:47.000000000 +0100
@@ -81,6 +81,17 @@
sub calcMetalink {
my ($indir, $path, $file, $block_size, $schema, $file_id) = @_;
my $f = "$indir$path/$file";
+
+ my $target;
+ eval {
+ my $dest = readlink($f);
+ if ($dest) {
+ $dest = Mojo::File->new($dest);
+ $target = $dest->basename if $dest->dirname eq '.' ||
$dest->dirname eq "$indir$path";
+ }
+ };
+ return $schema->resultset('Hash')->store($file_id, undef, 0, undef, undef,
undef, undef, undef, $target) if $target;
+
open my $fh, "<", $f or die "Unable to open $f!";
my $mtime = (stat($fh))[9];
my $d1 = Digest::SHA->new(1);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesImport.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/FolderHashesImport.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesImport.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/FolderHashesImport.pm
2021-12-16 07:57:47.000000000 +0100
@@ -51,7 +51,7 @@
$hq_url = "http://"; . $hq_url unless 'http' eq substr($hq_url, 0, 4);
my $mojo_url = Mojo::URL->new($hq_url);
- my $res = Mojo::UserAgent->new->get($mojo_url)->result;
+ my $res = Mojo::UserAgent->new->get($mojo_url, {'User-Agent' =>
'MirrorCache/hashes_import'})->result;
return $job->fail('Request to HEADQUARTER ' . $hq_url . ' failed, response
code ' . $res->code)
if $res->code != 200;
@@ -64,7 +64,7 @@
next unless $file;
eval {
$schema->resultset('Hash')->store($file->id, $hash->{mtime},
$hash->{size}, $hash->{md5},
- $hash->{sha1}, $hash->{sha256}, $hash->{piece_size},
$hash->{pieces});
+ $hash->{sha1}, $hash->{sha256}, $hash->{piece_size},
$hash->{pieces}, $hash->{target});
$last_import = Mojo::Date($hash->{dt}) if ($last_import &&
$hash->{dt} && $last_import->epoch < Mojo::Date->new($hash->{dt})->epoch);
$count++;
};
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/lib/MirrorCache/Task/FolderSync.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/FolderSync.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/FolderSync.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/FolderSync.pm 2021-12-16
07:57:47.000000000 +0100
@@ -92,12 +92,12 @@
} else {
my $count = 0;
my $sub = sub {
- my ($file, $size, $mmode, $mtime) = @_;
+ my ($file, $size, $mmode, $mtime, $target) = @_;
$file = $file . '/' if !$root->is_remote && $path ne '/' &&
$root->is_dir("$path/$file");
$file = $file . '/' if !$root->is_remote && $path eq '/' &&
$root->is_dir("$path$file");
$file = $file . '/' if $mmode && $root->is_remote && $mmode < 1000;
$count = $count+1;
- $schema->resultset('File')->create({folder_id => $folder->id, name
=> $file, size => $size, mtime => $mtime});
+ $schema->resultset('File')->create({folder_id => $folder->id, name
=> $file, size => $size, mtime => $mtime, target => $target});
return undef;
};
eval {
@@ -110,7 +110,7 @@
$update_db_last->();
eval {
$schema->txn_do(sub {
- $app->mc->root->foreach_filename($realpath, $sub);
+ $root->foreach_filename($realpath, $sub);
});
1;
} or return $job->fail('Error while reading files from root :' . $@);
@@ -131,36 +131,42 @@
my %dbfileids = ();
my %dbfilesizes = ();
my %dbfilemtimes = ();
+ my %dbfiletargets = ();
for my $file ($schema->resultset('File')->search({folder_id =>
$folder_id})) {
my $basename = $file->name;
next unless $basename;
$dbfileids{$basename} = $file->id;
$dbfilesizes{$basename} = $file->size if defined $file->size;
$dbfilemtimes{$basename} = $file->mtime if defined $file->mtime;
+ $dbfiletargets{$basename} = $file->target if defined $file->target;
}
my %dbfileidstodelete = %dbfileids;
my $cnt = 0, my $updated = 0;
my $sub = sub {
- my ($file, $size, $mmode, $mtime) = @_;
+ my ($file, $size, $mmode, $mtime, $target) = @_;
$file = $file . '/' if !$root->is_remote && $path ne '/' &&
$root->is_dir("$path/$file");
$file = $file . '/' if !$root->is_remote && $path eq '/' &&
$root->is_dir("$path$file");
$file = $file . '/' if $mmode && $root->is_remote && $mmode < 1000;
if ($dbfileids{$file}) {
my $id = delete $dbfileidstodelete{$file};
- if ((defined $size && defined $mtime) && ($size !=
$dbfilesizes{$file} || $mtime != $dbfilemtimes{$file})) {
+ if (
+ (defined $size && defined $mtime) && ($size !=
$dbfilesizes{$file} || $mtime != $dbfilemtimes{$file})
+ ||
+ (defined $target && $target ne ($dbfiletargets{$file} // ''))
+ ) {
$schema->storage->dbh->prepare(
- "UPDATE file set size = ?, mtime = ?, dt =
NOW()::timestamp(0) where id = ?"
- )->execute($size, $mtime, $id);
+ "UPDATE file set size = ?, mtime = ?, target = ?, dt =
NOW()::timestamp(0) where id = ?"
+ )->execute($size, $mtime, $target, $id);
$updated = $updated + 1;
}
return;
}
$cnt = $cnt + 1;
- $schema->resultset('File')->create({folder_id => $folder->id, name =>
$file, size => $size, mtime => $mtime});
+ $schema->resultset('File')->create({folder_id => $folder->id, name =>
$file, size => $size, mtime => $mtime, target => $target});
return undef;
};
- $app->mc->root->foreach_filename($realpath, $sub) or
+ $root->foreach_filename($realpath, $sub) or
return $job->fail('Error while reading files from root');
my @idstodelete = values %dbfileidstodelete;
$schema->storage->dbh->do(
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorCheckFromStat.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorCheckFromStat.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorCheckFromStat.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorCheckFromStat.pm
2021-12-16 07:57:47.000000000 +0100
@@ -51,7 +51,7 @@
my $cnt = 0;
$prev_stat_id = $stat_id;
my $ua = Mojo::UserAgent->new;
- my $tx = $ua->head($url);
+ my $tx = $ua->head($url, {'User-Agent' =>
'MirrorCache/check_from_stat'});
my $res = $tx->res;
if ($res->is_error) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorProbe.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorProbe.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorProbe.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorProbe.pm 2021-12-16
07:57:47.000000000 +0100
@@ -15,6 +15,7 @@
package MirrorCache::Task::MirrorProbe;
use Mojo::Base 'Mojolicious::Plugin';
+use Mojo::UserAgent;
use Net::URIProtocols qw/ProbeHttp ProbeHttps ProbeIpv4 ProbeIpv6/;
@@ -23,6 +24,8 @@
$app->minion->add_task(mirror_probe => sub { _probe($app, @_) });
$app->minion->add_task(mirror_force_downs => sub { _force_downs($app, @_)
});
$app->minion->add_task(mirror_force_ups => sub { _force_ups($app, @_) });
+
+ $app->minion->add_task(mirror_probe_projects => sub {
_probe_projects($app, @_) });
}
use constant SERVER_CAPABILITIES => (qw(http https ipv4 ipv6));
@@ -103,4 +106,38 @@
return ProbeIpv6($uri) if 'ipv6' eq $capability;
}
+sub _probe_projects {
+ my ($app, $job, $country) = @_;
+ $country = '' unless $country;
+
+ my $minion = $app->minion;
+ return $job->finish("Previous projects probe job is still active")
+ unless my $guard = $minion->guard('mirror_probe_projects', 6000);
+
+ my $schema = $app->schema;
+ my $rs = $schema->resultset('Server');
+ my $href = $rs->server_projects();
+
+ my %count;
+ my @keys = sort keys %$href;
+ $job->note(total => scalar(@keys));
+
+ for my $id (@keys) {
+ my $data = $href->{$id};
+ my $oldstate = $data->{oldstate};
+ my $success = 1;
+ my $code = -1;
+ eval {
+ $code = Mojo::UserAgent->new->max_redirects(5)->head($data->{uri},
{'User-Agent' => 'MirrorCache/probe_projects'})->result->code;
+ };
+ $success = 0 if $code ne 200;
+
+ $count{$code} = 0 unless $count{$code};
+ $count{$code}++;
+ $job->note("count$code" => $count{$code});
+
+ $rs->log_project_probe_outcome($data->{server_id},
$data->{project_id}, $data->{mirror_id}, $success, $code) unless defined
$oldstate && $success eq $oldstate;
+ }
+}
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScan.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorScan.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScan.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorScan.pm 2021-12-16
07:57:47.000000000 +0100
@@ -228,7 +228,7 @@
next;
}
- my $promise = $ua->get_p($url)->then($then)->catch(sub {
+ my $promise = $ua->get_p($url, {'User-Agent' =>
'MirrorCache/mirror_scan'})->then($then)->catch(sub {
my $err = shift;
return $app->emit_event('mc_mirror_probe_error', {mirror =>
$folder_on_mirror->{server_id}, url => "u$url", err => $err},
$folder_on_mirror->{server_id});
})->timeout(180)->wait;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
--- old/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
2021-12-16 07:57:47.000000000 +0100
@@ -23,7 +23,6 @@
}
my $DELAY = int($ENV{MIRRORCACHE_SCHEDULE_RETRY_INTERVAL} // 5);
-my $TIMEOUT = int($ENV{MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT} // 120);
sub _run {
my ($app, $job, $prev_stat_id) = @_;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2021-12-16
07:57:47.000000000 +0100
@@ -151,10 +151,17 @@
my $dm = shift;
my $route = $dm->route;
my ($path, $trailing_slash) = $dm->path;
- return undef if $trailing_slash || $path eq '/' || $dm->mirrorlist;
- return undef if $dm->must_render_from_root;
my $c = $dm->c;
+ my $ln;
+ $ln = $root->detect_ln($path);
+ if ($ln) {
+ # redirect to the symlink
+ $dm->redirect($dm->route . $ln);
+ return 1;
+ }
+ return undef if $trailing_slash || $path eq '/' || $dm->mirrorlist;
+ return undef if $dm->must_render_from_root;
my $subsidiary = $c->subsidiary;
my $url = $subsidiary->has($dm->region, $c->req->url);
if ($url) {
@@ -163,7 +170,7 @@
return 1;
}
# MIRRORCACHE_ROOT_COUNTRY must be set only with remote root and when no
mirrors should be used for the country
- return $root->render_file($dm, $dm->path_query, 1) if $dm->root_country &&
!$dm->trailing_slash && $dm->root_country eq $dm->country &&
$root->is_file($dm->_path) && !$dm->metalink && !$dm->mirrorlist;
+ return $root->render_file($dm, $path, 1) if $dm->root_country &&
!$dm->trailing_slash && $dm->root_country eq $dm->country &&
$root->is_file($dm->_path) && !$dm->metalink && !$dm->mirrorlist;
return undef;
}
@@ -269,12 +276,17 @@
$dm->folder_id($another_folder->id);
}
my $file;
- $file = $schema->resultset('File')->find({ name => $f->basename,
folder_id => $parent_folder->id }) if $parent_folder && !$trailing_slash;
+ $file =
$schema->resultset('File')->find_with_hash($parent_folder->id, $f->basename) if
$parent_folder && !$trailing_slash;
# folders are stored with trailing slash in file table, so they
will not be selected here
if ($file) {
- $dm->file_id($file->id);
- # find a mirror for it
- $c->mirrorcache->render_file($path, $dm);
+ if ($file->{target}) {
+ # redirect to the symlink
+ $dm->redirect($dm->route . $f->dirname . '/' .
$file->{target});
+ } else {
+ $dm->file_id($file->{id});
+ # find a mirror for it
+ $c->mirrorcache->render_file($path, $dm);
+ }
return 1;
}
}
@@ -316,7 +328,7 @@
# - redirected to another route => we must redirect it as well
my $path1 = $path . '/';
my $url1 = $url . '/'; # let's check if it is a folder
- $ua->head_p($url1)->then(sub {
+ $ua->head_p($url1, {'User-Agent' =>
'MirrorCache/guess_what_to_render'})->then(sub {
my $res = shift->res;
if (!$res->is_error || $res->code eq 403) {
@@ -375,13 +387,14 @@
my $dir = shift;
my $c = $dm->c;
my @files;
- my @childrenfiles = $c->schema->resultset('File')->search({folder_id =>
$id});
+ my $childrenfiles = $c->schema->resultset('File')->find_with_hash($id);
my $json = $dm->json;
- for my $child ( @childrenfiles ) {
- my $basename = $child->name;
- my $size = $child->size;
- my $mtime = $child->mtime;
+ for my $file_id ( keys %$childrenfiles ) {
+ my $child = $childrenfiles->{$file_id};
+ my $basename = $child->{name};
+ my $size = $child->{size};
+ my $mtime = $child->{mtime};
if ($json) {
push @files, {
name => $basename,
@@ -464,7 +477,6 @@
my $c = $dm->c;
return undef unless defined($c->param('hashes'));
my ($path, undef) = $dm->path;
- return undef unless $root->is_dir($path);
my $time_constraint;
if (defined $c->param("since") && $c->param("since")) {
@@ -475,13 +487,20 @@
}
my $schema = $c->app->schema;
- my $rsFolder = $schema->resultset('Folder');
- my $folder = $rsFolder->find({path => $path});
- return $c->render(status => 404, text => "path $path not found") unless
$folder;
+ my $folder = $schema->resultset('Folder')->find({path => $path});
- my $rsHash = $schema->resultset('Hash');
- my $folder_id = $folder->id;
- return $c->render(json => $rsHash->hashes_since($folder_id,
$time_constraint));
+ if ($folder) {
+ my $rsHash = $schema->resultset('Hash');
+ my $folder_id = $folder->id;
+ return $c->render(json => $rsHash->hashes_since($folder_id,
$time_constraint));
+ };
+
+ return $c->render(status => 404, text => "path $path not found") unless
$root->is_dir($path);
+
+ my $job_id =
$c->backstage->enqueue_unless_scheduled_with_parameter_or_limit('folder_sync',
$path);
+ # if scheduling didn't happen - return 404 so far
+ return $c->render(status => 404, text => "path $path not found") if
$job_id < 1;
+ return $c->render(status => 201, text => '[]');
}
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2021-12-16 07:57:47.000000000 +0100
@@ -30,6 +30,7 @@
sub register {
my ($self, $app) = @_;
+ $app->types->type(metalink => 'application/metalink+xml; charset=UTF-8');
$app->helper( 'mirrorcache.render_file' => sub {
my ($c, $filepath, $dm)= @_;
@@ -114,13 +115,12 @@
} else {
$origin = $url->scheme . '://' . $url->host;
$origin = $origin . ":" . $url->port if $url->port &&
$url->port != "80";
- $origin = $origin . $dm->route . $filepath;
+ $origin = $origin . $dm->route;
}
$origin = $origin . $filepath;
my $xml = _build_metalink(
$dm, $folder->path, $file, $country, $region,
\@mirrors_country, \@mirrors_region,
\@mirrors_rest, $origin, 'MirrorCache', $root->is_remote ?
$root->location($dm) : $root->redirect($dm, $folder->path) );
- $c->app->types->type(metalink => 'application/metalink+xml;
charset=UTF-8');
$c->res->headers->content_disposition('attachment; filename="'
.$basename. '.metalink"');
$c->render(data => $xml, format => 'metalink');
return 1;
@@ -165,7 +165,6 @@
my $size = $file->{size};
my $hsize = MirrorCache::Utils::human_readable_size($size) if
defined $size;
my $mtime = $file->{mtime};
- $mtime = $file->{hash_mtime} if $file->{hash_mtime};
my $hmtime = strftime("%d-%b-%Y %H:%M:%S", gmtime($mtime)) if
$mtime;
my $fileorigin;
my $fileoriginpath = $filepath;
@@ -264,7 +263,7 @@
}
my $url = $mirror->{url};
my $code;
- $ua->head_p($url)->then(sub {
+ $ua->head_p($url, {'User-Agent' =>
'MirrorCache/pedantic'})->then(sub {
my $result = shift->result;
$code = $result->code;
if ($code == 200 || $code == 302 || $code == 301) {
@@ -491,7 +490,11 @@
}
}
- if (!$found_count || ($found_count < $limit && !$dm->root_country) ||
$mirrorlist) {
+ if (
+ ($found_count < $limit && !$dm->root_country) ||
+ ($metalink && $found_count < 3) ||
+ $mirrorlist
+ ) {
$m = $rs->mirrors_query(
$country, $region, $folder_id, $file_id, $scheme,
$ipv, $lat, $lng, $avoid_countries, $limit, 1,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
--- old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
2021-12-16 07:57:47.000000000 +0100
@@ -1,4 +1,4 @@
-# Copyright (C) 2020 SUSE LLC
+# Copyright (C) 2020,2021 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -132,6 +132,32 @@
}
}
+sub _detect_ln {
+ my ($dir, $file) = @_;
+ return undef unless $file && $file =~ m/.*(Media|Current)\.iso(\.sha256)?/;
+
+ my $dest;
+ for my $root (@roots) {
+ eval {
+ $dest = readlink($root->[dir] . $dir . '/' . $file);
+ };
+ next unless $dest;
+ $dest = Mojo::File->new($dest);
+
+ return undef unless $dest->dirname eq '.' || $dest->dirname eq $dir;
+ return $dest->basename;
+ };
+ return undef;
+}
+
+sub detect_ln {
+ my ($self, $path) = @_;
+ my $f = Mojo::File->new($path);
+ my $res = _detect_ln($f->dirname, $f->basename);
+ return undef unless $res;
+ return $f->dirname . '/' . $res;
+}
+
# we cannot use $subtree here, because we may actually render from realdir,
which is outside subtree
sub foreach_filename {
my $self = shift;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
--- old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
2021-12-16 07:57:47.000000000 +0100
@@ -29,6 +29,8 @@
my $uaroot = Mojo::UserAgent->new->max_redirects(10)->request_timeout(1);
+my $nfs = $ENV{MIRRORCACHE_ROOT_NFS};
+
sub register {
my ($self, $app) = @_;
my $rooturl = $app->mc->rootlocation;
@@ -50,6 +52,7 @@
my $redirectvpns = $redirectvpn =~ s/http:/https:/r;
$self->rooturlredirectvpns($redirectvpns);
}
+ push @{$app->static->paths}, $nfs if $nfs;
$app->helper( 'mc.root' => sub { $self; });
}
@@ -90,6 +93,13 @@
sub render_file {
my ($self, $dm, $filepath, $not_miss) = @_;
my $c = $dm->c;
+
+ if ($nfs && $dm->must_render_from_root && -f $nfs . $filepath) {
+ $c->reply->static($filepath);
+ $c->stat->redirect_to_root($dm, $not_miss);
+ return 1;
+ }
+
$c->redirect_to($self->location($dm, $filepath));
$c->stat->redirect_to_root($dm, $not_miss);
return 1;
@@ -115,6 +125,35 @@
return 1;
};
+sub _detect_ln {
+ return undef unless $nfs;
+ my ($dir, $file) = @_;
+ return undef unless $file && $file =~ m/.*(Media|Current)\.iso(\.sha256)?/;
+
+ my $dest;
+ eval {
+ $dest = readlink($nfs . $dir . '/' . $file);
+ };
+ return undef unless $dest;
+ my $res;
+ eval {
+ $dest = Mojo::File->new($dest);
+
+ return undef unless $dest->dirname eq '.' || $dest->dirname eq $dir;
+ $res = $dest->basename;
+ };
+ return $res;
+}
+
+sub detect_ln {
+ return undef unless $nfs;
+ my ($self, $path) = @_;
+ my $f = Mojo::File->new($path);
+ my $res = _detect_ln($f->dirname, $f->basename);
+ return undef unless $res;
+ return $f->dirname . '/' . $res;
+}
+
# this is complicated to avoid storing big html in memory
# we parse and execute callback $sub on the fly
sub foreach_filename {
@@ -159,7 +198,8 @@
if ($t && ($href20 eq substr($t,0,20))) {
if ($desc{name}) {
- $sub->($desc{name}, $desc{size}, undef, $desc{mtime});
+ my $target = _detect_ln($dir, $desc{name});
+ $sub->($desc{name}, $desc{size}, undef, $desc{mtime}, $target);
%desc = ();
}
$desc{name} = $href;
@@ -199,7 +239,8 @@
$p->parse($chunk);
}
if ($desc{name}) {
- $sub->($desc{name}, $desc{size}, undef, $desc{mtime});
+ my $target = detect_ln($dir, $desc{name});
+ $sub->($desc{name}, $desc{size}, undef, $desc{mtime}, $target);
%desc = ();
}
$p->eof;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RootRsync.pm
new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RootRsync.pm
--- old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RootRsync.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/RootRsync.pm
2021-12-16 07:57:47.000000000 +0100
@@ -146,4 +146,9 @@
return 1;
}
+sub detect_ln {
+ return undef;
+}
+
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
--- old/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2021-12-16 07:57:47.000000000 +0100
@@ -65,6 +65,21 @@
$c->proxy->get_p($req);
});
}
+ my $basename = $ENV{MIRRORCACHE_GEOIP_BASENAME} // 'geoip';
+ $app->routes->get("/geoip" => sub {
+ my $c = shift;
+ my $dm = MirrorCache::Datamodule->new->app($c->app);
+ $dm->reset($c);
+
+ my $country = $dm->country;
+ my $region = $dm->region;
+ my $url = _has_subsidiary($c, $dm->region);
+ return $c->render(status => 204, text => '') unless $url;
+ $url = $url->to_abs;
+ $url =~ s/http(s)?:\/\///;
+ $c->res->headers->content_disposition("attachment;
filename=\"$basename\"");
+ $c->render(data =>
"<geoip><region>$region</region><country>$country</country><host>$url</host></geoip>",
format => 'xml');
+ });
$app->helper('subsidiary.has' => \&_has_subsidiary);
$app->helper('subsidiary.regions' => \&_regions);
@@ -74,7 +89,7 @@
sub _has_subsidiary {
return undef unless keys %subsidiary_urls;
- my ($self, $region, $origin_url) = @_;
+ my ($c, $region, $origin_url) = @_;
my $region_url = $subsidiary_urls{$region};
return $region_url unless $region_url && $origin_url;
my $url = $origin_url->to_abs->clone;
@@ -87,7 +102,7 @@
# return url for all subsidiaries
sub _regions {
return undef unless keys %subsidiary_urls;
- my ($self, $region) = @_;
+ my ($c, $region) = @_;
my $url = $subsidiary_urls{$region};
my @res = ($url? $region : '');
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/lib/MirrorCache/resources/migrations/pg.sql
new/MirrorCache-1.022/lib/MirrorCache/resources/migrations/pg.sql
--- old/MirrorCache-1.019/lib/MirrorCache/resources/migrations/pg.sql
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/lib/MirrorCache/resources/migrations/pg.sql
2021-12-16 07:57:47.000000000 +0100
@@ -25,6 +25,7 @@
size bigint,
mtime bigint,
dt timestamp,
+ target varchar(512),
unique(folder_id, name)
);
@@ -181,6 +182,7 @@
sha256 char(64),
piece_size int,
pieces text,
+ target varchar(512),
dt timestamp NOT NULL
);
create index hash_file_id_size on hash(file_id, size);
@@ -232,3 +234,15 @@
-- 15 up
alter table subsidiary add column if not exists local boolean default 'f';
alter table folder add column if not exists hash_last_import timestamp;
+-- 16 up
+alter table file add column if not exists target varchar(512);
+alter table hash add column if not exists target varchar(512);
+-- 17 up
+create table server_project (
+ server_id bigint NOT NULL references server on delete cascade,
+ project_id bigint NOT NULL references project on delete cascade,
+ state int, -- -1 - disabled, 0 - missing, 1 - present
+ extra varchar(1024),
+ dt timestamp,
+ unique(server_id, project_id)
+);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/01-smoke-plan-scan.sh
new/MirrorCache-1.022/t/environ/01-smoke-plan-scan.sh
--- old/MirrorCache-1.019/t/environ/01-smoke-plan-scan.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/01-smoke-plan-scan.sh 2021-12-16
07:57:47.000000000 +0100
@@ -3,11 +3,10 @@
mc=$(environ mc $(pwd))
-MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0
+
MIRRORCACHE_RESCAN_INTERVAL=0
-$mc/gen_env
MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=$MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT \
- MIRRORCACHE_RESCAN_INTERVAL=$MIRRORCACHE_RESCAN_INTERVAL
+$mc/gen_env MIRRORCACHE_RESCAN_INTERVAL=$MIRRORCACHE_RESCAN_INTERVAL
$mc/start
$mc/status
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/02-cleanup.sh
new/MirrorCache-1.022/t/environ/02-cleanup.sh
--- old/MirrorCache-1.019/t/environ/02-cleanup.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/02-cleanup.sh 2021-12-16
07:57:47.000000000 +0100
@@ -4,8 +4,7 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
-$mc/gen_env MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
-
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
+$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
$mc/start
$mc/status
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/02-files-hashes-import.sh
new/MirrorCache-1.022/t/environ/02-files-hashes-import.sh
--- old/MirrorCache-1.019/t/environ/02-files-hashes-import.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/02-files-hashes-import.sh 2021-12-16
07:57:47.000000000 +0100
@@ -13,6 +13,7 @@
mkdir -p $x/dt/{folder1,folder2,folder3}
echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
echo 1111111111 > $x/dt/folder1/file1.1.dat
+ echo 1111111112 > $x/dt/folder2/file1.1.dat
eval mc$i=$x
done
@@ -21,6 +22,7 @@
mkdir -p $x/dt/{folder1,folder2,folder3}
echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
echo 1111111111 > $x/dt/folder1/file1.1.dat
+ echo 1111111112 > $x/dt/folder2/file1.1.dat
eval ap$i=$x
$x/start
done
@@ -33,7 +35,7 @@
na_interface=127.0.0.2
# deploy db
-$mc9/gen_env MIRRORCACHE_HASHES_COLLECT=1 MIRRORCACHE_HASHES_PIECES_MIN_SIZE=5
"MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" MIRRORCACHE_BRANDING=SUSE
+$mc9/gen_env MIRRORCACHE_HASHES_COLLECT=1 MIRRORCACHE_HASHES_PIECES_MIN_SIZE=5
"MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" MIRRORCACHE_BRANDING=SUSE
MIRRORCACHE_WORKERS=4 MIRRORCACHE_DAEMON=1
$mc9/backstage/shoot
$mc9/sql "insert into subsidiary(hostname,region) select '$na_address','na'"
@@ -41,14 +43,13 @@
$mc9/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap1/print_address)','','t','jp','as'"
$mc9/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap2/print_address)','','t','jp','as'"
-$mc6/gen_env MIRRORCACHE_REGION=na MIRRORCACHE_HEADQUARTER=$hq_address
"MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'"
+$mc6/gen_env MIRRORCACHE_REGION=na MIRRORCACHE_HEADQUARTER=$hq_address
"MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" MIRRORCACHE_HASHES_IMPORT=1
$mc6/start
$mc6/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap3/print_address)','','t','us','na'"
$mc6/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap4/print_address)','','t','ca','na'"
for i in 9 6; do
mc$i/backstage/job -e folder_sync -a '["/folder1"]'
- [[ $i == 6 ]] && mc$i/backstage/job -e folder_hashes_import -a
'["/folder1"]' || :
mc$i/backstage/shoot
mc$i/backstage/shoot -q hashes
done
@@ -76,4 +77,16 @@
mc9/sql 'update file set size=0, mtime=null'
mc9/curl -sL /folder1/file1.1.dat.metalink | grep -F -C10 '<size>11</size>'
+echo Import folder unknown on master
+curl -si "http://$hq_address/folder2?hashes";
+mc9/backstage/shoot
+mc9/backstage/shoot -q hashes
+test d8f5889697e9ec5ba9a8ab4aede6e7d1d7858884e81db19b3e9780d6a64671a3 ==
$(mc9/db/sql 'select sha256 from hash where file_id=3')
+
+mc6/backstage/job -e folder_sync -a '["/folder2"]'
+mc6/backstage/shoot
+mc6/backstage/shoot -q hashes
+
+test d8f5889697e9ec5ba9a8ab4aede6e7d1d7858884e81db19b3e9780d6a64671a3 ==
$(mc6/db/sql 'select sha256 from hash where file_id=3')
+
echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/02-files-hashes.sh
new/MirrorCache-1.022/t/environ/02-files-hashes.sh
--- old/MirrorCache-1.019/t/environ/02-files-hashes.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/02-files-hashes.sh 2021-12-16
07:57:47.000000000 +0100
@@ -4,8 +4,7 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=1
-$mc/gen_env MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
-
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
+$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
MIRRORCACHE_HASHES_COLLECT=1 \
MIRRORCACHE_HASHES_PIECES_MIN_SIZE=5
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/02-files.sh
new/MirrorCache-1.022/t/environ/02-files.sh
--- old/MirrorCache-1.019/t/environ/02-files.sh 2021-12-01 13:56:26.000000000
+0100
+++ new/MirrorCache-1.022/t/environ/02-files.sh 2021-12-16 07:57:47.000000000
+0100
@@ -4,8 +4,7 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
-$mc/gen_env MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
-
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
+$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
$mc/start
$mc/status
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-hashes.sh
new/MirrorCache-1.022/t/environ/03-headquarter-subsidiaries-hashes.sh
--- old/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-hashes.sh
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/03-headquarter-subsidiaries-hashes.sh
2021-12-16 07:57:47.000000000 +0100
@@ -27,17 +27,17 @@
$mc9/gen_env MIRRORCACHE_TOP_FOLDERS="'folder1 folder2 folder3'"
MIRRORCACHE_HASHES_COLLECT=1
$mc9/backstage/shoot
-$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$na_address','na','/download'"
-$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$eu_address','eu','/download'"
-$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$as_address','as','/download'"
+$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$na_address','na',''"
+$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$eu_address','eu',''"
+$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$as_address','as',''"
$mc9/start
-$mc6/gen_env MIRRORCACHE_REGION=na MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address)
+$mc6/gen_env MIRRORCACHE_REGION=na MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address) MIRRORCACHE_TOP_FOLDERS="'folder1
folder2 folder3'"
$mc6/start
-$mc7/gen_env MIRRORCACHE_REGION=eu MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address)
+$mc7/gen_env MIRRORCACHE_REGION=eu MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address) MIRRORCACHE_TOP_FOLDERS="'folder1
folder2 folder3'"
$mc7/start
-$mc8/gen_env MIRRORCACHE_REGION=as MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address)
+$mc8/gen_env MIRRORCACHE_REGION=as MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address) MIRRORCACHE_TOP_FOLDERS="'folder1
folder2 folder3'"
$mc8/start
@@ -73,3 +73,18 @@
done
+echo Step 3. Add media symlinks and make sure they are imported properly
+for i in 9 6 7 8; do
+ ( cd mc$i/dt/folder1/ && ln -s file4.1.dat file-Media.iso )
+ mc$i/backstage/job -e folder_sync -a '["/folder1"]'
+ mc$i/backstage/shoot
+ mc$i/backstage/shoot -q hashes
+ mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id
= file_id where name='file-Media.iso'"
+ for x in '' .metalink .mirrorlist; do
+ mc$i/curl -I /folder1/file-Media.iso$x | grep -C 10 302 | grep
/folder1/file4.1.dat$x | grep -v /download/folder1/file4.1.dat$x
+ done
+done
+
+mc9/curl -IL /download/folder1/file-Media.iso | grep '200 OK'
+
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
new/MirrorCache-1.022/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
--- old/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
2021-12-16 07:57:47.000000000 +0100
@@ -43,7 +43,6 @@
as_interface=127.0.0.4
$mc9/gen_env "MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
# deploy db
@@ -93,6 +92,10 @@
curl -Is --interface $eu_interface
http://$hq_address/folder1/repodata/repomd.xml | grep 200
curl -Is --interface $eu_interface
http://$hq_address/folder1/repodata/repomd.xml.asc | grep 200
+# /geoip
+curl -s --interface $eu_interface http://$hq_address/geoip | grep
"<host>$eu_address</host>"
+curl -s --interface $na_interface http://$hq_address/geoip | grep
"<host>$na_address</host>"
+
###########################################
# test table demand_mirrorlist:
# if mirrorlist was requested for known file - all countries will be scanned
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-remote.sh
new/MirrorCache-1.022/t/environ/03-headquarter-subsidiaries-remote.sh
--- old/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-remote.sh
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/03-headquarter-subsidiaries-remote.sh
2021-12-16 07:57:47.000000000 +0100
@@ -0,0 +1,48 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+# root : ap9
+
+# mc environ by number:
+# 9 - headquarter
+# 6 - NA subsidiary
+# 7 - EU subsidiary
+
+root=$(environ ap9)
+
+mkdir -p $root/dt/{folder1,folder2,folder3}
+echo $root/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+mkdir $root/dt/folder1/repodata/
+touch $root/dt/folder1/repodata/file1.dat
+touch $root/dt/folder1/repodata/repomd.xml
+touch $root/dt/folder1/repodata/repomd.xml.asc
+
+$root/start
+
+mc9=$(environ mc9 $(pwd))
+$mc9/gen_env "MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" \
+ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 \
+ MIRRORCACHE_ROOT=http://$($root/print_address) \
+ MIRRORCACHE_ROOT_NFS=$root/dt
+
+# deploy db
+$mc9/backstage/shoot
+
+$mc9/sql "insert into subsidiary(hostname,region) select 'naaddress.com','na'"
+$mc9/sql "insert into subsidiary(hostname,region) select 'euaddress.net','eu'"
+$mc9/start
+
+$mc9/backstage/job -e folder_sync -a '["/folder1/repodata"]'
+$mc9/backstage/job -e mirror_scan -a '["/folder1/repodata"]'
+$mc9/backstage/shoot
+
+hq_address=$(mc9/print_address)
+eu_interface=127.0.0.3
+
+# repodata/repomd.xml is served from root even when asked from EU
+curl -si --interface $eu_interface http://$hq_address/geoip | grep -A 50
'200 OK' | grep "<host>euaddress.net</host>"
+curl -si http://$hq_address/geoip | grep -A 50
'200 OK' | grep "<host>naaddress.com</host>"
+
+curl -sI --interface 127.0.0.24 http://$hq_address/geoip | grep '204 No
Content'
+
+mc9/curl --interface 127.0.0.5 /folder1/repodata/file1.dat.metalink | grep
'origin="http://127.0.0.1:3190/folder1/repodata/file1.dat.metalink";'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-remote-current.sh
new/MirrorCache-1.022/t/environ/04-remote-current.sh
--- old/MirrorCache-1.019/t/environ/04-remote-current.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote-current.sh 2021-12-16
07:57:47.000000000 +0100
@@ -0,0 +1,62 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+ap9=$(environ ap9)
+
+$mc/gen_env MIRRORCACHE_PEDANTIC=1 \
+ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 \
+ MIRRORCACHE_ROOT=http://$($ap9/print_address) \
+ MIRRORCACHE_ROOT_NFS=$ap9/dt
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+
+for x in $ap7 $ap8 $ap9; do
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Media.iso | xargs
-n 1 touch
+ sha256sum $x/dt/folder1/file1.1-Media.iso >
$x/dt/folder1/file1.1-Media.iso.sha256
+ echo 111112 > $x/dt/folder1/file2.1-Media.iso
+ sha256sum $x/dt/folder1/file2.1-Media.iso >
$x/dt/folder1/file2.1-Media.iso.sha256
+ ( cd $x/dt/folder1 && ln -s file1.1-Media.iso file-Media.iso && ln -s
file1.1-Media.iso.sha256 file-Media.iso.sha256 )
+done
+
+for x in $ap7 $ap8 $ap9; do
+ $x/start
+done
+
+$mc/start
+$mc/status
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','us','na'"
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+
+
+$mc/sql "select * from file"
+$mc/sql_test "select count(*) from file where target is not null"
+
+################################################
+# Test unversioned Media.iso is redirected to file which is metioned inside
corresponding Media.iso.sha256
+$mc/curl -I /download/folder1/file-Media.iso | grep -C 10 302 | grep
/download/folder1/file1.1-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep
/download/folder1/file1.1-Media.iso.sha256
+$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 "
+
+echo now change the symlink and make sure redirect changes
+(
+ cd $ap9/dt/folder1
+ ln -sf file2.1-Media.iso file-Media.iso
+ ln -sf file2.1-Media.iso.sha256 file-Media.iso.sha256
+)
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+$mc/curl -I /download/folder1/file-Media.iso | grep -C 10 302 | grep
/download/folder1/file2.1-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep
/download/folder1/file2.1-Media.iso.sha256
+$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q
"2019dd7afaf5759c68cec4d0e7553227657f01c69da168489116a1c48e40270e "
+echo success
+
+
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-remote-down.sh
new/MirrorCache-1.022/t/environ/04-remote-down.sh
--- old/MirrorCache-1.019/t/environ/04-remote-down.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote-down.sh 2021-12-16
07:57:47.000000000 +0100
@@ -6,8 +6,7 @@
ap9=$(environ ap9)
$mc/gen_env MIRRORCACHE_PEDANTIC=1 \
- MIRRORCACHE_ROOT=http://$($ap9/print_address) \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0
+ MIRRORCACHE_ROOT=http://$($ap9/print_address)
ap8=$(environ ap8)
ap7=$(environ ap7)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-remote-link.sh
new/MirrorCache-1.022/t/environ/04-remote-link.sh
--- old/MirrorCache-1.019/t/environ/04-remote-link.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote-link.sh 2021-12-16
07:57:47.000000000 +0100
@@ -6,8 +6,8 @@
ap9=$(environ ap9)
$mc/gen_env MIRRORCACHE_PEDANTIC=1 \
- MIRRORCACHE_ROOT=http://$($ap9/print_address) \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0
+ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 \
+ MIRRORCACHE_ROOT=http://$($ap9/print_address)
ap8=$(environ ap8)
ap7=$(environ ap7)
@@ -50,9 +50,15 @@
################################################
# make sure symlinks still work after further folder_sync_schedule_from_misses
-sleep 10
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/sql_test 2 == "select count(*) from minion_jobs where task =
'folder_sync_schedule_from_misses' and state = 'finished'"
$mc/curl -I /download/link
$mc/curl -I /download/link | grep -A4 -E '301|302' | grep ': /download/folder1'
-sleep 10
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/sql_test 3 == "select count(*) from minion_jobs where task =
'folder_sync_schedule_from_misses' and state = 'finished'"
$mc/curl -I /download/link
$mc/curl -I /download/link | grep -A4 -E '301|302' | grep ': /download/folder1'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/04-remote-nginx-redirect.sh
new/MirrorCache-1.022/t/environ/04-remote-nginx-redirect.sh
--- old/MirrorCache-1.019/t/environ/04-remote-nginx-redirect.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote-nginx-redirect.sh 2021-12-16
07:57:47.000000000 +0100
@@ -10,7 +10,6 @@
MIRRORCACHE_ROOT=http://$($ng9/print_address) \
MIRRORCACHE_REDIRECT=http://$($ap9/print_address) \
MIRRORCACHE_REDIRECT_VPN=root.vpn.us \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
ng8=$(environ ng8)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-remote-nginx.sh
new/MirrorCache-1.022/t/environ/04-remote-nginx.sh
--- old/MirrorCache-1.019/t/environ/04-remote-nginx.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote-nginx.sh 2021-12-16
07:57:47.000000000 +0100
@@ -9,7 +9,6 @@
$mc/gen_env MIRRORCACHE_PEDANTIC=1 \
MIRRORCACHE_ROOT=http://$($ng9/print_address) \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
ng8=$(environ ng8)
@@ -189,3 +188,11 @@
$mc/backstage/shoot
$mc/curl /download/folder1/ | grep -B1 $f | grep '10 Byte'
+
+##########################
+# add a symlink and make sure size is correct for it as well
+( cd $ng9/dt/folder1 && ln -s $f ln-Media.iso )
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+
+$mc/curl /download/folder1/ | grep -B1 ln-Media.iso | grep '10 Byte'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-remote-recurs.sh
new/MirrorCache-1.022/t/environ/04-remote-recurs.sh
--- old/MirrorCache-1.019/t/environ/04-remote-recurs.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote-recurs.sh 2021-12-16
07:57:47.000000000 +0100
@@ -7,7 +7,6 @@
$mc/gen_env \
MIRRORCACHE_ROOT=http://$($ap9/print_address) \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
$mc/start
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-remote.sh
new/MirrorCache-1.022/t/environ/04-remote.sh
--- old/MirrorCache-1.019/t/environ/04-remote.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/04-remote.sh 2021-12-16
07:57:47.000000000 +0100
@@ -7,7 +7,6 @@
$mc/gen_env MIRRORCACHE_PEDANTIC=1 \
MIRRORCACHE_ROOT=http://$($ap9/print_address) \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
ap8=$(environ ap8)
@@ -16,6 +15,7 @@
for x in $ap7 $ap8 $ap9; do
mkdir -p $x/dt/{folder1,folder2,folder3}
echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+ echo 11111 > $x/dt/folder1/file1.1.dat
$x/start
done
@@ -25,7 +25,6 @@
$mc/db/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
$mc/db/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','us','na'"
-# remove folder1/file1.1.dt from ap8
rm $ap8/dt/folder1/file2.1.dat
# first request redirected to root
@@ -37,10 +36,13 @@
$mc/backstage/job mirror_scan_schedule
$mc/backstage/shoot
-$mc/db/sql "select * from file"
test 2 == $($mc/db/sql "select count(*) from folder_diff")
test 1 == $($mc/db/sql "select count(*) from folder_diff_file")
+# let's pretend hashes were imported somehow and make sure they are shown in UI
+$mc/sql 'insert into hash(file_id, size, mtime, dt) select 1, 5, extract(epoch
from now()), now()'
+$mc/curl /download/folder1/ | grep -A2 file1.1.dat | grep '5 Byte'
+
$mc/curl -I /download/folder1/file2.1.dat | grep $($ap7/print_address)
mv $ap7/dt/folder1/file2.1.dat $ap8/dt/folder1/
@@ -173,4 +175,4 @@
$mc/curl -I /download/folder1/file:4.dat | grep -E
"$($ap8/print_address)|$(ap7*/print_address)"
##################################
-
+grep MirrorCache/mirror_scan $ap7/dt/access_log
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/04-rsync.sh
new/MirrorCache-1.022/t/environ/04-rsync.sh
--- old/MirrorCache-1.019/t/environ/04-rsync.sh 2021-12-01 13:56:26.000000000
+0100
+++ new/MirrorCache-1.022/t/environ/04-rsync.sh 2021-12-16 07:57:47.000000000
+0100
@@ -10,7 +10,6 @@
MIRRORCACHE_ROOT=rsync://$USER:$USER@$($rs/print_address)/dt \
MIRRORCACHE_REDIRECT=some.address.fake.com \
MIRRORCACHE_REDIRECT_VPN=some.address.fake.vpn.us \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
ap8=$(environ ap8)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/06-remote-geo-dynamic.sh
new/MirrorCache-1.022/t/environ/06-remote-geo-dynamic.sh
--- old/MirrorCache-1.019/t/environ/06-remote-geo-dynamic.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/06-remote-geo-dynamic.sh 2021-12-16
07:57:47.000000000 +0100
@@ -11,7 +11,6 @@
$mc/gen_env MIRRORCACHE_ROOT=http://$($ap6/print_address) \
MIRRORCACHE_STAT_FLUSH_COUNT=1 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=1 \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
MIRRORCACHE_BACKSTAGE_WORKERS=5
for x in $ap6 $ap7 $ap8 $ap9; do
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/10-remote-redirect.sh
new/MirrorCache-1.022/t/environ/10-remote-redirect.sh
--- old/MirrorCache-1.019/t/environ/10-remote-redirect.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/10-remote-redirect.sh 2021-12-16
07:57:47.000000000 +0100
@@ -26,7 +26,6 @@
MOJO_CA_FILE=$(pwd)/ca/ca.pem \
MOJO_REVERSE_PROXY=1 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=2 \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=1 \
MIRRORCACHE_BACKSTAGE_WORKERS=8 \
MIRRORCACHE_METALINK_PUBLISHER_URL=http://metalink_publisher.net
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/14-project-capability.sh
new/MirrorCache-1.022/t/environ/14-project-capability.sh
--- old/MirrorCache-1.019/t/environ/14-project-capability.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/14-project-capability.sh 2021-12-16
07:57:47.000000000 +0100
@@ -0,0 +1,60 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+$mc/start
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+ap6=$(environ ap6)
+
+for x in $mc $ap7 $ap8 $ap6; do
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ mkdir -p $x/dt/project1/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+ echo $x/dt/project1/{folder1,folder2,folder3}/{file1.1,file2.1}.dat |
xargs -n 1 touch
+done
+
+$ap6/start
+$ap7/start
+$ap8/start
+
+# remove a file from ap7
+rm $ap7/dt/project1/folder2/file2.1.dat
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap6/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','us','na'"
+
+$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 1"
+
+$mc/sql "insert into server_project(server_id,project_id,state) select 3,1,-1"
+
+
+$mc/backstage/job -e folder_sync -a '["/project1/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/project1/folder1"]'
+$mc/backstage/shoot
+
+$mc/sql "select notes from minion_jobs where task = 'mirror_scan'" | grep
-C100 hash1 | grep hash2
+
+rc=0
+$mc/sql "select notes from minion_jobs where task = 'mirror_scan'" | grep -q
hash3 || rc=$?
+test $rc -gt 0 || fail 'hash3 should not be in notes'
+
+
+rm -r $ap7/dt/project1
+$mc/backstage/job -e mirror_probe_projects
+$mc/backstage/shoot
+$mc/sql_test 0 == "select state from server_project where server_id = 2 and
project_id = 1"
+$mc/sql_test -1 == "select state from server_project where server_id = 3 and
project_id = 1"
+
+
+$mc/sql "update server_project set state = 0 where server_id = 3 and
project_id = 1"
+$mc/backstage/job -e mirror_probe_projects
+$mc/backstage/shoot
+$mc/sql_test 0 == "select state from server_project where server_id = 2 and
project_id = 1"
+$mc/sql_test 1 == "select state from server_project where server_id = 3 and
project_id = 1"
+
+
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/15-local-symlink-subtree-rootredirect.sh
new/MirrorCache-1.022/t/environ/15-local-symlink-subtree-rootredirect.sh
--- old/MirrorCache-1.019/t/environ/15-local-symlink-subtree-rootredirect.sh
2021-12-01 13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/15-local-symlink-subtree-rootredirect.sh
2021-12-16 07:57:47.000000000 +0100
@@ -2,7 +2,7 @@
set -ex
mc=$(environ mc $(pwd))
-
+$mc/gen_env MIRRORCACHE_ROOT_COUNTRY=us
$mc/start
$mc/status
@@ -61,3 +61,6 @@
$mcsub/curl /tool/v1/file1.1.dat.mirrorlist | grep -C10
$($ap7/print_address)/folder1/file1.1.dat | grep -C20
$($ap8/print_address)/updates/tool/v1/file1.1.dat | grep
testhost.com/updates/tool/v1/file1.1.dat
$mcsub/curl /tool/v2/file1.1.dat.mirrorlist | grep -C20
$($ap8/print_address)/updates/tool/v2/file1.1.dat | grep
testhost.com/updates/tool/v2/file1.1.dat
+
+# test request from ROOT_COUNTRY is just served
+$mc/curl -i /download/folder1/file1.1.dat?COUNTRY=us | grep '200 OK'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.019/t/environ/16-rescan-forget-unused.sh
new/MirrorCache-1.022/t/environ/16-rescan-forget-unused.sh
--- old/MirrorCache-1.019/t/environ/16-rescan-forget-unused.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/16-rescan-forget-unused.sh 2021-12-16
07:57:47.000000000 +0100
@@ -3,10 +3,8 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
-MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0
$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
-
MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=$MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT \
MIRRORCACHE_PEDANTIC=1 \
MIRRORCACHE_RECKLESS=1
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/16-rescan-from-errors.sh
new/MirrorCache-1.022/t/environ/16-rescan-from-errors.sh
--- old/MirrorCache-1.019/t/environ/16-rescan-from-errors.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/16-rescan-from-errors.sh 2021-12-16
07:57:47.000000000 +0100
@@ -3,11 +3,9 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
-MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0
MIRRORCACHE_RESCAN_INTERVAL=$((7 * 24 * 60 * 60)) # set one week to avoid
automatic rescan
$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
-
MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=$MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT \
MIRRORCACHE_RESCAN_INTERVAL=$MIRRORCACHE_RESCAN_INTERVAL \
MIRRORCACHE_PEDANTIC=1 \
MIRRORCACHE_RECKLESS=0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/environ/16-rescan.sh
new/MirrorCache-1.022/t/environ/16-rescan.sh
--- old/MirrorCache-1.019/t/environ/16-rescan.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/environ/16-rescan.sh 2021-12-16
07:57:47.000000000 +0100
@@ -3,11 +3,10 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
-MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0
+
MIRRORCACHE_RESCAN_INTERVAL=$((7 * 24 * 60 * 60)) # set one week to avoid
automatic rescan
$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
-
MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=$MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT \
MIRRORCACHE_RESCAN_INTERVAL=$MIRRORCACHE_RESCAN_INTERVAL \
MIRRORCACHE_RECKLESS=0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/t/manual/02-headquarter-samedb.sh
new/MirrorCache-1.022/t/manual/02-headquarter-samedb.sh
--- old/MirrorCache-1.019/t/manual/02-headquarter-samedb.sh 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/t/manual/02-headquarter-samedb.sh 2021-12-16
07:57:47.000000000 +0100
@@ -27,10 +27,10 @@
MIRRORCACHE_REDIRECT=downloadcontent.opensuse.org \
MIRRORCACHE_HYPNOTOAD=1 \
MIRRORCACHE_PERMANENT_JOBS="'folder_sync_schedule_from_misses
folder_sync_schedule mirror_scan_schedule_from_misses
mirror_scan_schedule_from_path_errors mirror_scan_schedule cleanup
stat_agg_schedule mirror_check_from_stat'" \
- MIRRORCACHE_TOP_FOLDERS="'debug distribution tumbleweed factory
repositories'" \
+ MIRRORCACHE_TOP_FOLDERS="'debug distribution tumbleweed factory
repositories source'" \
MIRRORCACHE_TRUST_AUTH=127.0.0.16 \
MIRRORCACHE_PROXY_URL=http://$($x/print_address) \
- MIRRORCACHE_BACKSTAGE_WORKERS=4 \
+ MIRRORCACHE_BACKSTAGE_WORKERS=8 \
MIRRORCACHE_HEADQUARTER=$hq_address \
${xtra[$x]}
@@ -39,7 +39,10 @@
ln -s $hq/db $x/db
}
- $x/backstage/start # start backstage here to deploy db
+ $x/backstage/shoot # spawn backstage first here to deploy db
+ $x/backstage/start
+ test $x == $eu || $x/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
mc_test
+
done
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/templates/dir.html.ep
new/MirrorCache-1.022/templates/dir.html.ep
--- old/MirrorCache-1.019/templates/dir.html.ep 2021-12-01 13:56:26.000000000
+0100
+++ new/MirrorCache-1.022/templates/dir.html.ep 2021-12-16 07:57:47.000000000
+0100
@@ -49,7 +49,7 @@
% }
% for my $file (@$files) {
<tr>
- % if ($file->{mtime}) {
+ % if (defined $file->{size}) {
% my $colspan = $file->{dir} ? 'colspan=2 ' : '';
<td class="name"><a href="<%= $file->{url} %>"><%== $file->{name}
%></a></td>
<td <%= $colspan %>class="mtime"><%= $file->{mtime} %></td>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.019/templates/mirrorlist.html.ep
new/MirrorCache-1.022/templates/mirrorlist.html.ep
--- old/MirrorCache-1.019/templates/mirrorlist.html.ep 2021-12-01
13:56:26.000000000 +0100
+++ new/MirrorCache-1.022/templates/mirrorlist.html.ep 2021-12-16
07:57:47.000000000 +0100
@@ -36,8 +36,12 @@
<ul>
<li>Filename: <%= $file->{name} %></li>
<li>Path: <%= $cur_path %></li>
+ % if ($file->{hsize}) {
<li>Size: <%= $file->{hsize} %> (<%= $file->{size} %> bytes)</li>
+ % }
+ % if ($file->{mtime}) {
<li>Last modified: <%= $file->{hmtime} %> (Unix timestamp: <%=
$file->{mtime} %>)</li>
+ % }
% if ($file->{sha256}) {
<li>SHA-256 Hash: <%= $file->{sha256} %></li>
% }
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.U82QS7/_old 2021-12-23 17:54:06.175732262 +0100
+++ /var/tmp/diff_new_pack.U82QS7/_new 2021-12-23 17:54:06.175732262 +0100
@@ -1,6 +1,6 @@
name: MirrorCache
-version: 1.019
-mtime: 1638363386
-commit: 3ddb8020f3772d4c49953e24232946d874c85e90
+version: 1.022
+mtime: 1639637867
+commit: 8ba68c5b4ca73795dff412330402f6e64b86e032
