Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2022-03-16 20:21:33
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.25692 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Wed Mar 16 20:21:33 2022 rev:10 rq:962181 version:1.026
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2022-02-17
23:41:58.859700192 +0100
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.25692/MirrorCache.changes
2022-03-16 20:21:48.845028418 +0100
@@ -1,0 +2,8 @@
+Wed Mar 09 12:42:33 UTC 2022 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.026:
+ * Fix message for unknown folder render timeout (#260)
+ * Support single-country subsidiaries (#259)
+ * Add table server_stability (#258)
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.025.obscpio
New:
----
MirrorCache-1.026.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.EKrmnh/_old 2022-03-16 20:21:49.557028931 +0100
+++ /var/tmp/diff_new_pack.EKrmnh/_new 2022-03-16 20:21:49.565028936 +0100
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.025
+Version: 1.026
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.025.obscpio -> MirrorCache-1.026.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/Schema/ResultSet/Server.pm
new/MirrorCache-1.026/lib/MirrorCache/Schema/ResultSet/Server.pm
--- old/MirrorCache-1.025/lib/MirrorCache/Schema/ResultSet/Server.pm
2022-02-10 11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/lib/MirrorCache/Schema/ResultSet/Server.pm
2022-03-08 11:08:06.000000000 +0100
@@ -78,45 +78,38 @@
my $capabilityx = $capability eq 'http'? 'https' : 'http';
my $extra = '';
if ($schemastrict && $ipvstrict) {
- $extra = "WHERE no4 = 0 and no5 = 0";
+ $extra = "WHERE support_scheme > 0 and support_ipv > 0";
} elsif ($schemastrict) {
- $extra = "WHERE no4 = 0";
+ $extra = "WHERE support_scheme > 0";
} elsif ($ipvstrict) {
- $extra = "WHERE no5 = 0";
+ $extra = "WHERE support_ipv > 0";
}
my $hostname = $vpn? "CASE WHEN length(s.hostname_vpn)>0 THEN
s.hostname_vpn ELSE s.hostname END" : "s.hostname";
my $sql = <<"END_SQL";
select * from (
-select x.id as mirror_id, concat(case when no4 = 0 then '$capability' else
'$capabilityx' end, '://', uri) as url,
-case when $lat=0 and $lng=0 then 0 -- prefer servers which were checked
recently when geoip is unavailable
-else
-( 6371 * acos( cos( radians($lat) ) * cos( radians( lat ) ) * cos( radians(
lng ) - radians($lng) ) + sin( radians($lat) ) * sin( radians( lat ) ) ) )
-end as dist,
-(2*(yes1 * yes1) - 2*(case when no1 > 10 then no1 * no1 else 10 * no1 end) +
(case when yes2 < 5 then yes2 else 5 * yes2 end) - (case when no2 > 10 then no2
* no2 else 10 * no2 end)) weight1,
-case $weight_country_case when region $avoid_region= '$region' then 1 else 0
end weight_country,
-(yes3 * yes3) - (case when no3 > 5 then no3 * no3 else 5 * no3 end) weight2,
-last1, last2, last3, lastdt1, lastdt2, lastdt3, score, country, region, lng,
no4, no5
+select x.id as mirror_id, concat(case when support_scheme > 0 then
'$capability' else '$capabilityx' end, '://', uri) as url,
+dist,
+case $weight_country_case when region $avoid_region= '$region' then 1 else 0
end rating_country,
+score, country, region, lng,
+support_scheme,
+rating_scheme,
+support_ipv,
+rating_ipv
from (
select s.id,
concat($hostname,s.urldir,f.path,'/',s.name) as uri,
s.lat as lat,
s.lng as lng,
+case when $lat=0 and $lng=0 then 0
+else
+( 6371 * acos( cos( radians($lat) ) * cos( radians( lat ) ) * cos( radians(
lng ) - radians($lng) ) + sin( radians($lat) ) * sin( radians( lat ) ) ) )
+end as dist,
s.country, s.region, s.score,
-sum(case when chk.capability = '$capability' and chk.success then 1 else 0
end)/10 yes1,
-sum(case when chk.capability = '$capability' and not chk.success then 1 else 0
end) no1,
-sum(case when chk.capability = '$ipv' and chk.success then 1 else 0 end)/10
yes2,
-sum(case when chk.capability = '$ipv' and not chk.success then 1 else 0 end)
no2,
-sum(case when chk.capability = '$ipvx' and chk.success then 1 else 0 end)/10
yes3,
-sum(case when chk.capability = '$ipvx' and not chk.success then 1 else 0 end)
no3,
-sum(case when scd.server_id is not null or scf.server_id is not null then 1
else 0 end) no4,
-sum(case when scd2.server_id is not null or scf2.server_id is not null then 1
else 0 end) no5,
-(select success from server_capability_check where server_id = s.id and
capability = '$capability' order by dt desc limit 1) as last1,
-(select success from server_capability_check where server_id = s.id and
capability = '$ipv' order by dt desc limit 1) as last2,
-(select success from server_capability_check where server_id = s.id and
capability = '$ipvx' order by dt desc limit 1) as last3,
-(select date_trunc('minute',dt) from server_capability_check where server_id =
s.id and capability = '$capability' order by dt desc limit 1) as lastdt1,
-(select date_trunc('minute',dt) from server_capability_check where server_id =
s.id and capability = '$ipv' order by dt desc limit 1) as lastdt2,
-(select date_trunc('minute',dt) from server_capability_check where server_id =
s.id and capability = '$ipvx' order by dt desc limit 1) as lastdt3
+CASE WHEN COALESCE(stability_scheme.rating, 0) > 0 OR
COALESCE(stability_schemex.rating, 0) = 0 THEN 1 ELSE 0 END AS support_scheme,
-- we show here 0 only when opposite scheme is supported
+CASE WHEN COALESCE(stability_scheme.rating, 0) > 0 THEN
stability_scheme.rating WHEN COALESCE(stability_schemex.rating, 0) > 0 THEN
stability_schemex.rating ELSE 0 END AS rating_scheme,
+CASE WHEN COALESCE(stability_ipv.rating, 0) > 0 THEN 1 ELSE 0 END AS
support_ipv,
+CASE WHEN COALESCE(stability_ipv.rating, 0) > 0 THEN stability_ipv.rating
WHEN COALESCE(stability_ipvx.rating, 0) > 0 THEN stability_ipvx.rating
ELSE 0 END AS rating_ipv
from (
select s.*, fl.name
from folder_diff fd
@@ -127,18 +120,20 @@
where fd.folder_id = ? and fdf.file_id is NULL
) s
join folder f on f.id = ?
-left join server_capability_check chk on s.id = chk.server_id
-left join server_capability_declaration scd on s.id = scd.server_id and
scd.capability = '$capability' and NOT scd.enabled
-left join server_capability_force scf on s.id = scf.server_id and
scf.capability = '$capability'
+left join server_capability_declaration scd on s.id = scd.server_id and
scd.capability = '$capability' and NOT scd.enabled
+left join server_capability_force scf on s.id = scf.server_id and
scf.capability = '$capability'
left join server_capability_declaration scd2 on s.id = scd2.server_id and
scd.capability = '$ipv' and NOT scd.enabled
-left join server_capability_force scf2 on s.id = scf2.server_id and
scf2.capability = '$ipv'
-group by s.id, s.country, s.region, s.score, $hostname, s.urldir, s.name,
s.lat, s.lng, f.path
+left join server_capability_force scf2 on s.id = scf2.server_id and
scf2.capability = '$ipv'
+left join server_stability stability_scheme on s.id =
stability_scheme.server_id and stability_scheme.capability = '$capability'
+left join server_stability stability_schemex on s.id =
stability_schemex.server_id and stability_schemex.capability = '$capabilityx'
+left join server_stability stability_ipv on s.id = stability_ipv.server_id
and stability_ipv.capability = '$ipv'
+left join server_stability stability_ipvx on s.id =
stability_ipvx.server_id and stability_ipvx.capability = '$ipvx'
) x
$extra
-order by no4, no5, last1 desc nulls last, last2 desc nulls last,
weight_country desc, weight1 desc, weight2 desc, score, lastdt1 desc nulls
last, lastdt2 desc nulls last, last3 desc, lastdt3 desc, random()
-limit $limit
+order by rating_country desc, (dist/100)::int, support_scheme desc,
rating_scheme desc, support_ipv desc, rating_ipv desc, score, random()
+limit (case when $limit > 10 then $limit+$limit else 10 end)
) xx
-order by no4, no5, last1 desc nulls last, last2 desc nulls last,
weight_country desc, weight1 desc, (dist/100)::int, weight2 desc, score, last3
desc nulls last, dist, random()
+order by support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv
desc, rating_country desc, (dist/100)::int, score, random()
limit $limit;
END_SQL
my $prep = $dbh->prepare($sql);
@@ -199,8 +194,8 @@
my $sql = <<'END_SQL';
select concat(s.id, '::', p.id) as key,
- s.id as server_id,
- p.id as project_id,
+ s.id as server_id,
+ p.id as project_id,
concat(CASE WHEN length(s.hostname_vpn)>0 THEN s.hostname_vpn ELSE
s.hostname END,s.urldir, p.path) as uri,
sp.server_id as mirror_id,
coalesce(sp.state, -2) oldstate
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
new/MirrorCache-1.026/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
---
old/MirrorCache-1.025/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
2022-02-10 11:37:36.000000000 +0100
+++
new/MirrorCache-1.026/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
2022-03-08 11:08:06.000000000 +0100
@@ -34,16 +34,40 @@
COALESCE(fhttp.server_id = 0, COALESCE(http.enabled,'t')) as http,
COALESCE(fhttps.server_id = 0, COALESCE(https.enabled,'t')) as https,
COALESCE(fipv4.server_id = 0, COALESCE(ipv4.enabled,'t')) as ipv4,
- COALESCE(fipv6.server_id = 0, COALESCE(ipv6.enabled,'t')) as ipv6
+ COALESCE(fipv6.server_id = 0, COALESCE(ipv6.enabled,'t')) as ipv6,
+ stability_http.rating as rating_http,
+ stability_https.rating as rating_https,
+ stability_ipv4.rating as rating_ipv4,
+ stability_ipv6.rating as rating_ipv6,
+ extract(epoch from (now() - check_http.dt))/60 :: int as min_http,
+ extract(epoch from (now() - check_https.dt))/60 :: int as min_https,
+ extract(epoch from (now() - check_ipv4.dt))/60 :: int as min_ipv4,
+ extract(epoch from (now() - check_ipv6.dt))/60 :: int as min_ipv6
from server s
left join server_capability_declaration http on http.server_id = s.id
and http.capability = 'http'
left join server_capability_declaration https on https.server_id = s.id
and https.capability = 'https'
left join server_capability_declaration ipv4 on ipv4.server_id = s.id
and ipv4.capability = 'ipv4'
- left join server_capability_declaration ipv6 on ipv6.server_id = s.id
and ipv4.capability = 'ipv6'
+ left join server_capability_declaration ipv6 on ipv6.server_id = s.id
and ipv6.capability = 'ipv6'
left join server_capability_force fhttp on fhttp.server_id = s.id and
fhttp.capability = 'http'
left join server_capability_force fhttps on fhttps.server_id = s.id and
fhttps.capability = 'https'
left join server_capability_force fipv4 on fipv4.server_id = s.id and
fipv4.capability = 'ipv4'
left join server_capability_force fipv6 on fipv6.server_id = s.id and
fipv6.capability = 'ipv6'
+ left join server_stability stability_http on stability_http.server_id =
s.id and stability_http.capability = 'http'
+ left join server_stability stability_https on stability_https.server_id =
s.id and stability_https.capability = 'https'
+ left join server_stability stability_ipv4 on stability_ipv4.server_id =
s.id and stability_ipv4.capability = 'ipv4'
+ left join server_stability stability_ipv6 on stability_ipv6.server_id =
s.id and stability_ipv6.capability = 'ipv6'
+ left join (
+ select server_id, max(dt) as dt from server_capability_check x where
x.capability = 'http' and dt > now() - interval '24 hours' group by server_id
+ ) check_http on check_http.server_id = s.id
+ left join (
+ select server_id, max(dt) as dt from server_capability_check x where
x.capability = 'https' and dt > now() - interval '24 hours' group by server_id
+ ) check_https on check_https.server_id = s.id
+ left join (
+ select server_id, max(dt) as dt from server_capability_check x where
x.capability = 'ipv4' and dt > now() - interval '24 hours' group by server_id
+ ) check_ipv4 on check_ipv4.server_id = s.id
+ left join (
+ select server_id, max(dt) as dt from server_capability_check x where
x.capability = 'ipv6' and dt > now() - interval '24 hours' group by server_id
+ ) check_ipv6 on check_ipv6.server_id = s.id
where 't'
AND (fhttp.server_id IS NULL or fhttps.server_id IS NULL) -- do not show
servers which have both http and https force disabled
AND (fipv4.server_id IS NULL or fipv6.server_id IS NULL) -- do not show
servers which have both ipv4 and ipv6 force disabled
@@ -55,19 +79,57 @@
return $dbh->selectall_hashref($sql, 'id', {}, $country);
}
-sub log_probe_outcome {
- my ($self, $server_id, $capability, $success, $error) = @_;
+sub insert_stability_row {
+ my ($self, $server_id, $capability) = @_;
my $rsource = $self->result_source;
my $schema = $rsource->schema;
my $dbh = $schema->storage->dbh;
my $sql = <<'END_SQL';
+insert into server_stability(server_id, capability, rating, dt)
+values (?, ?, 1, now())
+END_SQL
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($server_id, $capability);
+}
+
+sub reset_stability {
+ my ($self, $server_id, $capability, $error) = @_;
+
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql = <<'END_SQL';
+update server_stability
+set dt = now(), rating = 0
+where server_id = ? and capability = ?
+END_SQL
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($server_id, $capability);
+
+ $sql = <<'END_SQL';
insert into server_capability_check(server_id, capability, dt, success, extra)
-values (?, ?, now(), ?, ?);
+values (?, ?, now(), 'f', ?);
+END_SQL
+ $prep = $dbh->prepare($sql);
+ $prep->execute($server_id, $capability, $error);
+}
+
+sub update_stability {
+ my ($self, $server_id, $capability, $rating) = @_;
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql = <<'END_SQL';
+update server_stability
+set dt = now(), rating = ?
+where server_id = ? and capability = ?
END_SQL
my $prep = $dbh->prepare($sql);
- $prep->execute($server_id, $capability, $success, $error);
+ $prep->execute($rating, $server_id, $capability);
}
sub search_all_downs {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/Task/MirrorProbe.pm
new/MirrorCache-1.026/lib/MirrorCache/Task/MirrorProbe.pm
--- old/MirrorCache-1.025/lib/MirrorCache/Task/MirrorProbe.pm 2022-02-10
11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/lib/MirrorCache/Task/MirrorProbe.pm 2022-03-08
11:08:06.000000000 +0100
@@ -14,6 +14,9 @@
# with this program; if not, see <http://www.gnu.org/licenses/>.
package MirrorCache::Task::MirrorProbe;
+
+use POSIX;
+
use Mojo::Base 'Mojolicious::Plugin';
use Mojo::UserAgent;
@@ -42,14 +45,26 @@
my $rs = $schema->resultset('ServerCapabilityDeclaration');
my $href = $rs->search_by_country($country);
- for my $id (sort keys %$href) {
+ my @server_ids = sort keys %$href;
+ for my $id (@server_ids) {
my $data = $href->{$id};
for my $capability (SERVER_CAPABILITIES) {
next unless ($data->{$capability});
- my $success = 1;
+ $rs->insert_stability_row($id, $capability) unless (defined
$data->{"rating_$capability"});
+
my $error = _probe_capability($data->{'uri'}, $capability);
- $success = 0 if $error;
- $rs->log_probe_outcome($data->{'id'}, $capability, $success,
$error);
+ if($error) {
+ $rs->reset_stability($data->{'id'}, $capability, $error);
+ next;
+ }
+ my $new_rating = 1000;
+ my $min = $data->{"min_$capability"};
+ # $job->note("$id$capability" => $min);
+ if (ceil($min // 0) > 0) {
+ $new_rating = 100 if $min < 24*60;
+ $new_rating = 10 if $min < 60;
+ }
+ $rs->update_stability($id, $capability, $new_rating) if
$new_rating != ($data->{"rating_$capability"} // 0);
}
}
$minion->unlock('mirror_probe_incomplete_for_' . $country);
@@ -121,7 +136,7 @@
my %count;
my @keys = sort keys %$href;
$job->note(total => scalar(@keys));
-
+
for my $id (@keys) {
my $data = $href->{$id};
my $oldstate = $data->{oldstate};
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.026/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.025/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-02-10
11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-03-08
11:08:06.000000000 +0100
@@ -94,12 +94,12 @@
# render_dir_remote tries to render dir when RootRemote cannot find it in DB
-my $RENDER_DIR_REMOTE_PROMISE_TIMEOUT = 15;
+my $RENDER_DIR_REMOTE_PROMISE_TIMEOUT =
int($ENV{MIRRORCACHE_RENDER_DIR_REMOTE_PROMISE_TIMEOUT} // 15) // 15;
sub render_dir_remote {
my $dm = shift;
my $dir = shift;
- my $c = $dm->c;
+ my $c = $dm->c;
my $tx = $c->render_later->tx;
my $job_id = 0;
@@ -110,7 +110,7 @@
}
my $handle_error = sub {
- my $reason = $_;
+ my $reason = shift;
if ($reason eq 'Promise timeout') {
return _render_dir($dm, $dir);
}
@@ -167,7 +167,7 @@
return undef if $trailing_slash || $path eq '/' || $dm->mirrorlist;
return undef if $dm->must_render_from_root;
my $subsidiary = $c->subsidiary;
- my $url = $subsidiary->has($dm->region, $c->req->url);
+ my $url = $subsidiary->has($dm, $c->req->url);
if ($url) {
$c->redirect_to($url);
$c->stat->redirect_to_region($dm);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.026/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.025/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2022-02-10 11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2022-03-08 11:08:06.000000000 +0100
@@ -210,7 +210,7 @@
sha256 => $file->{sha256},
};
- my @regions = $c->subsidiary->regions($region);
+ my @regions = $c->subsidiary->regions($region, $country);
$c->stash('nonavbar' => 1) if ($ENV{MIRRORCACHE_BRANDING});
$c->stash('mirrorlist' => 1);
my ($lat, $lng) = $dm->coord;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
new/MirrorCache-1.026/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
--- old/MirrorCache-1.025/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2022-02-10 11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2022-03-08 11:08:06.000000000 +0100
@@ -21,6 +21,7 @@
my $subsidiaries_initialized = 0;
my %subsidiary_urls;
+my %subsidiary_country; # countries that have dedicated instance
my %subsidiary_local;
my @regions;
@@ -42,6 +43,7 @@
$url = $url . $s->uri if $s->uri;
my $region = lc($s->region);
next unless $region;
+ $subsidiary_country{$region} = 1 unless ( $region =~
m/af|as|eu|na|oc|sa/ );
push @regions, $region;
my $weight = int($s->weight) // 1;
my $obj = Mojo::URL->new($url)->to_abs;
@@ -84,7 +86,7 @@
my $country = $dm->country;
my $region = $dm->region;
- my $url = _has_subsidiary($c, $dm->region);
+ my $url = _has_subsidiary($c, $dm);
return $c->render(status => 204, text => '') unless $url;
$url = $url->to_abs;
$url =~ s/http(s)?:\/\///;
@@ -100,7 +102,10 @@
sub _has_subsidiary {
return undef unless keys %subsidiary_urls;
- my ($c, $region, $origin_url) = @_;
+ my ($c, $dm, $origin_url) = @_;
+ my $region = $dm->country;
+ $region = $dm->region unless $subsidiary_country{$region};
+
my $arr = $subsidiary_urls{$region};
return undef if !$arr || 'ARRAY' ne ref $arr;
my $region_url = $arr->[rand @$arr]; # this how we respect weight of each
node
@@ -117,7 +122,8 @@
# return url for all subsidiaries
sub _regions {
return undef unless keys %subsidiary_urls;
- my ($c, $region) = @_;
+ my ($c, $region, $country) = @_;
+ $region = $country if $subsidiary_country{$country};
my $url = $subsidiary_urls{$region};
my @res = ($url? $region : '');
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/lib/MirrorCache/resources/migrations/pg.sql
new/MirrorCache-1.026/lib/MirrorCache/resources/migrations/pg.sql
--- old/MirrorCache-1.025/lib/MirrorCache/resources/migrations/pg.sql
2022-02-10 11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/lib/MirrorCache/resources/migrations/pg.sql
2022-03-08 11:08:06.000000000 +0100
@@ -262,4 +262,11 @@
drop constraint subsidiary_pkey,
add column weight int default '1';
-- 20 up
-CREATE INDEX if not exists folder_diff_id_index ON
folder_diff_file(folder_diff_id);
+create index if not exists folder_diff_id_index on
folder_diff_file(folder_diff_id);
+-- 21 up
+create table if not exists server_stability (
+ server_id int references server on delete cascade,
+ capability server_capability_t,
+ dt timestamp,
+ rating int -- 0 - bad, 1 - unknown, 10 - some issues last hour, 100 - some
issues last 24 hours, 1000 - no issues recorder last 24 hours.
+);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.025/t/environ/02-cleanup.sh
new/MirrorCache-1.026/t/environ/02-cleanup.sh
--- old/MirrorCache-1.025/t/environ/02-cleanup.sh 2022-02-10
11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/t/environ/02-cleanup.sh 2022-03-08
11:08:06.000000000 +0100
@@ -54,7 +54,12 @@
test 4 == $($mc/db/sql "select count(*) from folder_diff")
test 4 == $($mc/db/sql "select count(*) from folder_diff_file")
-test 8 == $($mc/db/sql "select count(*) from server_capability_check")
+test 4 == $($mc/db/sql "select count(*) from server_capability_check")
+test 8 == $($mc/db/sql "select count(*) from server_stability")
+test 2 == $($mc/db/sql "select count(*) from server_stability where capability
= 'http' and rating > 0")
+test 2 == $($mc/db/sql "select count(*) from server_stability where capability
= 'https' and rating = 0")
+test 2 == $($mc/db/sql "select count(*) from server_stability where capability
= 'ipv4' and rating > 0")
+test 2 == $($mc/db/sql "select count(*) from server_stability where capability
= 'ipv6' and rating = 0")
# update dt to look older and save number of audit events
$mc/db/sql "update audit_event set dt = dt - interval '50 day'"
@@ -67,6 +72,6 @@
# test for reduced number of rows
test 2 == $($mc/db/sql "select count(*) from folder_diff")
test 3 == $($mc/db/sql "select count(*) from folder_diff_file")
-test 4 == $($mc/db/sql "select count(*) from server_capability_check")
+test 2 == $($mc/db/sql "select count(*) from server_capability_check")
test $audit_events -gt $($mc/db/sql "select count(*) from audit_event")
echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.025/t/environ/03-headquarter-subsidiaries-country.sh
new/MirrorCache-1.026/t/environ/03-headquarter-subsidiaries-country.sh
--- old/MirrorCache-1.025/t/environ/03-headquarter-subsidiaries-country.sh
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.026/t/environ/03-headquarter-subsidiaries-country.sh
2022-03-08 11:08:06.000000000 +0100
@@ -0,0 +1,69 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+# environ by number:
+# 9 - headquarter
+# 6 - NA subsidiary
+# 7 - EU subsidiary
+# 8 - Japan subsidiary
+
+for i in 6 7 8 9; do
+ x=$(environ mc$i $(pwd))
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+ eval mc$i=$x
+done
+
+hq_address=$($mc9/print_address)
+na_address=$($mc6/print_address)
+na_interface=127.0.0.2
+eu_address=$($mc7/print_address)
+eu_interface=127.0.0.3
+cn_address=$($mc8/print_address)
+cn_interface=127.0.0.4
+
+# deploy db
+$mc9/gen_env MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'
+$mc9/backstage/shoot
+
+$mc9/db/sql "insert into subsidiary(hostname,region) select '$na_address','na'"
+$mc9/db/sql "insert into subsidiary(hostname,region) select '$eu_address','eu'"
+$mc9/db/sql "insert into subsidiary(hostname,region) select '$cn_address','cn'"
+
+$mc9/start
+$mc6/gen_env MIRRORCACHE_REGION=na
+$mc6/start
+$mc7/gen_env MIRRORCACHE_REGION=eu
+$mc7/start
+$mc8/gen_env MIRRORCACHE_REGION=cn
+$mc8/start
+
+echo the root folder is not redirected
+curl --interface $eu_interface -Is http://$hq_address/ | grep '200 OK'
+
+echo check redirection from headquarter
+curl --interface $na_interface -Is
http://$hq_address/download/folder1/file1.1.dat | grep "Location:
http://$na_address/download/folder1/file1.1.dat";
+curl --interface $eu_interface -Is
http://$hq_address/download/folder1/file1.1.dat | grep "Location:
http://$eu_address/download/folder1/file1.1.dat";
+curl --interface $cn_interface -Is
http://$hq_address/download/folder1/file1.1.dat | grep "Location:
http://$cn_address/download/folder1/file1.1.dat";
+
+curl -si --interface $eu_interface http://$hq_address/geoip | grep -A 50
'200 OK' | grep "<host>$eu_address</host>"
+curl -si --interface $cn_interface http://$hq_address/geoip | grep -A 50
'200 OK' | grep "<host>$cn_address</host>"
+
+echo check redirection from na
+curl --interface $na_interface -Is
http://$na_address/download/folder1/file1.1.dat | grep '200 OK'
+curl --interface $eu_interface -Is
http://$na_address/download/folder1/file1.1.dat | grep '200 OK'
+
+echo check redirection from eu
+curl --interface $eu_interface -Is
http://$eu_address/download/folder1/file1.1.dat | grep '200 OK'
+
+echo check redirection from cn
+curl --interface $cn_interface -Is
http://$cn_address/download/folder1/file1.1.dat | grep '200 OK'
+curl --interface $cn_interface -Is
http://$cn_address/download/folder1/file1.1.dat?COUNTRY=cn | grep '200 OK'
+
+echo check non-download routers shouldnt be redirected
+curl --interface $na_interface -Is http://$hq_address/rest/server | grep '200
OK'
+curl --interface $eu_interface -Is http://$hq_address/rest/server | grep '200
OK'
+curl --interface $cn_interface -Is http://$hq_address/rest/server | grep '200
OK'
+curl --interface $cn_interface -Is http://$cn_address/rest/server | grep '200
OK'
+curl --interface $na_interface -Is http://$na_address/rest/server | grep '200
OK'
+curl --interface $eu_interface -Is http://$eu_address/rest/server | grep '200
OK'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.025/t/environ/04-remote-folder.sh
new/MirrorCache-1.026/t/environ/04-remote-folder.sh
--- old/MirrorCache-1.025/t/environ/04-remote-folder.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.026/t/environ/04-remote-folder.sh 2022-03-08
11:08:06.000000000 +0100
@@ -0,0 +1,24 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+ap=$(environ ap)
+
+$mc/gen_env \
+ MIRRORCACHE_ROOT=http://$($ap/print_address) \
+ MIRRORCACHE_RENDER_DIR_REMOTE_PROMISE_TIMEOUT=1
+
+mkdir -p $ap/dt/{folder1,folder2,folder3}
+touch $ap/dt/folder1/file1.1.dat
+
+$ap/start
+
+$mc/start
+$mc/status
+
+$mc/curl /download/folder1/ | grep 'Waiting in queue'
+$mc/backstage/shoot
+$mc/curl /download/folder1/ | grep file1.1.dat
+
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.025/t/environ/09-stability-01-probe.sh
new/MirrorCache-1.026/t/environ/09-stability-01-probe.sh
--- old/MirrorCache-1.025/t/environ/09-stability-01-probe.sh 2022-02-10
11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/t/environ/09-stability-01-probe.sh 2022-03-08
11:08:06.000000000 +0100
@@ -41,13 +41,13 @@
$mc/backstage/shoot
# log current audit_event count
-cnt="$($mc/db/sql "select count(*) from audit_event")"
+cnt="$($mc/sql "select count(*) from audit_event")"
# make sure now it redirects to ap8
$mc/curl -I /download/folder1/file2.1.dat | grep $($ap8/print_address)
# audit event shouldn't contain recent mirror_probe event, becuase we know
that ap7 is not preferable because of recent probe error
-test 0 == $($mc/db/sql "select count(*) from audit_event where name =
'mirror_probe' and id > $cnt")
+$mc/sql_test 0 == "select count(*) from audit_event where name =
'mirror_probe' and id > $cnt"
# now shut down ap8 and start ap7, then probe mirrors explicitly
$ap8/stop
@@ -59,4 +59,20 @@
# make sure now it redirects to ap7
$mc/curl -I /download/folder1/file2.1.dat | grep $($ap7/print_address)
# audit event shouldn't contain recent mirror_probe event, becuase we know
that ap7 is not preferable because of recent probe error
-test 0 == $($mc/db/sql "select count(*) from audit_event where name =
'mirror_probe' and id > $cnt")
+$mc/sql_test 0 == "select count(*) from audit_event where name =
'mirror_probe' and id > $cnt"
+
+
+$mc/sql_test 0 == "select rating from server_stability where (server_id,
capability) = (2, 'http')"
+$mc/sql_test 10 == "select rating from server_stability where (server_id,
capability) = (1, 'http')"
+
+$mc/sql "update server_capability_check set dt = dt - interval '1 hour' where
(server_id, capability) = (1, 'http')"
+$mc/backstage/job -e mirror_probe -a '["us"]'
+$mc/backstage/shoot
+$mc/sql_test 100 == "select rating from server_stability where (server_id,
capability) = (1, 'http')"
+
+$mc/sql "update server_capability_check set dt = dt - interval '24 hour' where
(server_id, capability) = (1, 'http')"
+$mc/backstage/job -e mirror_probe -a '["us"]'
+$mc/backstage/shoot
+$mc/sql_test 1000 == "select rating from server_stability where (server_id,
capability) = (1, 'http')"
+
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.025/t/environ/10-https-probe.sh
new/MirrorCache-1.026/t/environ/10-https-probe.sh
--- old/MirrorCache-1.025/t/environ/10-https-probe.sh 2022-02-10
11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/t/environ/10-https-probe.sh 2022-03-08
11:08:06.000000000 +0100
@@ -61,8 +61,6 @@
$mc/backstage/job -e mirror_probe -a '["us"]'
$mc/backstage/shoot
test f == $($mc/db/sql "select success from server_capability_check where
server_id=1 and capability='http'")
-test t == $($mc/db/sql "select success from server_capability_check where
server_id=1 and capability='https'")
-test t == $($mc/db/sql "select success from server_capability_check where
server_id=2 and capability='http'")
test f == $($mc/db/sql "select success from server_capability_check where
server_id=2 and capability='https'")
# now explicitly force disable corresponding capabilities
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.025/t/environ/10-ipv-probe.sh
new/MirrorCache-1.026/t/environ/10-ipv-probe.sh
--- old/MirrorCache-1.025/t/environ/10-ipv-probe.sh 2022-02-10
11:37:36.000000000 +0100
+++ new/MirrorCache-1.026/t/environ/10-ipv-probe.sh 2022-03-08
11:08:06.000000000 +0100
@@ -35,10 +35,8 @@
$mc/backstage/job -e mirror_probe -a '["us"]'
$mc/backstage/shoot
-test t == $($mc/db/sql "select success from server_capability_check where
server_id=1 and capability='ipv4'")
test f == $($mc/db/sql "select success from server_capability_check where
server_id=1 and capability='ipv6'")
test f == $($mc/db/sql "select success from server_capability_check where
server_id=2 and capability='ipv4'")
-test t == $($mc/db/sql "select success from server_capability_check where
server_id=2 and capability='ipv6'")
# now explicitly force disable corresponding capabilities
$mc/db/sql "insert into server_capability_force(server_id,capability,dt)
select 1,'ipv6',now()"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.025/t/manual/02-headquarter-country.sh
new/MirrorCache-1.026/t/manual/02-headquarter-country.sh
--- old/MirrorCache-1.025/t/manual/02-headquarter-country.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.026/t/manual/02-headquarter-country.sh 2022-03-08
11:08:06.000000000 +0100
@@ -0,0 +1,65 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+hq=$(environ mc1 $(pwd))
+us=$(environ mc2 $(pwd))
+eu=$(environ mc3 $(pwd))
+oc=$(environ mc4 $(pwd))
+
+hq_address=$($hq/print_address)
+us_address=$($us/print_address)
+us_interface=127.0.0.2
+eu_address=$($eu/print_address)
+eu_interface=127.0.0.3
+oc_address=$($oc/print_address)
+oc_interface=127.0.0.4
+
+declare -A xtra
+xtra[$hq]=''
+xtra[$eu]="MIRRORCACHE_REGION=eu MIRRORCACHE_HEADQUARTER=$hq_address"
+xtra[$us]="MIRRORCACHE_REGION=us MIRRORCACHE_HEADQUARTER=$hq_address"
+xtra[$oc]="MIRRORCACHE_REGION=oc MIRRORCACHE_HEADQUARTER=$hq_address"
+
+for x in $hq $eu $us $oc; do
+
+$x/gen_env MIRRORCACHE_RECKLESS=0 \
+ MIRRORCACHE_ROOT=http://download.opensuse.org \
+ MIRRORCACHE_REDIRECT=downloadcontent.opensuse.org \
+ MIRRORCACHE_HYPNOTOAD=1 \
+ MIRRORCACHE_PERMANENT_JOBS="'folder_sync_schedule_from_misses
folder_sync_schedule mirror_scan_schedule_from_misses
mirror_scan_schedule_from_path_errors mirror_scan_schedule cleanup
stat_agg_schedule mirror_check_from_stat'" \
+ MIRRORCACHE_TOP_FOLDERS="'debug distribution tumbleweed factory
repositories'" \
+ MIRRORCACHE_TRUST_AUTH=127.0.0.16 \
+ MIRRORCACHE_PROXY_URL=http://$($x/print_address) \
+ MIRRORCACHE_BACKSTAGE_WORKERS=4 \
+ ${xtra[$x]}
+
+ $x/backstage/start # start backstage here to deploy db
+done
+
+
+$hq/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/mirrors-rest.sql
mc_test
+$hq/sql "update server set enabled='f' where region = 'oc'"
+$hq/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/mirrors-na.sql
mc_test
+$hq/sql "delete from server where country = 'us'"
+$hq/sql "insert into subsidiary(hostname,region) select '$us_address','us'"
+$hq/sql "insert into subsidiary(hostname,region) select '$eu_address','eu'"
+$hq/sql "insert into subsidiary(hostname,region) select '$oc_address','oc'"
+$hq/start
+
+$us/start
+$us/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/mirrors-na.sql
mc_test
+$us/sql "delete from server where country != 'us'"
+
+$eu/start
+$eu/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/mirrors-eu.sql
mc_test
+
+$oc/start
+$oc/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/mirrors-rest.sql
mc_test
+$oc/sql "update server set enabled='f' where region != 'oc'"
+
+# $hq/backstage/job -e folder_tree -a '["/distribution/leap/15.3"]'
+# $us/backstage/job -e folder_tree -a '["/distribution/leap/15.3/iso"]'
+
+curl -IL
http://127.0.0.1:3110/distribution/leap/15.3/iso/openSUSE-Leap-15.3-2-NET-x86_64-Current.iso
+sleep 10
+$us/curl -IL
/distribution/leap/15.3/iso/openSUSE-Leap-15.3-2-NET-x86_64-Current.iso
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.EKrmnh/_old 2022-03-16 20:21:49.825029123 +0100
+++ /var/tmp/diff_new_pack.EKrmnh/_new 2022-03-16 20:21:49.829029126 +0100
@@ -1,5 +1,5 @@
name: MirrorCache
-version: 1.025
-mtime: 1644489456
-commit: a025a68eac1adc2c1bdaf5df1c45dbc011bf94bd
+version: 1.026
+mtime: 1646734086
+commit: 5d9a1dc1a57882ca2742876fa162f5eda91200c1
