Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2023-01-19 16:44:11
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.32243 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Thu Jan 19 16:44:11 2023 rev:23 rq:1059502 version:1.052
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2023-01-05
15:01:54.577399206 +0100
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.32243/MirrorCache.changes
2023-01-19 16:44:28.333852717 +0100
@@ -1,0 +2,11 @@
+Thu Jan 12 08:43:17 UTC 2023 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.052:
+ * Detect redirects with remote root (#336)
+ * Add json parsing to FolderSync task (#337)
+ * Redirect Current.iso when no nfs mount (#339)
+ * Improve sorting of files in JavaScript (#340)
+ * Redirect huge files (#342)
+ * Secure last_id for jobs monitoring stat (#341)
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.051.obscpio
New:
----
MirrorCache-1.052.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.NONKJq/_old 2023-01-19 16:44:29.093856675 +0100
+++ /var/tmp/diff_new_pack.NONKJq/_new 2023-01-19 16:44:29.097856696 +0100
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
perl(Config::IniFiles)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.051
+Version: 1.052
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.051.obscpio -> MirrorCache-1.052.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/assets/javascripts/browse.js
new/MirrorCache-1.052/assets/javascripts/browse.js
--- old/MirrorCache-1.051/assets/javascripts/browse.js 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/assets/javascripts/browse.js 2023-01-12
09:40:42.000000000 +0100
@@ -8,6 +8,7 @@
data: 'name',
className: 'name',
defaultContent: "",
+ type: "version-string",
render: function (data, type, row, meta) {
if(type === 'display'){
var d = data;
@@ -76,6 +77,16 @@
}
});
+ jQuery.extend( jQuery.fn.dataTableExt.oSort, {
+ "version-string-asc" : function (a, b) {
+ return a.localeCompare(b, undefined, { numeric: true, sensitivity:
'base' });
+ },
+ "version-string-desc" : function (a, b) {
+ return b.localeCompare(a, undefined, { numeric: true, sensitivity:
'base' });
+ }
+ });
+
+
var url = $("#browse_api_url").val();
var table = $('.browsetable');
var dataTable = table.DataTable({
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/lib/MirrorCache/Config.pm
new/MirrorCache-1.052/lib/MirrorCache/Config.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Config.pm 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Config.pm 2023-01-12
09:40:42.000000000 +0100
@@ -34,6 +34,9 @@
has redirect => $ENV{MIRRORCACHE_REDIRECT};
has redirect_vpn => $ENV{MIRRORCACHE_REDIRECT_VPN};
+has redirect_huge => $ENV{MIRRORCACHE_REDIRECT_HUGE};
+has huge_file_size => int($ENV{MIRRORCACHE_HUGE_FILE_SIZE} // 0) ||
40*1024*1024;
+
has plugin_status => $ENV{MIRRORCACHE_PLUGIN_STATUS};
has db_provider => undef;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/Schema/ResultSet/Folder.pm
new/MirrorCache-1.052/lib/MirrorCache/Schema/ResultSet/Folder.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Schema/ResultSet/Folder.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Schema/ResultSet/Folder.pm
2023-01-12 09:40:42.000000000 +0100
@@ -375,4 +375,31 @@
$self->delete_cascade($id, 1);
}
+sub add_redirect {
+ my ($self, $pathfrom, $pathto) = @_;
+
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql;
+if ($dbh->{Driver}->{Name} eq 'Pg') {
+ $sql = <<'END_SQL';
+insert into redirect(pathfrom, pathto)
+values (?, ?)
+on conflict(pathfrom) do update set
+pathto = ?
+END_SQL
+} else {
+ $sql = <<'END_SQL';
+insert into redirect(pathfrom, pathto)
+values (?, ?)
+on duplicate key update
+pathto = ?
+END_SQL
+}
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($pathfrom, $pathto, $pathto);
+}
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/Schema/ResultSet/Stat.pm
new/MirrorCache-1.052/lib/MirrorCache/Schema/ResultSet/Stat.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Schema/ResultSet/Stat.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Schema/ResultSet/Stat.pm
2023-01-12 09:40:42.000000000 +0100
@@ -254,4 +254,13 @@
return ($id, \@folder_ids, \@country_list);
}
+sub secure_max_id {
+ my ($self, $prev_stat_id) = @_;
+ my $max_id = $self->get_column("id")->max;
+
+ $prev_stat_id = $max_id - 100000 if !$prev_stat_id || $max_id -
$prev_stat_id > 10000 || $prev_stat_id > $max_id;
+ $prev_stat_id = 0 if $prev_stat_id < 0;
+ return $prev_stat_id;
+}
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/lib/MirrorCache/Task/Cleanup.pm
new/MirrorCache-1.052/lib/MirrorCache/Task/Cleanup.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Task/Cleanup.pm 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Task/Cleanup.pm 2023-01-12
09:40:42.000000000 +0100
@@ -34,10 +34,24 @@
my $schema = $app->schema;
$minion->enqueue('mirror_probe_projects');
- # purge unreferenced folder_diff
+ # detect stalled backstage jobs
my $sql;
if ($schema->pg) {
$sql = <<'END_SQL';
+update minion_jobs set state = 'failed', result = concat("killed; old result:
", result) where state = 'active' and started < now() - interval '1 hour'
+END_SQL
+ } else {
+$sql = <<'END_SQL';
+update minion_jobs set state = 'failed', result = concat_ws("", "killed; old
result: ", result) where state = 'active' and started < now() - interval 2 hour
+END_SQL
+}
+ eval {
+ $schema->storage->dbh->prepare($sql)->execute();
+ 1;
+ } or $job->note(last_warning_0 => $@, at_0 => datetime_now());
+
+ if ($schema->pg) {
+$sql = <<'END_SQL';
with DiffToDelete as (
select fd.id
from folder_diff fd
@@ -65,7 +79,7 @@
eval {
$schema->storage->dbh->prepare($sql)->execute();
1;
- } or $job->note(last_warning => $@, at => datetime_now());
+ } or $job->note(last_warning_1 => $@, at_1 => datetime_now());
# delete rows from server_capability_check to avoid overload
my $sqlservercap;
@@ -84,7 +98,7 @@
eval {
$schema->storage->dbh->prepare($sqlservercap)->execute();
1;
- } or $job->note(last_warning => $@, at => datetime_now());
+ } or $job->note(last_warning_2 => $@, at_2 => datetime_now());
# delete rows from audit_event
my $fail_count;
@@ -92,7 +106,7 @@
eval {
($fail_count, $last_warning) =
$schema->resultset('AuditEvent')->cleanup_audit_events($app);
$fail_count ? 0 : 1;
- } or $job->note(last_warning => $last_warning, delete_fail_count =>
$fail_count, at => datetime_now());
+ } or $job->note(last_warning_3 => $last_warning, delete_fail_count =>
$fail_count, at_3 => datetime_now());
# delete rows from stat
my $sqlstat;
@@ -111,7 +125,7 @@
eval {
$schema->storage->dbh->prepare($sqlservercap)->execute();
1;
- } or $job->note(last_warning => $@, at => datetime_now());
+ } or $job->note(last_warning_4 => $@, at_4 => datetime_now());
return $job->retry({delay => $DELAY});
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/lib/MirrorCache/Task/FolderSync.pm
new/MirrorCache-1.052/lib/MirrorCache/Task/FolderSync.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Task/FolderSync.pm 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Task/FolderSync.pm 2023-01-12
09:40:42.000000000 +0100
@@ -39,11 +39,13 @@
my $root = $app->mc->root;
$job->note($path => 1);
- my $realpath = $root->realpath($path);
+ my $realpath = $root->realpath($path, 1);
$realpath = $path unless $realpath;
if ($realpath ne $path) {
$job->note(realpath => $realpath);
$job->note($realpath => 1);
+
+ $schema->resultset('Folder')->add_redirect($path, $realpath);
}
my $folder = $schema->resultset('Folder')->find({path => $realpath});
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
new/MirrorCache-1.052/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
2023-01-12 09:40:42.000000000 +0100
@@ -46,6 +46,9 @@
my $schema = $app->schema;
my $limit = 10000;
+ $prev_stat_id = $schema->resultset('Stat')->secure_max_id($prev_stat_id);
+ print(STDERR "$pref id after adjust: $prev_stat_id\n") if $MCDEBUG;
+
my ($stat_id, $folders, $country_list) =
$schema->resultset('Stat')->path_misses($prev_stat_id, $limit);
$common_guard = undef;
my $rs = $schema->resultset('Folder');
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/Task/MirrorCheckFromStat.pm
new/MirrorCache-1.052/lib/MirrorCache/Task/MirrorCheckFromStat.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Task/MirrorCheckFromStat.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Task/MirrorCheckFromStat.pm
2023-01-12 09:40:42.000000000 +0100
@@ -45,6 +45,8 @@
my $schema = $app->schema;
+ $prev_stat_id = $schema->resultset('Stat')->secure_max_id($prev_stat_id);
+
my ($stat_id, $mirror_id, $country, $url, $folder, $folder_id) =
$schema->resultset('Stat')->latest_hit($prev_stat_id);
my $last_run = 0;
while ($stat_id && $stat_id > $prev_stat_id) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
new/MirrorCache-1.052/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
--- old/MirrorCache-1.051/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
2023-01-12 09:40:42.000000000 +0100
@@ -47,6 +47,8 @@
my $schema = $app->schema;
my $limit = $prev_stat_id ? 1000 : 10;
+ $prev_stat_id = $schema->resultset('Stat')->secure_max_id($prev_stat_id);
+
my ($stat_id, $folder_ids, $country_list) =
$schema->resultset('Stat')->mirror_misses($prev_stat_id, $limit);
$common_guard = undef;
my $rs = $schema->resultset('Folder');
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2023-01-12
09:40:42.000000000 +0100
@@ -27,13 +27,17 @@
use MirrorCache::Datamodule;
my $root;
+my $mc_config;
my @top_folders;
+my $MCDEBUG = $ENV{MCDEBUG_DIR} // $ENV{MCDEBUG_ALL} // 0;
+
sub register {
my $self = shift;
my $app = shift;
$root = $app->mc->root;
+ $mc_config = $app->mc->config;
if ($ENV{MIRRORCACHE_TOP_FOLDERS}) {
@top_folders = split /[:,\s]+/, $ENV{MIRRORCACHE_TOP_FOLDERS};
@@ -167,19 +171,25 @@
my $c = $dm->c;
# each project may have a redirect defined in DB, so all requests are
redirected for it
- my $redirect = $c->mcproject->redirect($path, $dm->region);
- if ($redirect) {
- $dm->redirect($dm->scheme . '://' . $redirect . $path);
- $c->stat->redirect_to_region($dm);
- return 1;
- }
+ unless ($trailing_slash) {
+ my $redirect = $c->mcproject->redirect($path, $dm->region);
+ if ($redirect) {
+ $dm->redirect($dm->scheme . '://' . $redirect . $path);
+ $c->stat->redirect_to_region($dm);
+ return 1;
+ }
- my $ln = $root->detect_ln($path);
- if ($ln) {
- # redirect to the symlink
- $dm->redirect($dm->route . $ln);
- $c->stat->redirect_to_region($dm);
- return 1;
+ $c->log->error('pedantic: ' . ($dm->pedantic // 'undef')) if $MCDEBUG;
+ if ($path =~ m/.*(Media|Current)\.iso(\.sha256(\.asc)?)?/ &&
$dm->pedantic) {
+ my $ln = $root->detect_ln_in_the_same_folder($path);
+ $c->log->error("ln for $path : " . ($ln // 'null')) if $MCDEBUG;
+ if ($ln) {
+ # redirect to the symlink
+ $c->log->error('redirect detected: ' . $ln) if $MCDEBUG;
+ $dm->redirect($dm->route . $ln);
+ return 1;
+ }
+ }
}
return undef if $trailing_slash || $path eq '/' || $dm->mirrorlist;
return undef if $dm->must_render_from_root;
@@ -278,8 +288,6 @@
return 1;
}
-my $MCDEBUG = $ENV{MCDEBUG_DIR} // $ENV{MCDEBUG_ALL} // 0;
-
sub _render_from_db {
my $dm = shift;
my $c = $dm->c;
@@ -291,7 +299,8 @@
$c->log->error($c->dumper('$file_pattern_in_folder',
$file_pattern_in_folder)) if $MCDEBUG;
if ( (!$trailing_slash && $path ne '/') || $file_pattern_in_folder ) {
my $f = Mojo::File->new($path);
- my $dirname = $root->realpath($file_pattern_in_folder? $path :
$f->dirname);
+ my $dirname = ($file_pattern_in_folder? $path : $f->dirname);
+ $dirname = $root->realpath($dirname);
$dirname = $dm->root_subtree . ($file_pattern_in_folder? $path :
$f->dirname) unless $dirname;
$c->log->error($c->dumper('dirname:', $dirname)) if $MCDEBUG;
if (my $parent_folder = $rsFolder->find({path => $dirname})) {
@@ -310,7 +319,7 @@
my $xtra = '';
$xtra = '.zsync' if $dm->zsync && !$dm->accept_zsync;
my $file;
- $c->log->error($c->dumper('parent_folder:', $parent_folder)) if
$MCDEBUG;
+ $c->log->error($c->dumper('parent_folder:', $parent_folder->path))
if $MCDEBUG;
$file =
$schema->resultset('File')->find_with_hash($parent_folder->id, $f->basename,
$xtra, $dm->regex, $dm->glob_regex) if $parent_folder;
$c->log->error($c->dumper('file:', $f->basename, $file)) if
$MCDEBUG;
@@ -323,6 +332,12 @@
$dm->_path($dm->path . '.zsync');
$path = $path . '.zsync';
}
+
+ $c->log->error($c->dumper('file_size: ', $file->{size} //
'undef', 'huge_file_size: ', $mc_config->huge_file_size)) if $MCDEBUG;
+ if ($root->is_remote && !$dm->extra && $file->{size} &&
$mc_config->redirect_huge && $mc_config->huge_file_size <= $file->{size}) {
+ $dm->redirect($dm->scheme . '://' .
$mc_config->redirect_huge . $path);
+ return 1;
+ }
if ($file->{target}) {
# redirect to the symlink
$dm->redirect($dm->route . $dirname . '/' .
$file->{target});
@@ -335,6 +350,7 @@
}
}
} elsif (my $folder = $rsFolder->find_folder_or_redirect($dm->root_subtree
. $path)) {
+ $c->log->error('found redirect : ', $folder->{pathto}) if $MCDEBUG &&
$folder->{pathto};
return $dm->redirect($folder->{pathto}) if $folder->{pathto};
# folder must have trailing slash, otherwise it will be a challenge to
render links on webpage
$dm->folder_id($folder->{id});
@@ -353,6 +369,7 @@
my ($path, $trailing_slash) = $dm->path;
if ($dm->extra) {
+ $c->log->error('guess what to render extra : ', $dm->extra) if
$MCDEBUG;
return $root->render_file($dm, $path) if $dm->accept_all &&
!$trailing_slash;
# the file is unknown, we cannot show generate meither mirrorlist or
metalink
my $res = $c->render(status => 425, text => "The file is unknown,
retry later");
@@ -373,6 +390,7 @@
# - redirected to another route => we must redirect it as well
my $path1 = $path . '/';
my $url1 = $url . '/'; # let's check if it is a folder
+ my $redirect;
$ua->head_p($url1, {'User-Agent' =>
'MirrorCache/guess_what_to_render'})->then(sub {
my $res = shift->res;
@@ -394,7 +412,9 @@
if ($rootlocation eq substr($location, 0,
length($rootlocation))) {
$location = substr($location,
length($rootlocation));
}
- return $dm->redirect($dm->route . $location .
$trailing_slash)
+ $c->log->error('redirect guessed: ' . $location .
$trailing_slash) if $MCDEBUG;
+ $redirect = 1;
+ return $dm->redirect($dm->route . $location .
$trailing_slash);
}
}
}
@@ -412,11 +432,13 @@
$c->app->log->fatal($msg); # it is not fatal, but needed in production
log
my $reftx = $tx;
my $refua = $ua;
- })->timeout(2)->wait;
+ })->timeout(5)->then(sub {
+
$c->backstage->enqueue_unless_scheduled_with_parameter_or_limit('folder_sync',
$path) if $redirect;
+ })->wait;
}
sub _by_filename {
- (($b->{dir} // 0) cmp ($a->{dir} // 0)) ||
+ versioncmp(lc($b->{dir} // ''), lc($a->{dir} // '')) ||
versioncmp(lc($a->{name} // ''), lc($b->{name} // ''));
}
@@ -484,31 +506,31 @@
my $basename = $child->{name};
my $size = $child->{size};
my $mtime = $child->{mtime};
+ my $desc = $folderDesc{$c->mcbranding}{$dir}{$basename};
if ($json) {
push @files, {
name => $basename,
size => $size,
mtime => $mtime,
- desc => $folderDesc{$c->mcbranding}{$dir}{$basename},
};
- next;
- }
- $size = MirrorCache::Utils::human_readable_size($size) if $size;
- $mtime = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if $mtime;
+ } else {
+ $size = MirrorCache::Utils::human_readable_size($size) if
$size;
+ $mtime = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if
$mtime;
- my $is_dir = '/' eq substr($basename, -1)? 1 : 0;
- my $encoded = Encode::decode_utf8( './' . $basename );
- my $mime_type = $dm->mime || 'text/plain';
+ my $is_dir = '/' eq substr($basename, -1)? 1 : 0;
+ my $encoded = Encode::decode_utf8( './' . $basename );
+ my $mime_type = $dm->mime || 'text/plain';
- push @files, {
- url => $encoded,
- name => $basename,
- size => $size,
- type => $mime_type,
- mtime => $mtime,
- dir => $is_dir,
- desc => $folderDesc{$c->mcbranding}{$dir}{$basename},
- };
+ push @files, {
+ url => $encoded,
+ name => $basename,
+ size => $size,
+ type => $mime_type,
+ mtime => $mtime,
+ dir => $is_dir,
+ };
+ }
+ $files[-1]->{desc} = $desc if $desc;
}
my @items = sort _by_filename @files;
return $c->render( json => { data => \@items } ) if $dm->jsontable;
@@ -536,32 +558,31 @@
$basename = $basename . '/' if $stat && -d $stat;
my $size = $stat->size if $stat;
my $mtime = $stat->mtime if $stat;
+ my $desc = $folderDesc{$c->mcbranding}{$dir}{$basename};
if ($json) {
push @files, {
name => $basename,
size => $size,
mtime => $mtime,
- desc => $folderDesc{$c->mcbranding}{$dir}{$basename},
};
- next;
- }
-
- $size = MirrorCache::Utils::human_readable_size($size) if $size;
- $mtime = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if $mtime;
+ } else {
+ $size = MirrorCache::Utils::human_readable_size($size) if
$size;
+ $mtime = strftime("%d-%b-%Y %H:%M", gmtime($mtime)) if
$mtime;
- my $is_dir = '/' eq substr($basename, -1)? 1 : 0;
- my $encoded = Encode::decode_utf8( './' . $basename );
- my $mime_type = $dm->mime || 'text/plain';
+ my $is_dir = '/' eq substr($basename, -1)? 1 : 0;
+ my $encoded = Encode::decode_utf8( './' . $basename );
+ my $mime_type = $dm->mime || 'text/plain';
- push @files, {
- url => $encoded,
- name => $basename,
- size => $size,
- type => $mime_type,
- mtime => $mtime,
- dir => $is_dir,
- desc => $folderDesc{$c->mcbranding}{$dir}{$basename},
- };
+ push @files, {
+ url => $encoded,
+ name => $basename,
+ size => $size,
+ type => $mime_type,
+ mtime => $mtime,
+ dir => $is_dir,
+ };
+ }
+ $files[-1]->{desc} = $desc if $desc;
}
my @items = sort _by_filename @files;
return $c->render( json => { data => \@items } ) if $dm->jsontable;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2023-01-12 09:40:42.000000000 +0100
@@ -29,6 +29,7 @@
use Mojo::IOLoop::Subprocess;
my $MCDEBUG = $ENV{MCDEBUG_RENDER_FILE_FROM_MIRROR} // $ENV{MCDEBUG_ALL} // 0;
+my $mc_config;
sub register {
my ($self, $app) = @_;
@@ -39,6 +40,7 @@
$app->helper( 'mirrorcache.render_file' => sub {
my ($c, $filepath, $dm, $file)= @_;
$c->log->error($c->dumper('RENDER START', $filepath)) if $MCDEBUG;
+ $mc_config = $app->mc->config;
my $root = $c->mc->root;
my $f = Mojo::File->new($filepath);
my $dirname = $f->dirname;
@@ -111,7 +113,13 @@
my $baseurl; # just hostname + eventual urldir (without folder and
file)
my $fullurl; # baseurl with path and filename
if ($dm->metalink || $dm->meta4 || $dm->torrent || $dm->zsync ||
$dm->magnet) {
- $baseurl = $root->is_remote ? $root->location($dm) :
$root->redirect($dm, $dirname) # we must pass $path here because it potenially
has impact
+ if (!$root->is_remote) {
+ $baseurl = $root->redirect($dm, $dirname); # we must pass
$path here because it potenially has impact
+ } elsif ($file->{size} && $mc_config->redirect_huge &&
$mc_config->huge_file_size <= $file->{size}) {
+ $baseurl = $dm->scheme . '://' . $mc_config->redirect_huge .
$filepath;
+ } else {
+ $baseurl = $root->location($dm);
+ }
}
if ($dm->torrent || $dm->zsync || $dm->magnet) {
if ($baseurl) {
@@ -245,7 +253,11 @@
$fileorigin = $ENV{MIRRORCACHE_METALINK_PUBLISHER_URL};
$fileorigin = $dm->scheme . "://" . $fileorigin unless
$fileorigin =~ m/^http/;
} elsif ($root->is_remote) {
- $fileorigin = $root->location($dm);
+ if ($file->{size} && $mc_config->redirect_huge &&
$mc_config->huge_file_size <= $file->{size}) {
+ $fileorigin = $dm->scheme . '://' .
$mc_config->redirect_huge;
+ } else {
+ $fileorigin = $root->location($dm);
+ }
} else {
my $redirect = $root->redirect($dm, $filepath);
if ($redirect) {
@@ -590,7 +602,7 @@
return if $root_included and !$print;
$writer->comment("File origin location: ") if $print;
- if ($METALINK_GREEDY && $METALINK_GREEDY <= (100 -
$preference)) {
+ if ($METALINK_GREEDY && $METALINK_GREEDY <= (100 -
$preference) || $root_included) {
$writer->comment($rooturl . $fullname);
} else {
$writer->startTag('url', type =>
substr($rooturl,0,$colon), location => uc($dm->root_country), preference =>
$preference);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
--- old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
2023-01-12 09:40:42.000000000 +0100
@@ -31,6 +31,8 @@
my $root_subtree = $ENV{MIRRORCACHE_SUBTREE} // "";
has 'urlredirect';
+has 'urlredirect_huge';
+has 'huge_file_size';
sub register {
(my $self, $app) = @_;
@@ -42,6 +44,9 @@
}
$self->urlredirect($app->mcconfig->redirect);
+ $self->urlredirect_huge($app->mcconfig->redirect_huge);
+ $self->huge_file_size($app->mcconfig->huge_file_size);
+
$app->helper( 'mc.root' => sub { $self; });
}
@@ -106,10 +111,15 @@
$filepath = "" unless $filepath;
for my $root (@roots) {
next unless ( -e $root->[dir] . $root_subtree . $filepath || (
$root_subtree && ( -e $root->[dir] . $filepath ) ) );
+ if ($self->urlredirect_huge) {
+ my $size = -s $root->[dir] . $filepath;
+ return $dm->scheme . "://" . $self->urlredirect_huge if (($size //
0) >= $self->huge_file_size);
+ }
+
return $dm->scheme . "://" . $root->[host_vpn] if ($dm->vpn &&
$root->[host_vpn]);
return $dm->scheme . "://" . $root->[host] if ($root->[host]);
return $dm->scheme . "://" . $self->urlredirect if
($self->urlredirect);
- return undef;
+ last;
}
return undef;
}
@@ -149,7 +159,7 @@
}
}
-sub _detect_ln {
+sub _detect_ln_in_the_same_folder {
my ($dir, $file) = @_;
return undef unless $file && $file =~ m/.*(Media|Current)\.iso(\.sha256)?/;
@@ -167,10 +177,10 @@
return undef;
}
-sub detect_ln {
+sub detect_ln_in_the_same_folder {
my ($self, $path) = @_;
my $f = Mojo::File->new($path);
- my $res = _detect_ln($f->dirname, $f->basename);
+ my $res = _detect_ln_in_the_same_folder($f->dirname, $f->basename);
return undef unless $res;
return $f->dirname . '/' . $res;
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
--- old/MirrorCache-1.051/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/lib/MirrorCache/WebAPI/Plugin/RootRemote.pm
2023-01-12 09:40:42.000000000 +0100
@@ -18,6 +18,7 @@
use Mojo::Base 'Mojolicious::Plugin';
use Mojo::Util ('trim');
use Encode ();
+use Mojo::JSON qw(decode_json);
use URI::Escape ('uri_unescape');
use File::Basename;
use HTML::Parser;
@@ -60,17 +61,39 @@
}
sub realpath {
- return undef unless $nfs;
-
- my ($self, $path) = @_;
+ my ($self, $path, $deep) = @_;
return undef unless $path;
- my $localpath = $nfs . $path;
- my $realpathlocal = Cwd::realpath($localpath);
-
- if ($realpathlocal && (0 == rindex($realpathlocal, $nfs, 0))) {
- my $realpath = substr($realpathlocal, length($nfs));
- return $realpath if $realpath ne $path;
+ if ($nfs) {
+ my $localpath = $nfs . $path;
+ my $realpathlocal = Cwd::realpath($localpath);
+
+ if ($realpathlocal && (0 == rindex($realpathlocal, $nfs, 0))) {
+ my $realpath = substr($realpathlocal, length($nfs));
+ return $realpath if $realpath ne $path;
+ }
+ } elsif ($deep) {
+ my $path1 = $path . '/';
+ my $rootlocation = $self->rooturl;
+ my $url = $rootlocation . $path1;
+ my $ua = Mojo::UserAgent->new->max_redirects(0)->request_timeout(1);
+ my $tx = $ua->head($url, {'User-Agent' =>
'MirrorCache/detect_redirect'});
+ my $res = $tx->res;
+
+ # redirect on oneself
+ if ($res->is_redirect && $res->headers) {
+ my $location1 = $res->headers->location;
+ if ($location1 && $path1 ne substr($location1, -length($path1))) {
+ my $i = rindex($location1, $rootlocation, 0);
+ if ($i ne -1) {
+ # remove trailing slash we added earlier
+ my $location = substr($location1, 0, -1);
+ if ($rootlocation eq substr($location, 0,
length($rootlocation))) {
+ return substr($location, length($rootlocation));
+ }
+ }
+ }
+ }
}
return undef;
}
@@ -148,10 +171,28 @@
return 1;
};
-sub _detect_ln {
- return undef unless $nfs;
- my ($dir, $file) = @_;
- return undef unless $file && $file =~ m/.*(Media|Current)\.iso(\.sha256)?/;
+sub _detect_ln_in_the_same_folder {
+ my ($self, $dir, $file) = @_;
+
+ unless ($nfs) {
+ my $dir1 = $dir . '/';
+ my $rootlocation = $self->rooturl;
+ my $url = $rootlocation . $dir1 . $file;
+ my $ua = Mojo::UserAgent->new->max_redirects(0)->request_timeout(2);
+ my $tx = $ua->head($url, {'User-Agent' =>
'MirrorCache/detect_redirect'});
+ my $res = $tx->res;
+
+ # redirect on oneself
+ if ($res->is_redirect && $res->headers) {
+ my $location = $res->headers->location;
+ my $url1 = $rootlocation . $dir1;
+ if ($location && $url1 eq substr($location, 0, length($url1))) {
+ my $ln = substr($location, length($url1));
+ return $ln if -1 == index($ln, '/');
+ }
+ }
+ return undef;
+ }
my $dest;
eval {
@@ -168,11 +209,11 @@
return $res;
}
-sub detect_ln {
- return undef unless $nfs;
+# this is simillar to self->realpath, just detects symlinks in current folder
+sub detect_ln_in_the_same_folder {
my ($self, $path) = @_;
my $f = Mojo::File->new($path);
- my $res = _detect_ln($f->dirname, $f->basename);
+ my $res = $self->_detect_ln_in_the_same_folder($f->dirname, $f->basename);
return undef unless $res;
return $f->dirname . '/' . $res;
}
@@ -191,10 +232,23 @@
return 1;
}
my $ua = Mojo::UserAgent->new;
- my $tx = $ua->get($self->rooturl . $dir . '/?F=1');
+ my $tx = $ua->get($self->rooturl . $dir . '/?F=1&json');
return 0 unless $tx->result->code == 200;
- # we cannot use mojo dom here because it takes too much RAM for huge html
- # my $dom = $tx->result->dom;
+
+ return $self->_foreach_filename_json($dir, $sub, $P, $ua, $tx) if -1 <
index($tx->result->headers->content_type, 'json');
+
+ return $self->_foreach_filename_html($dir, $sub, $P, $ua, $tx);
+}
+
+# we cannot use mojo dom here because it takes too much RAM for huge html
+# my $dom = $tx->result->dom;
+sub _foreach_filename_html {
+ my $self = shift;
+ my $dir = shift;
+ my $sub = shift;
+ my $P = shift;
+ my $ua = shift;
+ my $tx = shift;
my $href = '';
my $href20 = '';
@@ -222,7 +276,7 @@
if ($t && ($href20 eq substr($t,0,20))) {
if ($desc{name} && (!$P || $desc{name} =~ $P)) {
- my $target = _detect_ln($dir, $desc{name});
+ my $target = $self->_detect_ln_in_the_same_folder($dir,
$desc{name});
$sub->($desc{name}, $desc{size}, undef, $desc{mtime}, $target);
%desc = ();
}
@@ -263,7 +317,7 @@
$p->parse($chunk);
}
if ($desc{name} && (!$P || $desc{name} =~ $P)) {
- my $target = detect_ln($dir, $desc{name});
+ my $target = $self->_detect_ln_in_the_same_folder($dir, $desc{name});
$sub->($desc{name}, $desc{size}, undef, $desc{mtime}, $target);
%desc = ();
}
@@ -271,4 +325,32 @@
return 1;
}
+sub _foreach_filename_json {
+ my $self = shift;
+ my $dir = shift;
+ my $sub = shift;
+ my $P = shift;
+ my $ua = shift;
+ my $tx = shift;
+
+ my $cnt = 0;
+
+ my $value = decode_json $tx->result->body;
+ die "JSON top level is not array" unless ref($value) eq 'ARRAY';
+
+ for my $hashref (@{$value}) {
+ my $n = $hashref->{name} // $hashref->{n};
+ $n = $n . '/' if $hashref->{type} // '' eq 'directory' && '/' ne
substr $n, -1;
+ next unless $n;
+ $sub->(
+ $n,
+ $hashref->{size} // $hashref->{s},
+ undef,
+ $hashref->{mtime} // $hashref->{m},
+ ) if $sub;
+ $cnt++;
+ }
+ return $cnt;
+}
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/t/environ/02-files.sh
new/MirrorCache-1.052/t/environ/02-files.sh
--- old/MirrorCache-1.051/t/environ/02-files.sh 2022-12-29 10:16:50.000000000
+0100
+++ new/MirrorCache-1.052/t/environ/02-files.sh 2023-01-12 09:40:42.000000000
+0100
@@ -93,7 +93,8 @@
# first request will miss
$mc/curl -I /download/folder1/file3.1.dat | grep 200
-$mc/backstage/job folder_sync_schedule_from_misses
+# pass too big value for prev_stat_id and make sure it is automatically
adjusted
+$mc/backstage/job -e folder_sync_schedule_from_misses -a '["1000000"]'
$mc/backstage/job folder_sync_schedule
$mc/backstage/shoot
$mc/backstage/job mirror_scan_schedule
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/t/environ/03-headquarter-subsidiaries-remote.sh
new/MirrorCache-1.052/t/environ/03-headquarter-subsidiaries-remote.sh
--- old/MirrorCache-1.051/t/environ/03-headquarter-subsidiaries-remote.sh
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/t/environ/03-headquarter-subsidiaries-remote.sh
2023-01-12 09:40:42.000000000 +0100
@@ -1,22 +1,25 @@
#!lib/test-in-container-environ.sh
set -ex
-# root : ap9
+# root : ng9
# mc environ by number:
# 9 - headquarter
# 6 - NA subsidiary
# 7 - EU subsidiary
-root=$(environ ap9)
+root=$(environ ng9)
mkdir -p $root/dt/{folder1,folder2,folder3}
echo $root/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
mkdir $root/dt/folder1/repodata/
touch $root/dt/folder1/repodata/file1.dat
SMALL_FILE_SIZE=3
+HUGE_FILE_SIZE=9
+FAKEURL="notexists${RANDOM}.com"
echo -n 1234 > $root/dt/folder1/repodata/filebig1.1.dat
echo -n 123 > $root/dt/folder1/repodata/filesmall1.1.dat
+echo -n 123456789 > $root/dt/folder1/repodata/filehuge1.1.dat
echo repomdcontent > $root/dt/folder1/repodata/repomd.xml
touch $root/dt/folder1/repodata/repomd.xml.asc
@@ -26,6 +29,8 @@
$mc9/gen_env "MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 \
MIRRORCACHE_ROOT=http://$($root/print_address) \
+ MIRRORCACHE_REDIRECT_HUGE=$FAKEURL \
+ MIRRORCACHE_HUGE_FILE_SIZE=$HUGE_FILE_SIZE \
MIRRORCACHE_SMALL_FILE_SIZE=$SMALL_FILE_SIZE \
MIRRORCACHE_ROOT_NFS=$root/dt
@@ -41,6 +46,7 @@
$mc9/backstage/shoot
hq_address=$(mc9/print_address)
+hq_interface=127.0.0.10
eu_interface=127.0.0.3
# repodata/repomd.xml is served from root even when asked from EU
@@ -56,9 +62,19 @@
curl -sI --interface 127.0.0.24 http://$hq_address/geoip | grep '204 No
Content'
$mc9/db/start
-$mc9/curl --interface 127.0.0.5 /folder1/repodata/file1.dat.metalink | grep
'origin="http://127.0.0.1:3190/folder1/repodata/file1.dat.metalink";'
+$mc9/curl --interface $hq_interface /folder1/repodata/file1.dat.metalink |
grep 'origin="http://127.0.0.1:3190/folder1/repodata/file1.dat.metalink";'
-# filebig is redirected to EU
+echo filebig is redirected to EU
$mc9/curl -I --interface $eu_interface /folder1/repodata/filebig1.1.dat | grep
'302 Found'
-# filesmall is served right away
+echo filesmall is served right away
$mc9/curl -I --interface $eu_interface /folder1/repodata/filesmall1.1.dat |
grep '200 OK'
+
+echo check huge files are redirected to FAKEURL, but we need to scan folder
first
+$mc9/curl -I --interface $hq_interface
/download/folder1/repodata/filehuge1.1.dat | grep "Location:
http://$FAKEURL/folder1/repodata/filehuge1.1.dat";
+$mc9/curl -I --interface $hq_interface
/download/folder1/repodata/filebig1.1.dat | grep "Location: " | grep
$($root/print_address)
+
+$mc9/curl --interface $hq_interface
/download/folder1/repodata/filehuge1.1.dat.meta4 | grep
"http://$FAKEURL/folder1/repodata/filehuge1.1.dat";
+$mc9/curl --interface $hq_interface
/download/folder1/repodata/filehuge1.1.dat.mirrorlist | grep
"http://$FAKEURL/folder1/repodata/filehuge1.1.dat";
+
+echo success
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/t/environ/03-headquarter-subsidiaries-weight.sh
new/MirrorCache-1.052/t/environ/03-headquarter-subsidiaries-weight.sh
--- old/MirrorCache-1.051/t/environ/03-headquarter-subsidiaries-weight.sh
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/t/environ/03-headquarter-subsidiaries-weight.sh
2023-01-12 09:40:42.000000000 +0100
@@ -52,7 +52,7 @@
# do requests and check that both na instances are being used and instance 2
is used more frequently
out=$(
-counter=30
+counter=180
while test $counter -gt 0
do
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/t/environ/03-headquarter-subsidiaries.sh
new/MirrorCache-1.052/t/environ/03-headquarter-subsidiaries.sh
--- old/MirrorCache-1.051/t/environ/03-headquarter-subsidiaries.sh
2022-12-29 10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/t/environ/03-headquarter-subsidiaries.sh
2023-01-12 09:40:42.000000000 +0100
@@ -8,17 +8,21 @@
# 8 - ASIA subsidiary
SMALL_FILE_SIZE=3
+HUGE_FILE_SIZE=9
+FAKEURL="notexists${RANDOM}.com"
for i in 6 7 8 9; do
x=$(environ mc$i $(pwd))
mkdir -p $x/dt/{folder1,folder2,folder3}
echo -n 1234 > $x/dt/folder1/filebig1.1.dat
echo -n 123 > $x/dt/folder1/filesmall1.1.dat
+ echo -n 123456789 > $x/dt/folder1/filehuge1.1.dat
echo '[]' > $x/dt/folder1/file.json
eval mc$i=$x
done
hq_address=$($mc9/print_address)
+hq_interface=127.0.0.10
na_address=$($mc6/print_address)
na_interface=127.0.0.2
eu_address=$($mc7/print_address)
@@ -28,6 +32,8 @@
# deploy db
$mc9/gen_env MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3' \
+ MIRRORCACHE_HUGE_FILE_SIZE=$HUGE_FILE_SIZE \
+ MIRRORCACHE_REDIRECT_HUGE=$FAKEURL \
MIRRORCACHE_SMALL_FILE_SIZE=$SMALL_FILE_SIZE
$mc9/backstage/shoot
@@ -75,6 +81,9 @@
curl --interface $na_interface -Is
http://$hq_address/download/folder1/filebig1.1.dat | grep "Location:
http://$na_address/download/folder1/filebig1.1.dat";
curl --interface $na_interface -Is
http://$hq_address/download/folder1/filesmall1.1.dat | grep "200 OK"
+echo check huge files are redirected to FAKEURL
+curl --interface $hq_interface -Is
http://$hq_address/download/folder1/filehuge1.1.dat | grep "Location:
http://$FAKEURL/folder1/filehuge1.1.dat";
+
ct=$($mc9/curl -I /download/folder1/file.json | grep Content-Type)
[[ "$ct" =~ application/json ]]
ct=$($mc9/curl -I /folder1/file.json | grep Content-Type)
@@ -96,4 +105,11 @@
echo unless /download is asked explicitly
$mc9/curl -H 'User-Agent: Chromium/xyz' /download/folder1/ | grep file.json
+echo check metalink/mirrorlist for huge files reference FAKEURL, but need to
scan them first
+$mc9/backstage/job -e folder_sync_schedule_from_misses
+$mc9/backstage/job -e folder_sync_schedule
+$mc9/backstage/shoot
+curl --interface $hq_interface -s
http://$hq_address/download/folder1/filehuge1.1.dat.metalink | grep
"http://$FAKEURL/folder1/filehuge1.1.dat";
+curl --interface $hq_interface -s
http://$hq_address/download/folder1/filehuge1.1.dat.mirrorlist | grep
"http://$FAKEURL/folder1/filehuge1.1.dat";
+
echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.051/t/environ/04-remote-current-no-nfs.sh
new/MirrorCache-1.052/t/environ/04-remote-current-no-nfs.sh
--- old/MirrorCache-1.051/t/environ/04-remote-current-no-nfs.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.052/t/environ/04-remote-current-no-nfs.sh 2023-01-12
09:40:42.000000000 +0100
@@ -0,0 +1,69 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+ap9=$(environ ap9)
+
+$mc/gen_env \
+ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 \
+ MIRRORCACHE_ROOT=http://$($ap9/print_address) \
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+
+for x in $ap7 $ap8 $ap9; do
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}-Media.iso | xargs
-n 1 touch
+ sha256sum $x/dt/folder1/file1.1-Media.iso >
$x/dt/folder1/file1.1-Media.iso.sha256
+ echo 111112 > $x/dt/folder1/file2.1-Media.iso
+ sha256sum $x/dt/folder1/file2.1-Media.iso >
$x/dt/folder1/file2.1-Media.iso.sha256
+ ( cd $x/dt/folder1 && ln -s file1.1-Media.iso file-Media.iso && ln -s
file1.1-Media.iso.sha256 file-Media.iso.sha256 )
+done
+
+echo ' RewriteEngine On
+ RewriteBase "/"
+ RewriteRule ^folder1/file-Media.iso(.*)?$ folder1/file1.1-Media.iso$1 [R]
+' > $ap9/directory-rewrite.conf
+
+echo 'LoadModule rewrite_module /usr/lib64/apache2-prefork/mod_rewrite.so'
> $ap9/extra-rewrite.conf
+
+for x in $ap7 $ap8 $ap9; do
+ $x/start
+done
+
+$mc/start
+$mc/status
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','us','na'"
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+
+
+$mc/sql "select * from file"
+
+################################################
+# Test unversioned Media.iso is redirected to file which is metioned inside
corresponding Media.iso.sha256
+$mc/curl -I /download/folder1/file-Media.iso | grep -C 10 302 | grep
/download/folder1/file1.1-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep
/download/folder1/file1.1-Media.iso.sha256
+$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 "
+
+echo now change the symlink and make sure redirect changes
+echo ' RewriteEngine On
+ RewriteBase "/"
+ RewriteRule ^folder1/file-Media.iso(.*)?$ folder1/file2.1-Media.iso$1 [R]
+' > $ap9/directory-rewrite.conf
+
+$ap9/stop
+$ap9/start
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+$mc/curl -I /download/folder1/file-Media.iso | grep -C 10 302 | grep
/download/folder1/file2.1-Media.iso
+$mc/curl -I /download/folder1/file-Media.iso.sha256 | grep -C 10 302 | grep
/download/folder1/file2.1-Media.iso.sha256
+$mc/curl -L /download/folder1/file-Media.iso.sha256 | grep -q
"2019dd7afaf5759c68cec4d0e7553227657f01c69da168489116a1c48e40270e "
+
+echo success
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/t/environ/04-remote-link.sh
new/MirrorCache-1.052/t/environ/04-remote-link.sh
--- old/MirrorCache-1.051/t/environ/04-remote-link.sh 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/t/environ/04-remote-link.sh 2023-01-12
09:40:42.000000000 +0100
@@ -1,7 +1,7 @@
#!lib/test-in-container-environ.sh
set -ex
-mc=$(environ mc $(pwd))
+mc=$(environ mc9 $(pwd))
ap9=$(environ ap9)
@@ -17,7 +17,6 @@
echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
done
-ln -s $ap9/dt/folder1 $ap9/dt/link
echo ' RewriteEngine On
RewriteBase "/"
RewriteRule ^link(/.*)?$ folder1$1 [R]
@@ -54,6 +53,7 @@
$mc/backstage/job folder_sync_schedule
$mc/backstage/shoot
$mc/sql_test 2 == "select count(*) from minion_jobs where task =
'folder_sync_schedule_from_misses' and state = 'finished'"
+$mc/sql_test 1 == "select count(*) from redirect"
$mc/curl -I /download/link
$mc/curl -I /download/link | grep -A4 -E '301|302' | grep ': /download/folder1'
$mc/backstage/job folder_sync_schedule_from_misses
@@ -62,3 +62,22 @@
$mc/sql_test 3 == "select count(*) from minion_jobs where task =
'folder_sync_schedule_from_misses' and state = 'finished'"
$mc/curl -I /download/link
$mc/curl -I /download/link | grep -A4 -E '301|302' | grep ': /download/folder1'
+
+
+echo now change redirect from /link on remote to /folder2
+
+echo ' RewriteEngine On
+ RewriteBase "/"
+ RewriteRule ^link(/.*)?$ folder2$1 [R]
+' > $ap9/directory-rewrite.conf
+
+$ap9/stop
+$ap9/start
+
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+
+$mc/curl -I /download/link
+$mc/curl -I /download/link | grep -A4 -E '301|302' | grep ': /download/folder2'
+
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/t/environ/04-remote-mc.sh
new/MirrorCache-1.052/t/environ/04-remote-mc.sh
--- old/MirrorCache-1.051/t/environ/04-remote-mc.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.052/t/environ/04-remote-mc.sh 2023-01-12
09:40:42.000000000 +0100
@@ -0,0 +1,223 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+# we name it the same way as in test 04-remote-nginx.sh to simplify diff
+ng9=$(environ mc2 $(pwd))
+
+MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
+
+$mc/gen_env MIRRORCACHE_PEDANTIC=1 \
+ MIRRORCACHE_ROOT=http://$($ng9/print_address)/download \
+ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
+
+ng8=$(environ ng8)
+ng7=$(environ ng7)
+
+for x in $ng7 $ng8 $ng9; do
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+ echo -n 0123456789 > $x/dt/folder1/file2.1.dat
+ $x/start
+done
+
+$mc/start
+$mc/status
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ng7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ng8/print_address)','','t','us','na'"
+
+# remove folder1/file1.1.dt from ng8
+rm $ng8/dt/folder1/file2.1.dat
+
+# first request redirected to root
+$mc/curl -I /download/folder1/file2.1.dat | grep $($ng9/print_address)
+
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+$mc/db/sql "select * from file"
+test 0 == $($mc/db/sql "select size from file where name='file1.1.dat'")
+test 10 == $($mc/db/sql "select size from file where name='file2.1.dat'")
+
+test 2 == $($mc/db/sql "select count(*) from folder_diff")
+test 1 == $($mc/db/sql "select count(*) from folder_diff_file")
+
+$mc/curl -I /download/folder1/file2.1.dat | grep $($ng7/print_address)
+
+mv $ng7/dt/folder1/file2.1.dat $ng8/dt/folder1/
+
+# gets redirected to root again
+$mc/curl -I /download/folder1/file2.1.dat | grep $($ng9/print_address)
+
+$mc/backstage/job mirror_scan_schedule_from_path_errors
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+$mc/curl -H "Accept: */*, application/metalink+xml"
/download/folder1/file2.1.dat | grep $($ng9/print_address)
+
+# now redirects to ng8
+$mc/curl -I /download/folder1/file2.1.dat | grep $($ng8/print_address)
+
+# now add new file everywhere
+for x in $ng9 $ng7 $ng8; do
+ touch $x/dt/folder1/file3.1.dat
+done
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+# now expect to hit
+$mc/curl -I /download/folder1/file3.1.dat | grep -E
"$($ng8/print_address)|$($ng7/print_address)"
+
+# now add new file only on main server and make sure it doesn't try to redirect
+touch $ng9/dt/folder1/file4.dat
+
+$mc/curl -I /download/folder1/file4.dat | grep -E "$($ng9/print_address)"
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+test 2 == $($mc/db/sql "select count(*) from folder_diff_server")
+
+cnt="$($mc/db/sql "select count(*) from audit_event")"
+
+$mc/curl -I /download/folder1/file4.dat
+
+# it shouldn't try to probe yet, because scanner didn't find files on the
mirrors
+test 0 == $($mc/db/sql "select count(*) from audit_event where name =
'mirror_probe' and id > $cnt")
+
+for x in $ng9 $ng7 $ng8; do
+ mkdir $x/dt/folder1/folder11
+ touch $x/dt/folder1/folder11/file1.1.dat
+done
+
+# this is needed for schedule jobs to retry on next shoot
+$mc/curl -I /download/folder1/folder11/file1.1.dat
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+$mc/curl -I /download/folder1/folder11/file1.1.dat | grep -E
"$($ng7/print_address)|$($ng8/print_address)"
+$mc/curl /download/folder1/folder11/ | grep file1.1.dat
+
+
+$mc/curl /download/folder1?status=all | grep -E '"recent":"?2"?'| grep -E
'"not_scanned":"?0"?' | grep -E '"outdated":"?0"?'
+$mc/curl /download/folder1?status=recent | grep $($ng7/print_address) | grep
$($ng8/print_address)
+test {} == $($mc/curl /download/folder1?status=outdated)
+test {} == $($mc/curl /download/folder1?status=not_scanned)
+
+##################################
+# let's test path distortions
+# remember number of folders in DB
+cnt=$($mc/db/sql "select count(*) from folder")
+$mc/curl -I /download//folder1//file1.1.dat
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+test $cnt == $($mc/db/sql "select count(*) from folder")
+
+$mc/curl -I /download//folder1//file1.1.dat | grep -C 10 -P
'[^/]/folder1/file1.1.dat' | grep 302
+$mc/curl -I /download//folder1///file1.1.dat | grep -C 10 -P
'[^/]/folder1/file1.1.dat' | grep 302
+$mc/curl -I /download/./folder1/././file1.1.dat | grep -C 10 -P
'[^/]/folder1/file1.1.dat' | grep 302
+$mc/curl -I /download/./folder1/../folder1/./file1.1.dat | grep -C 10 -P
'[^/]/folder1/file1.1.dat' | grep 302
+##################################
+
+
+##################################
+# test file with colon ':'
+for x in $ng9 $ng7 $ng8; do
+ touch $x/dt/folder1/file:4.dat
+done
+
+# first request will miss
+$mc/curl -I /download/folder1/file:4.dat | grep -E "$($ng9/print_address)"
+
+$mc/db/sql "select s.id, s.hostname, fd.id, fd.hash, fl.name, fd.dt, fl.dt
+from
+folder_diff fd
+join folder_diff_server fds on fd.id = fds.folder_diff_id
+join server s on s.id = fds.server_id
+left join folder_diff_file fdf on fdf.folder_diff_id = fd.id
+left join file fl on fdf.file_id = fl.id
+left join folder f on fd.folder_id = f.id
+order by f.id, s.id, fl.name
+"
+
+# force rescan
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+$mc/db/sql "select s.id, s.hostname, fd.id, fd.hash, fl.name, fd.dt, fl.dt
+from
+folder_diff fd
+join folder_diff_server fds on fd.id = fds.folder_diff_id
+join server s on s.id = fds.server_id
+left join folder_diff_file fdf on fdf.folder_diff_id = fd.id
+left join file fl on fdf.file_id = fl.id
+left join folder f on fd.folder_id = f.id
+order by f.id, s.id, fl.name
+"
+
+# now expect to hit
+$mc/curl /download/folder1/ | grep file1.1.dat
+$mc/curl /download/folder1/ | grep file:4.dat
+$mc/curl -I /download/folder1/file:4.dat | grep -E
"$($ng8/print_address)|$(ng7*/print_address)"
+##################################
+
+f=0123456789012345678901234567890123456789.\(\#@~\)abcdefghijklmnoprst.dat
+e=0123456789012345678901234567890123456789.%28%23%40~%29abcdefghijklmnoprst.dat
+
+for x in $ng9 $ng7 $ng8; do
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ echo -n 0123456789 > $x/dt/folder1/$f
+done
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+$mc/curl /download/folder1/ | grep -B2 $f | grep '10 Byte'
+
+##########################
+# add a symlink and make sure size is correct for it as well
+( cd $ng9/dt/folder1 && ln -s $f ln-Media.iso )
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/shoot
+
+$mc/curl /download/folder1/ | grep -B2 ln-Media.iso | grep '10 Byte'
+
+$mc/curl -IL /download/folder1/$e | grep '200 OK'
+$mc/curl -I /download/folder1/$e | grep -C20 '302 Found' | grep -E
"$($ng7/print_address)|$($ng8/print_address)" | grep "/folder1/$e"
+
+
+for x in $mc $ng9; do
+ for pattern in 'P=*2.1*' 'GLOB=*2.1*' 'REGEX=.*2\.1.*'; do
+ for extra in '' '&json' '&jsontable'; do
+ echo $x $pattern $extra
+ $x/curl /download/folder1/?$pattern$extra | grep -o file2.1.dat
+ rc=0
+ $x/curl /download/folder1/?$pattern$extra | grep -o file1.1.dat ||
rc=$?
+ test $rc -gt 0
+ done
+ done
+done
+
+$mc/curl "/download/folder1/?GLOB=*2.1*&mirrorlist" | grep '<li>Filename:
file2.1.dat</li>'
+$mc/curl "/download/folder1/?REGEX=.*2\.1.*&mirrorlist" | grep '<li>Filename:
file2.1.dat</li>'
+
+
+echo success
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/t/manual/01-download.o.o.sh
new/MirrorCache-1.052/t/manual/01-download.o.o.sh
--- old/MirrorCache-1.051/t/manual/01-download.o.o.sh 2022-12-29
10:16:50.000000000 +0100
+++ new/MirrorCache-1.052/t/manual/01-download.o.o.sh 2023-01-12
09:40:42.000000000 +0100
@@ -6,19 +6,20 @@
$mc/gen_env MIRRORCACHE_RECKLESS=0 \
MIRRORCACHE_ROOT=http://download.opensuse.org \
MIRRORCACHE_REDIRECT=downloadcontent.opensuse.org \
- MIRRORCACHE_HYPNOTOAD=1 \
+ MIRRORCACHE_HYPNOTOAD=0 \
MIRRORCACHE_PERMANENT_JOBS="'folder_sync_schedule_from_misses
folder_sync_schedule mirror_scan_schedule_from_misses
mirror_scan_schedule_from_path_errors mirror_scan_schedule cleanup
stat_agg_schedule mirror_check_from_stat'" \
- MIRRORCACHE_TOP_FOLDERS="'debug distribution tumbleweed factory
repositories'" \
+ MIRRORCACHE_TOP_FOLDERS="'debug distribution tumbleweed factory
repositories update'" \
MIRRORCACHE_AUTH_URL=https://www.opensuse.org/openid/user/ \
MIRRORCACHE_TRUST_AUTH=127.0.0.2 \
MIRRORCACHE_PROXY_URL=http://127.0.0.1:3110 \
+ MIRRORCACHE_BRANDING=openSUSE \
MIRRORCACHE_BACKSTAGE_WORKERS=32
-
$mc/start
$mc/backstage/start
$mc/db/sql -f
dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/mirrors-eu.sql
mc_test
-$mc/backstage/job -e folder_tree -a '["/distribution"]'
-$mc/backstage/job -e folder_tree -a '["/tumbleweed"]'
+# $mc/backstage/job -e folder_tree -a '["/distribution"]'
+# $mc/backstage/job -e folder_tree -a '["/tumbleweed"]'
+# $mc/backstage/job -e folder_tree -a '["/update"]'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.051/templates/dir.html.ep
new/MirrorCache-1.052/templates/dir.html.ep
--- old/MirrorCache-1.051/templates/dir.html.ep 2022-12-29 10:16:50.000000000
+0100
+++ new/MirrorCache-1.052/templates/dir.html.ep 2023-01-12 09:40:42.000000000
+0100
@@ -5,7 +5,7 @@
% my $full_path = $cur_path;
% $full_path = $route . $cur_path unless $route eq '/';
% my @breadcrumbs = split '/', $full_path;
-% my $bc_last = pop @breadcrumbs;
+% my $bc_last = pop @breadcrumbs // '';
%= include 'layouts/info'
% my $mc_branding = eval '$branding' // '';
@@ -21,11 +21,11 @@
<style type='text/css'>
-a {
+tr.td.a {
display: flex;
}
-a::after{
+tr.td.a::after{
content: var(--desc);
color: grey;
margin-left: auto;
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.NONKJq/_old 2023-01-19 16:44:29.465858613 +0100
+++ /var/tmp/diff_new_pack.NONKJq/_new 2023-01-19 16:44:29.469858633 +0100
@@ -1,5 +1,5 @@
name: MirrorCache
-version: 1.051
-mtime: 1672305410
-commit: a668bdb9d15d6a15752b6d4b5fb87e3f651096fc
+version: 1.052
+mtime: 1673512842
+commit: 4b3bfe9c1345b5228f40fb6ad75626a67ab7aee5
