Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package openQA for openSUSE:Factory checked in at 2025-10-11 22:50:42 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openQA (Old) and /work/SRC/openSUSE:Factory/.openQA.new.5300 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openQA" Sat Oct 11 22:50:42 2025 rev:766 rq:1310683 version:5.1760108577.fd2f2a48 Changes: -------- --- /work/SRC/openSUSE:Factory/openQA/openQA.changes 2025-10-08 18:20:29.461438428 +0200 +++ /work/SRC/openSUSE:Factory/.openQA.new.5300/openQA.changes 2025-10-11 22:51:39.252834829 +0200 @@ -1,0 +2,11 @@ +Fri Oct 10 15:12:32 UTC 2025 - [email protected] + +- Update to version 5.1760108577.fd2f2a48: + * Log unavailability due to high load only as warning + * Filter job stats of scheduled products also by arch and build + * Document how to disable image optimizations + * Make image optimization errors stop the job producing an incomplete job + * Improve wording in description about job stats API + * Run `optipng` for real and handle errors if it fails + +------------------------------------------------------------------- Old: ---- openQA-5.1759912962.689b31ed.obscpio New: ---- openQA-5.1760108577.fd2f2a48.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openQA-client-test.spec ++++++ --- /var/tmp/diff_new_pack.qc0aaE/_old 2025-10-11 22:51:40.176873890 +0200 +++ /var/tmp/diff_new_pack.qc0aaE/_new 2025-10-11 22:51:40.180874058 +0200 @@ -18,7 +18,7 @@ %define short_name openQA-client Name: %{short_name}-test -Version: 5.1759912962.689b31ed +Version: 5.1760108577.fd2f2a48 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA-devel-test.spec ++++++ --- /var/tmp/diff_new_pack.qc0aaE/_old 2025-10-11 22:51:40.212875411 +0200 +++ /var/tmp/diff_new_pack.qc0aaE/_new 2025-10-11 22:51:40.216875581 +0200 @@ -18,7 +18,7 @@ %define short_name openQA-devel Name: %{short_name}-test -Version: 5.1759912962.689b31ed +Version: 5.1760108577.fd2f2a48 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA-test.spec ++++++ --- /var/tmp/diff_new_pack.qc0aaE/_old 2025-10-11 22:51:40.256877271 +0200 +++ /var/tmp/diff_new_pack.qc0aaE/_new 2025-10-11 22:51:40.260877441 +0200 @@ -18,7 +18,7 @@ %define short_name openQA Name: %{short_name}-test -Version: 5.1759912962.689b31ed +Version: 5.1760108577.fd2f2a48 Release: 0 Summary: Test package for openQA License: GPL-2.0-or-later ++++++ openQA-worker-test.spec ++++++ --- /var/tmp/diff_new_pack.qc0aaE/_old 2025-10-11 22:51:40.292878794 +0200 +++ /var/tmp/diff_new_pack.qc0aaE/_new 2025-10-11 22:51:40.296878962 +0200 @@ -18,7 +18,7 @@ %define short_name openQA-worker Name: %{short_name}-test -Version: 5.1759912962.689b31ed +Version: 5.1760108577.fd2f2a48 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA.spec ++++++ --- /var/tmp/diff_new_pack.qc0aaE/_old 2025-10-11 22:51:40.328880315 +0200 +++ /var/tmp/diff_new_pack.qc0aaE/_new 2025-10-11 22:51:40.328880315 +0200 @@ -99,7 +99,7 @@ %define devel_requires %devel_no_selenium_requires chromedriver Name: openQA -Version: 5.1759912962.689b31ed +Version: 5.1760108577.fd2f2a48 Release: 0 Summary: The openQA web-frontend, scheduler and tools License: GPL-2.0-or-later ++++++ openQA-5.1759912962.689b31ed.obscpio -> openQA-5.1760108577.fd2f2a48.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/docs/Pitfalls.asciidoc new/openQA-5.1760108577.fd2f2a48/docs/Pitfalls.asciidoc --- old/openQA-5.1759912962.689b31ed/docs/Pitfalls.asciidoc 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/docs/Pitfalls.asciidoc 2025-10-10 17:02:57.000000000 +0200 @@ -58,10 +58,11 @@ appropriate timeout margin. On some less powerful systems (like Raspberry Pi), reducing screenshots size -with optipng may take significant amount of time. In this case, you can switch -to `pngquant` which is significantly faster, but uses lossy compression. To do -that, install `pngquant` package and set `USE_PNGQUANT=1` in worker or job -settings. +with `optipng` may take a significant amount of time. In this case, you can +switch to `pngquant` which is significantly faster, but uses lossy compression. +To do that, install `pngquant` package and set `USE_PNGQUANT=1` in worker or job +settings. Alternatively, you may disable optimizing images altogether via the +setting `OPTIMIZE_IMAGES=0`. [[db-migration]] == DB migration from SQlite to postgreSQL diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/lib/OpenQA/Schema/ResultSet/ScheduledProducts.pm new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/Schema/ResultSet/ScheduledProducts.pm --- old/openQA-5.1759912962.689b31ed/lib/OpenQA/Schema/ResultSet/ScheduledProducts.pm 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/Schema/ResultSet/ScheduledProducts.pm 2025-10-10 17:02:57.000000000 +0200 @@ -30,7 +30,7 @@ return {jobs_cancelled => $count}; } -sub job_statistics ($self, $distri, $version, $flavor) { +sub job_statistics ($self, $distri, $version, $flavor, $arch, $build) { my $sth = $self->result_source->schema->storage->dbh->prepare( <<~'END_SQL' WITH RECURSIVE @@ -48,7 +48,7 @@ FROM scheduled_products WHERE - status in ('new', 'scheduling', 'scheduled') and distri = ? and version = ? and flavor = ? + status in ('new', 'scheduling', 'scheduled') and distri = ? and version = ? and flavor = ? and arch = ? and build = ? GROUP BY arch ) @@ -112,6 +112,8 @@ $sth->bind_param(1, $distri); $sth->bind_param(2, $version); $sth->bind_param(3, $flavor); + $sth->bind_param(4, $arch); + $sth->bind_param(5, $build); $sth->execute; return $sth->fetchall_hashref([qw(latest_job_state latest_job_result)]); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm --- old/openQA-5.1759912962.689b31ed/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/WebAPI/Controller/API/V1/Iso.pm 2025-10-10 17:02:57.000000000 +0200 @@ -60,9 +60,9 @@ =item job_statistics() -Returns job statistics about the most recent scheduled products for each ARCH -matching the specified DISTRI, VERSION and FLAVOR. Only scheduled products that -are cancelling/cancelled are not considered. +Returns job statistics about the most recent scheduled products matching the +specified DISTRI, VERSION, FLAVOR, ARCH and BUILD parameters. Scheduled products +that are cancelling/cancelled are not considered. This allows to determine whether all jobs that have been scheduled for a certain purpose are done and whether the jobs have passed. If jobs have been @@ -90,7 +90,7 @@ sub job_statistics ($self) { my $validation = $self->validation; - my @param_keys = (qw(distri version flavor)); + my @param_keys = (qw(distri version flavor arch build)); $validation->required($_) for @param_keys; return $self->reply->validation_error({format => 'json'}) if $validation->has_error; my @params = map { $validation->param($_) } @param_keys; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/lib/OpenQA/Worker/Job.pm new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/Worker/Job.pm --- old/openQA-5.1759912962.689b31ed/lib/OpenQA/Worker/Job.pm 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/Worker/Job.pm 2025-10-10 17:02:57.000000000 +0200 @@ -5,7 +5,7 @@ use Mojo::Base 'Mojo::EventEmitter', -signatures; use OpenQA::Constants qw(DEFAULT_MAX_JOB_TIME DEFAULT_MAX_SETUP_TIME WORKER_COMMAND_ABORT WORKER_COMMAND_QUIT - WORKER_COMMAND_CANCEL WORKER_COMMAND_OBSOLETE WORKER_SR_SETUP_FAILURE WORKER_SR_API_FAILURE WORKER_SR_TIMEOUT + WORKER_COMMAND_CANCEL WORKER_COMMAND_OBSOLETE WORKER_SR_BROKEN WORKER_SR_SETUP_FAILURE WORKER_SR_API_FAILURE WORKER_SR_TIMEOUT WORKER_SR_DONE WORKER_SR_DIED); use OpenQA::Jobs::Constants; use OpenQA::Worker::Engines::isotovideo; @@ -547,6 +547,9 @@ return 'quit: worker has been stopped or restarted' if $reason eq WORKER_COMMAND_QUIT; # the result is sufficient here return undef if $reason eq WORKER_COMMAND_CANCEL; + # return upload errors due to the worker being broken itself + my $result_upload_error = $self->{_result_upload_internal_error} // $self->{_result_upload_error}; + return "worker broken: $result_upload_error" if $reason eq WORKER_SR_BROKEN && $result_upload_error; # consider other reasons as os-autoinst specific; retrieve extended reason if available if ($state) { @@ -584,7 +587,6 @@ return "$reason: terminated prematurely, see log output for details" if $reason eq WORKER_SR_DIED; # return API failure if final result upload ended with an error and there's no more relevant reason - my $result_upload_error = $self->{_result_upload_error}; return "api failure: $result_upload_error" if $result_upload_error && $reason eq WORKER_SR_DONE; # discard the reason if it is just WORKER_SR_DONE or the same as the result; otherwise return it @@ -870,12 +872,22 @@ } } +my $OPTIMIZE_ERROR = 'Unable to optimize image:'; +my $OPTIMIZE_SUGGESTION = 'You may disable optimizations via OPTIMIZE_IMAGES=0.'; + sub _upload_results_step_2_2_upload_images ($self, $callback, $error) { - chomp $error; - log_error($self->{_result_upload_error} = "Unable to upload images: $error") if $error; $self->{_images_to_send} = {}; $self->{_files_to_send} = {}; - $callback->(); + return $callback->() unless $error; + chomp $error; + $error =~ s/\s+(at.*line|OpenQA::).*//s; # avoid too lengthy log message so $OPTIMIZE_SUGGESTION is not elided + my $is_optimize_error = index($error, $OPTIMIZE_ERROR) == 0; + $error = $is_optimize_error ? "$error - $OPTIMIZE_SUGGESTION" : "Unable to upload images: $error"; + log_error($self->{_result_upload_error} = $error); + return $callback->() unless $is_optimize_error; + $self->{_result_upload_internal_error} = $error; + $self->stop(WORKER_SR_BROKEN); + return $self->_upload_results_step_3_finalize(undef, $callback); } sub _upload_results_step_2_upload_images ($self, $callback) { @@ -1222,19 +1234,19 @@ return \%ret; } -sub _optimize_image ($image, $job_settings) { +sub _optimize_image ($image, $job_settings, $optipng_bin = 'optipng', $pngquant_bin = 'pngquant') { + my $optimize_flag = $job_settings->{OPTIMIZE_IMAGES}; + return 0 if defined($optimize_flag) && !$optimize_flag; log_debug("Optimizing $image"); - { - # treat as "best-effort". If no pngquant nor optipng is found, ignore - no warnings; - if ($job_settings->{USE_PNGQUANT}) { - system('pngquant', '--force', '--output', $image, $image); - } - else { - system('optipng', '-quiet', '-o2', $image) if ($? == -1); - } - } - return undef; + my @command + = $job_settings->{USE_PNGQUANT} + ? ($pngquant_bin, '--force', '--output', $image, $image) + : ($optipng_bin, '-quiet', '-o2', $image); + system @command; + die "$OPTIMIZE_ERROR failed to execute $command[0]: $!\n" if $? == -1; + die sprintf("$OPTIMIZE_ERROR %s failed with signal %d\n", $command[0], $? & 127) if $? & 127; + die sprintf("$OPTIMIZE_ERROR %s exited with non-zero return code %d\n", $command[0], $? >> 8) if $?; + return 1; } sub _ignore_known_images ($self, $known_images = undef) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/lib/OpenQA/Worker.pm new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/Worker.pm --- old/openQA-5.1759912962.689b31ed/lib/OpenQA/Worker.pm 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/lib/OpenQA/Worker.pm 2025-10-10 17:02:57.000000000 +0200 @@ -278,8 +278,9 @@ # note: This assigns $self->current_error if there's an error and therefore prevents us from grabbing # a job while broken. The error is propagated to the web UIs. $self->configure_cache_client; - $self->set_current_error_based_on_availability; - log_error 'Unavailable: ' . $self->current_error if $self->current_error; + my $current_error = $self->set_current_error_based_on_availability; + $self->current_error_is_fatal ? log_error('Unavailable: ' . $current_error) : log_warning($current_error) + if $current_error; # register error handler to stop the current job when a critical/unhandled error occurs Mojo::IOLoop->singleton->reactor->on( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/t/24-worker-jobs.t new/openQA-5.1760108577.fd2f2a48/t/24-worker-jobs.t --- old/openQA-5.1759912962.689b31ed/t/24-worker-jobs.t 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/t/24-worker-jobs.t 2025-10-10 17:02:57.000000000 +0200 @@ -27,7 +27,7 @@ use Mojo::IOLoop; use OpenQA::Constants qw(DEFAULT_MAX_JOB_TIME DEFAULT_MAX_SETUP_TIME WORKER_COMMAND_CANCEL WORKER_COMMAND_QUIT WORKER_COMMAND_OBSOLETE WORKER_SR_SETUP_FAILURE WORKER_SR_TIMEOUT WORKER_EC_ASSET_FAILURE WORKER_EC_CACHE_FAILURE - WORKER_SR_API_FAILURE WORKER_SR_DIED WORKER_SR_DONE); + WORKER_SR_API_FAILURE WORKER_SR_DIED WORKER_SR_DONE WORKER_SR_BROKEN); use OpenQA::Worker::Job; use OpenQA::Worker::Settings; use OpenQA::Test::FakeWebSocketTransaction; @@ -234,6 +234,8 @@ $job->{_engine} = 1; # pretend isotovideo has been started like $job->_format_reason({}, TIMEOUT_EXCEEDED, WORKER_SR_TIMEOUT), qr/timeout: test execution exceeded/, 'test timeout'; + $job->{_result_upload_internal_error} = 'Unable…'; + is $job->_format_reason({}, INCOMPLETE, WORKER_SR_BROKEN), 'worker broken: Unable…', 'internal upload error'; }; subtest 'Lost WebSocket connection' => sub { @@ -1023,6 +1025,7 @@ $client->sent_messages([]); $job->{_status} = 'running'; $job->{_is_uploading_results} = 1; # stopping the job should still go as far as uploading logs and assets + $job->{_settings}->{OPTIMIZE_IMAGES} = 0; $job->stop; is_deeply( $client->sent_messages, @@ -1076,6 +1079,20 @@ $upload_stats->{uploaded_files} = []; }; +subtest 'Internal error occurrred during upload' => sub { + my $job = OpenQA::Worker::Job->new($worker, $client, {id => 7, URL => $engine_url}); + my $callback_invoked = 0; + my $cb = sub { $callback_invoked = 1 }; + combined_like { + $job->_upload_results_step_2_2_upload_images($cb, 'Unable to optimize image: foobar'); + Mojo::IOLoop->one_tick; + } + qr/Unable to optimize image: foobar/, 'error is logged'; + is $job->status, 'stopped', 'errors when optimizing are considered fatal so job is stopped'; + like $job->{_result_upload_internal_error}, qr/Unable to optimize.*disable.*OPTIMIZE_IMAGES/, 'disabling suggested'; + ok $callback_invoked, 'upload callback is invoked on fatal errors as well'; +}; + subtest 'Final upload triggered and job inncompleted when job stopped due to obsoletion' => sub { my $job = OpenQA::Worker::Job->new($worker, $client, {id => 7, URL => $engine_url}); my $res = {}; @@ -1219,12 +1236,20 @@ $upload_stats = {upload_result => 1, uploaded_files => [], uploaded_assets => []}; }; -subtest 'optipng' => sub { - is OpenQA::Worker::Job::_optimize_image('foo', {}), undef, 'optipng call is "best-effort"'; -}; - -subtest 'pngquant' => sub { - is OpenQA::Worker::Job::_optimize_image('foo', {USE_PNGQUANT => 1}), undef, 'pngquant call is "best-effort"'; +subtest 'image optimization' => sub { + my $opt = \&OpenQA::Worker::Job::_optimize_image; + local $ENV{OPENQA_LOGFILE} = undef; + combined_like { + is $opt->('foo', {OPTIMIZE_IMAGES => 0}), 0, 'image optimization can be skipped'; + throws_ok { $opt->('foo', {}) } + qr/(failed to execute optipng|optipng exited with non-zero return code)/, + 'failing to run optipng is a hard error'; + throws_ok { $opt->('foo', {USE_PNGQUANT => 1}) } + qr/(failed to execute pngquant|pngquant exited with non-zero return code)/, + 'failing to run pngquant is a hard error'; + is $opt->('bar', {}, 'true'), 1, 'successful optimization'; + } + qr/Optimizing foo.*Optimizing bar/s, 'optimizing logged'; }; subtest '_read_module_result' => sub { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1759912962.689b31ed/t/api/02-iso.t new/openQA-5.1760108577.fd2f2a48/t/api/02-iso.t --- old/openQA-5.1759912962.689b31ed/t/api/02-iso.t 2025-10-08 10:42:42.000000000 +0200 +++ new/openQA-5.1760108577.fd2f2a48/t/api/02-iso.t 2025-10-10 17:02:57.000000000 +0200 @@ -317,7 +317,7 @@ $jobs->find(99988)->update({state => DONE, result => FAILED}); $jobs->find(99993)->update({state => DONE, result => PASSED}); $jobs->find(99994)->update({state => DONE, result => PASSED}); - $t->get_ok('/api/v1/isos/job_stats?distri=opensuse&version=13.1&flavor=DVD')->status_is(200); + $t->get_ok('/api/v1/isos/job_stats?distri=opensuse&version=13.1&flavor=DVD&arch=i586&build=0091')->status_is(200); $schema->txn_rollback; my $json = $t->tx->res->json; is_deeply [sort keys %$json], [DONE, SCHEDULED], 'expected states present'; ++++++ openQA.obsinfo ++++++ --- /var/tmp/diff_new_pack.qc0aaE/_old 2025-10-11 22:52:01.801788007 +0200 +++ /var/tmp/diff_new_pack.qc0aaE/_new 2025-10-11 22:52:01.817788684 +0200 @@ -1,5 +1,5 @@ name: openQA -version: 5.1759912962.689b31ed -mtime: 1759912962 -commit: 689b31ed43631da2d90235c9d1e61532f6387acd +version: 5.1760108577.fd2f2a48 +mtime: 1760108577 +commit: fd2f2a488f6a81343c6a2a423082ca21847d5d9e
