Package: release.debian.org Severity: normal User: release.debian....@packages.debian.org Usertags: unblock X-Debbugs-Cc: i...@packages.debian.org Control: affects -1 + src:inn2
Please unblock package inn2 This is tagged as a snapshot but is actually 2.7.1 RC1. It contains many documentation fixes, small improvements and fixes to pullnews, and the new ovsqlite-util program which can be used to debug and repair an ovsqlite database. The new package has been used in production for 3 weeks on one of my servers. I am attaching the git diff between debian/2.7.1_20230306-1 and debian/2.7.1_20230322-1, abridged of documentation changes. The full changelog can be consulted at https://salsa.debian.org/md/inn2/-/commits/master . unblock inn2/2.7.1~20230322-1 -- ciao, Marco
diff --git a/.gitignore b/.gitignore index 274716315..9960002af 100644 --- a/.gitignore +++ b/.gitignore @@ -176,6 +176,7 @@ /storage/ovmethods.h /storage/buffindexed/buffindexed_d /storage/ovsqlite/ovsqlite-server +/storage/ovsqlite/ovsqlite-util /storage/ovsqlite/sql-init.c /storage/ovsqlite/sql-init.h /storage/ovsqlite/sql-main.c diff --git a/MANIFEST b/MANIFEST index 35e05aef2..7254d27aa 100644 --- a/MANIFEST +++ b/MANIFEST @@ -210,6 +210,7 @@ doc/man/ovdb_server.8 Manpage for ovdb_server doc/man/ovdb_stat.8 Manpage for ovdb_stat doc/man/overchan.8 Manpage for overchan backend doc/man/ovsqlite-server.8 Manpage for ovsqlite-server +doc/man/ovsqlite-util.8 Manpage for ovsqlite-util doc/man/ovsqlite.5 Manpage for the ovsqlite overview module doc/man/passwd.nntp.5 Manpage for passwd.nntp config file doc/man/perl-nocem.8 Manpage for perl-nocem @@ -331,6 +332,7 @@ doc/pod/ovdb_server.pod Master file for ovdb_server.8 doc/pod/ovdb_stat.pod Master file for ovdb_stat.8 doc/pod/overchan.pod Master file for overchan.8 doc/pod/ovsqlite-server.pod Master file for ovsqlite-server.8 +doc/pod/ovsqlite-util.pod Master file for ovsqlite-util.8 doc/pod/ovsqlite.pod Master file for ovsqlite.5 doc/pod/passwd.nntp.pod Master file for passwd.nntp.5 doc/pod/procbatch.pod Master file for procbatch.8 @@ -774,6 +776,7 @@ storage/ovsqlite/ovmethod.mk Make rules for ovsqlite storage/ovsqlite/ovsqlite-private.c Private code for ovsqlite storage/ovsqlite/ovsqlite-private.h Private header for ovsqlite storage/ovsqlite/ovsqlite-server.c SQLite database exclusive owner +storage/ovsqlite/ovsqlite-util.in Utility program for ovsqlite storage/ovsqlite/ovsqlite.c ovsqlite implementation storage/ovsqlite/ovsqlite.h ovsqlite interface storage/ovsqlite/sql-init.c Generated database setup implementation diff --git a/Makefile.global.in b/Makefile.global.in index 8a185ed39..db42dee2e 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -20,7 +20,7 @@ ## be complying with the NNTP protocol. VERSION = 2.7.1 -VERSION_EXTRA = prerelease +VERSION_EXTRA = rc1 version ## The absolute path to the top of the build directory, used to find the ## libraries built as part of INN. Using relative paths confuses libtool diff --git a/debian/changelog b/debian/changelog index ffbb0e6a6..eff319e64 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +inn2 (2.7.1~20230322-1) unstable; urgency=medium + + * New release candidate 1 of the stable branch. + + -- Marco d'Itri <m...@linux.it> Mon, 27 Mar 2023 04:30:21 +0200 + inn2 (2.7.1~20230306-1) unstable; urgency=medium * New upstream snapshot of the stable branch. diff --git a/doc/man/Makefile b/doc/man/Makefile index 906725ebd..30a87587f 100644 --- a/doc/man/Makefile +++ b/doc/man/Makefile @@ -30,9 +30,9 @@ SEC8 = actsync.8 archive.8 batcher.8 buffchan.8 ckpasswd.8 \ innupgrade.8 innwatch.8 innxbatch.8 innxmit.8 mailpost.8 makedbz.8 \ makehistory.8 mod-active.8 news.daily.8 news2mail.8 ninpaths.8 \ nnrpd.8 nntpsend.8 ovdb_init.8 ovdb_monitor.8 ovdb_server.8 \ - ovdb_stat.8 overchan.8 ovsqlite-server.8 perl-nocem.8 procbatch.8 \ - prunehistory.8 radius.8 \ - rc.news.8 scanlogs.8 scanspool.8 send-ihave.8 send-uucp.8 sendinpaths.8 \ + ovdb_stat.8 overchan.8 ovsqlite-server.8 ovsqlite-util.8 perl-nocem.8 \ + procbatch.8 prunehistory.8 radius.8 rc.news.8 \ + scanlogs.8 scanspool.8 send-ihave.8 send-uucp.8 sendinpaths.8 \ tally.control.8 tdx-util.8 tinyleaf.8 writelog.8 all: diff --git a/doc/pod/Makefile b/doc/pod/Makefile index 792ccf568..2fe219533 100644 --- a/doc/pod/Makefile +++ b/doc/pod/Makefile @@ -48,6 +48,7 @@ MAN8 = ../man/actsync.8 ../man/archive.8 ../man/auth_krb5.8 \ ../man/nnrpd.8 ../man/nntpsend.8 \ ../man/ovdb_init.8 ../man/ovdb_monitor.8 ../man/ovdb_server.8 \ ../man/ovdb_stat.8 ../man/overchan.8 ../man/ovsqlite-server.8 \ + ../man/ovsqlite-util.8 \ ../man/procbatch.8 ../man/prunehistory.8 ../man/radius.8 \ ../man/rc.news.8 ../man/scanlogs.8 ../man/scanspool.8 \ ../man/send-ihave.8 ../man/sendinpaths.8 \ @@ -172,6 +173,7 @@ maintclean: distclean ../man/ovdb_stat.8: ovdb_stat.pod ; $(POD2MAN) -s 8 $? > $@ ../man/overchan.8: overchan.pod ; $(POD2MAN) -s 8 $? > $@ ../man/ovsqlite-server.8: ovsqlite-server.pod ; $(POD2MAN) -s 8 $? > $@ +../man/ovsqlite-util.8: ovsqlite-util.pod ; $(POD2MAN) -s 8 $? > $@ ../man/procbatch.8: procbatch.pod ; $(POD2MAN) -s 8 $? > $@ ../man/prunehistory.8: prunehistory.pod ; $(POD2MAN) -s 8 $? > $@ ../man/radius.8: radius.pod ; $(POD2MAN) -s 8 $? > $@ diff --git a/frontends/pullnews.in b/frontends/pullnews.in index 99ebfbf84..b21ce29b4 100644 --- a/frontends/pullnews.in +++ b/frontends/pullnews.in @@ -11,9 +11,10 @@ # Full changelog can be found in the Git commit history of the # INN project. Major changes are: # -# February 2023: +# February-March 2023: # Julien Élie added TLS support for both downstream and upstream -# servers. Also made pullnews robust on socket timeout. +# servers. Also made pullnews robust on socket timeout, and +# added -L (largest article size wanted). # # January 2010: # Geraint A. Edwards added header-only feeding (-B); @@ -131,9 +132,10 @@ END { my $usage = "Usage: $0 [-BhnOqRx] [-a hashfeed] [-b fraction] [-c config] [-C width] [-d level] [-f fraction] [-F fakehop] [-g groups] [-G newsgroups] - [-H headers] [-k checkpt] [-l logfile] [-m header_pats] [-M num] [-N num] - [-p port] [-P hop_limit] [-Q level] [-r file] [-s host[:port][_tlsmode]] - [-S num] [-t retries] [-T seconds] [-w num] [-z num] [-Z num] + [-H headers] [-k checkpt] [-l logfile] [-L size] [-m header_pats] + [-M num] [-N num] [-p port] [-P hop_limit] [-Q level] [-r file] + [-s host[:port][_tlsmode]] [-S num] [-t retries] [-T seconds] + [-w num] [-z num] [-Z num] [upstream_host ...] Options: @@ -187,6 +189,8 @@ Options: -l logfile log progress/stats to logfile (default is stdout). + -L size largest wanted article size in bytes for articles to download. + -m 'Hdr1:regexp1 !Hdr2:regexp2 #Hdr3:regexp3 !#Hdr4:regexp4 ...' feed article only if: the Hdr1 header field body matches regexp1; @@ -258,10 +262,10 @@ sub HELP_MESSAGE { } use vars qw($opt_a $opt_b $opt_B $opt_c $opt_C $opt_d $opt_f $opt_F - $opt_g $opt_G $opt_h $opt_H $opt_k $opt_l $opt_m $opt_M $opt_n + $opt_g $opt_G $opt_h $opt_H $opt_k $opt_l $opt_L $opt_m $opt_M $opt_n $opt_N $opt_O $opt_p $opt_P $opt_q $opt_Q $opt_r $opt_R $opt_s $opt_S $opt_t $opt_T $opt_w $opt_x $opt_z $opt_Z); -getopts("a:b:Bc:C:d:f:F:g:G:hH:k:l:m:M:nN:Op:P:qQ:r:Rs:S:t:T:w:xz:Z:") +getopts("a:b:Bc:C:d:f:F:g:G:hH:k:l:L:m:M:nN:Op:P:qQ:r:Rs:S:t:T:w:xz:Z:") || die $usage; HELP_MESSAGE() if defined $opt_h; @@ -522,7 +526,7 @@ if (not $rnews) { if ($localTLS == 1 && !$localcxn->starttls()) { die " Can't use STARTTLS on $localServer: " . $localcxn->code() . " " - . $localcxn->message() . "\n"; + . join('//', split(/\r?\n/, $localcxn->message())) . "\n"; } if (exists $passwd{$localServer} @@ -530,7 +534,8 @@ if (not $rnews) { { warn sprintf( " failed to authorize: %s %s\n", - $localcxn->code(), $localcxn->message() + $localcxn->code(), + join('//', split(/\r?\n/, $localcxn->message())) ); } } @@ -627,7 +632,8 @@ foreach my $server (@servers) { if ($upstreamTLS == 1 && !$upstream->starttls()) { warn sprintf( "can't use STARTTLS: %s %s\n", - $upstream->code(), $upstream->message() + $upstream->code(), + join('//', split(/\r?\n/, $upstream->message())) ); next; } @@ -635,7 +641,8 @@ foreach my $server (@servers) { if ($username && !$upstream->authinfo($username, $passwd)) { warn sprintf( "failed to authorize: %s %s\n", - $upstream->code(), $upstream->message() + $upstream->code(), + join('//', split(/\r?\n/, $upstream->message())) ); next; } @@ -826,7 +833,8 @@ sub crossFeedGroup { if (!defined($narticles)) { # Group command failed. warn sprintf( "Group command failed for $group: %s %s\n", - $fromServer->code() || 'NO_CODE', $fromServer->message() + $fromServer->code() || 'NO_CODE', + join('//', split(/\r?\n/, $fromServer->message())) ); return 0; } @@ -874,16 +882,46 @@ sub crossFeedGroup { my $i; my @warns; my $skip_article; - for ($i = ($first > $high ? $first : $high + 1); $i <= $last; $i++) { + my $overview; + my $begin = ($first > $high ? $first : $high + 1); + for ($i = $begin; $i <= $last; $i++) { $skip_article = 0; last if defined $maxArts and $count >= $maxArts; last if defined $opt_f and $count >= $toget; $count++; $art_total_count++; sleep $opt_z if defined $opt_z and $count > 1; + + # Do not download articles whose size exceeds the largest wanted size. + # Field 3 contains the Message-ID, field 5 the article size. + if (defined($opt_L)) { + # Retrieve overview data by chunks, so that articles keep being + # downloaded instead of a possible long wait at the start of the + # process of each newsgroup. + if (($count % $progressWidth) == 1) { + # Do not directly use $i + $progressWidth, as the result may + # exceed the maximum article number supported by the server. + my @range + = ($i, + $i + $progressWidth - 1 > $last + ? $last + : $i + $progressWidth - 1); + $overview = $fromServer->xover(\@range); + } + + if (defined($$overview{$i}[5]) and $$overview{$i}[5] > $opt_L) { + print LOG "." unless $quiet; + print LOG "\tDEBUGGING $i\t-- not downloading " + . "article $$overview{$i}[3] " + . "which has $$overview{$i}[5] bytes\n" + if $debug >= 1; + $skip_article = 1; + } + } + # "Optimized mode" -- check if the article is wanted # *before* downloading it. - if (defined $opt_O) { + if (not $skip_article and defined($opt_O)) { # 223 n <a> article retrieved # -- request text separately (after STAT) # 423 no such article number in this group @@ -896,11 +934,12 @@ sub crossFeedGroup { my $new_msgid = $toServer->nntpstat($org_msgid); my $new_code = $toServer->code(); print LOG - "\tDEBUGGING $i\t$org_msgid ($org_code) => $new_code\n" + "\n\tDEBUGGING $i\t$org_msgid ($org_code) => $new_code\n" if $debug >= 3; # Skip the article if it already exists # on the downstream server. if ($new_code == 223) { + print LOG "." unless $quiet; print LOG "\tDEBUGGING $i\t-- not downloading " . "already existing message $org_msgid code=$new_code\n" if $debug >= 1; @@ -928,6 +967,7 @@ sub crossFeedGroup { push @{$article}, "\n" if not $is_control_art; } } + if (not $skip_article and (not $header_only or $is_control_art or $add_bytes_header)) { @@ -1071,13 +1111,13 @@ sub crossFeedGroup { my $cut = join("\n\t", splice(@{$article}, $idx, 1)); $tx_len -= length($cut); $idx_blank_pre_body--; - print LOG "\tDEBUGGING $i\tcut1 $cut" if $debug >= 2; + print LOG "\tDEBUGGING $i\tcut1 $cut\n" if $debug >= 2; while ($article->[$idx] =~ /^[[:space:]](.+)/) { # Folded lines. my $cut = join("\n\t", splice(@{$article}, $idx, 1)); $tx_len -= length($cut); $idx_blank_pre_body--; - print LOG "\tDEBUGGING $i\tcut_ $cut" if $debug >= 2; + print LOG "\tDEBUGGING $i\tcut_ $cut\n" if $debug >= 2; } } @@ -1139,6 +1179,7 @@ sub crossFeedGroup { $pulled->{$server}->{$group}++; if ($skip_due_to_hdrs) { + print LOG "m" unless $quiet; if ($debug >= 2) { print LOG "\tDEBUGGING $i\tskip_art: " . ( @@ -1152,7 +1193,6 @@ sub crossFeedGroup { ) ) . "\n"; } - print LOG "m" unless $quiet; } elsif ($rnews) { printf RNEWS "#! rnews %d\n", $tx_len; map { print RNEWS $_ } @{$article}; @@ -1171,9 +1211,6 @@ sub crossFeedGroup { # 441 posting failed my $code = $toServer->code(); my $msg = $toServer->message(); - print LOG "\tDEBUGGING $i\tPost $code: Msg: <" - . join('//', split(/\r?\n/, $msg)) . ">\n" - if $debug >= 1; $msg =~ s/^340 .*?\n(?=.)//o; if ($msg =~ /^240 /) { print LOG "+" unless $quiet; @@ -1198,6 +1235,10 @@ sub crossFeedGroup { saveConfig(); exit(1); } + print LOG "\tDEBUGGING $i\tPost $code: Msg: <" + . join('//', split(/\r?\n/, $toServer->message())) + . ">\n" + if $debug >= 1; } elsif (not $reader and not $toServer->ihave($msgid, $article)) @@ -1210,9 +1251,6 @@ sub crossFeedGroup { # 437 article rejected -- do not try again my $code = $toServer->code(); my $msg = $toServer->message(); - print LOG "\tDEBUGGING $i\tPost $code: Msg: <" - . join('//', split(/\r?\n/, $msg)) . ">\n" - if $debug >= 1; if ($code == 435) { print LOG "." unless $quiet; $refused{$group}++; @@ -1229,14 +1267,17 @@ sub crossFeedGroup { saveConfig(); exit(1); } + print LOG "\tDEBUGGING $i\tPost $code: Msg: <" + . join('//', split(/\r?\n/, $msg)) . ">\n" + if $debug >= 1; } else { my $code = $toServer->code(); my $msg = $toServer->message(); - print LOG "\tDEBUGGING $i\tPost $code: Msg: <" - . join('//', split(/\r?\n/, $msg)) . ">\n" - if $debug >= 1; print LOG "+" unless $quiet; + print LOG "\tDEBUGGING $i\tPost $code: Msg: <" + . join('//', split(/\r?\n/, $msg)) . ">\n" + if $debug >= 1; $fed{$group}++; $info{server}->{$server}->{fed}++; $info{fed}++; @@ -1244,8 +1285,9 @@ sub crossFeedGroup { } $shash->{$group} = [time, $high = $i]; } elsif ($skip_article) { - # Optimized mode (-O) decided to skip this article... - print LOG "." unless $quiet; + # Optimized mode (-O) or article size check (-L) decided to skip + # this article... + # The "." resulting treatment has already been output. $refused{$group}++; $info{server}->{$server}->{refused}++; $info{refused}++; @@ -1262,20 +1304,23 @@ sub crossFeedGroup { # Net::NNTP is a subclass) when the connection is no longer # active. warn "\nArticle retrieval failed (" - . $fromServer->message() . ")\n\n"; + . join('//', split(/\r?\n/, $fromServer->message())) + . ")\n\n"; return 2; } else { warn "\nUnexpected response from server (" . $fromServer->code() . " " - . $fromServer->message() . ")\n"; + . join('//', split(/\r?\n/, $fromServer->message())) + . ")\n"; saveConfig(); exit(1); } } print LOG "x" unless $quiet; printf LOG ( - "\nDEBUGGING $i %s %s\n", $fromServer->code(), - $fromServer->message() + "\tDEBUGGING $i\t-- article unavailable %s %s\n", + $fromServer->code(), + join('//', split(/\r?\n/, $fromServer->message())) ) if $debug >= 1; } saveConfig() if $checkPoint and ($art_total_count % $checkPoint) == 0; diff --git a/scripts/innreport_inn.pm b/scripts/innreport_inn.pm index a63e17fe8..33d79a373 100644 --- a/scripts/innreport_inn.pm +++ b/scripts/innreport_inn.pm @@ -1670,6 +1670,8 @@ sub collect($$$$$$) { =~ /^python: dynamic authorization access type is not known: /o; # during daily expiration return 1 if $left =~ /^\S+ rejected Expiring process \d+$/o; + # during ovsqlite-util + return 1 if $left =~ /^\S+ rejected ovsqlite-util fixes$/o; # during scanlogs return 1 if $left =~ /^\S+ rejected Flushing log and syslog files$/o; return 1 if $left =~ /^\S+ rejected Snapshot log and syslog files$/o; diff --git a/storage/ovsqlite/ovmethod.config b/storage/ovsqlite/ovmethod.config index c945ba877..57b932ebb 100644 --- a/storage/ovsqlite/ovmethod.config +++ b/storage/ovsqlite/ovmethod.config @@ -2,6 +2,6 @@ name = ovsqlite number = 5 sources = ovsqlite.c ovsqlite-private.c extra-sources = ovsqlite-server.c sql-main.c sql-init.c sqlite-helper.c -programs = ovsqlite-server +programs = ovsqlite-server ovsqlite-util clean = sqlite-helper-gen maintclean = sql-init.c sql-init.h sql-main.c sql-main.h diff --git a/storage/ovsqlite/ovmethod.mk b/storage/ovsqlite/ovmethod.mk index 3e56b3f83..009d3d259 100644 --- a/storage/ovsqlite/ovmethod.mk +++ b/storage/ovsqlite/ovmethod.mk @@ -8,6 +8,9 @@ ovsqlite/ovsqlite-server: $(OVSQLITEOBJECTS) libinnstorage.$(EXTLIB) $(LIBSTORAGE) $(LIBHIST) $(LIBINN) $(STORAGE_LIBS) $(SQLITE3_LIBS) \ $(LIBS) +ovsqlite/ovsqlite-util: ovsqlite/ovsqlite-util.in $(FIXSCRIPT) + $(FIX) ovsqlite/ovsqlite-util.in + ovsqlite/sqlite-helper-gen: ovsqlite/sqlite-helper-gen.in $(FIXSCRIPT) $(FIX) -i ovsqlite/sqlite-helper-gen.in diff --git a/storage/ovsqlite/ovsqlite-util.in b/storage/ovsqlite/ovsqlite-util.in new file mode 100644 index 000000000..626949b82 --- /dev/null +++ b/storage/ovsqlite/ovsqlite-util.in @@ -0,0 +1,501 @@ +#! /usr/bin/perl -w +# fixscript will replace this line with code to load INN::Config + +## Overview manipulation utility for ovsqlite. +## +## Initial version written in March 2023 by Julien ÉLIE. + +use Compress::Zlib; +use Getopt::Std; +use POSIX qw(strftime locale_h); +use strict; + +$0 =~ s!.*/!!; + +# Bail out if the needed DBI Perl module is not installed. +eval { + require DBI; + require DBD::SQLite; + my $err = $DBI::errstr; # Just to silence "used only once" warning. + 1; +} + or die "DBI Perl module with SQLite driver needed" + . " (usually packaged as libdbd-sqlite3-perl, perl-DBD-SQLite," + . " or p5-DBD-SQLite)"; + +# Name of the database file (not configurable for ovsqlite). +my $dbfile = "ovsqlite.db"; + +my $usage = "Usage: + $0 [-AFghioO] [-a article] [-n newsgroup] [-p path] + +Options: + -a article Specify an article number or a range of article numbers on + which to act. + -A Audit the overview database for problems, and report them to + standard error, without trying to fix them. + -F Audit the overview database for problems, fixing them where + possible. To see what would be changed, run $0 + with -A first. + -g Dump overall overview information for the newsgroup specified + with -n. + -h Print this help message. + -i Dump newsgroup-related overview information for all newsgroups + or the newsgroup specified with -n. + -n newsgroup Specify the newsgroup on which to act. + -o Dump overview information for articles in the newsgroup + specified with -n, in the format returned to clients. + -O Dump overview information for articles in the newsgroup + specified with -n, in the format used by overchan. + -p path Read $dbfile database file in path directory instead of + default $INN::Config::pathoverview directory. +"; + +sub HELP_MESSAGE { + print $usage; + exit(0); +} + +my %opt; +getopts("a:AFghioOn:p:", \%opt) || die $usage; + +HELP_MESSAGE() if defined($opt{'h'}); + +my $modes = 0; +$modes++ if defined($opt{'A'}); +$modes++ if defined($opt{'F'}); +$modes++ if defined($opt{'g'}); +$modes++ if defined($opt{'i'}); +$modes++ if defined($opt{'o'}); +$modes++ if defined($opt{'O'}); + +die "Can't use both -A and -F\n\n$usage" + if defined($opt{'A'}) + and defined($opt{'F'}); +die "No action specified\n\n$usage" + if $modes == 0; +die "Only one action allowed at the same time\n\n$usage" + if $modes > 1; +die "A newsgroup must be specified with -n\n\n$usage" + if !defined($opt{'n'}) + and (defined($opt{'g'}) || defined($opt{'o'}) || defined($opt{'O'})); + +my ($low, $high, $compress, $basedict); +my $sql_extraclause_artinfo = ""; +my $sql_extraclause_groupinfo = ""; +my $dbdir = $opt{'p'} || $INN::Config::pathoverview; +my $datasource = "dbi:SQLite:dbname=$dbdir/$dbfile"; +my $pausemsg = "ovsqlite-util fixes"; # Message when pausing INN. + # Known line in innreport. +my $ispaused = 0; + +# To determine the length of compressed overview data. +my @pack_length_bias = ( + 0, + 0x80, + 0x4080, + 0x204080, + 0x10204080, +); + +# Open the connection. The username and password fields are left empty. +# Enabling RaiseError permits not checking every return error codes. +my $dbh = DBI->connect( + $datasource, '', '', + { PrintError => 0, RaiseError => 1, AutoCommit => 0 } +) or die "Can't connect to database: $DBI::errstr"; + +# To process multiple SQL statements in a do() handle. +$dbh->{sqlite_allow_multiple_statements} = 1; + +# Check the specified newsgroup exists, and create appropriate SQL requests. +if (defined($opt{'n'})) { + my $groupid = get_groupid($opt{'n'}); + if ($groupid == 0) { + printf STDERR "Cannot find newsgroup $opt{'n'} in overview\n"; + exit(1); + } else { + $sql_extraclause_artinfo = "where groupid = $groupid"; + $sql_extraclause_groupinfo + = "where cast(groupname as text) = '$opt{'n'}'"; + } +} + +# Parse the specified range of articles. +if (defined($opt{'a'})) { + if ($opt{'a'} =~ /^(\d*)-(\d*)$/) { + $low = $1; + $high = $2; + } elsif ($opt{'a'} =~ /^\d+$/) { + $low = $opt{'a'}; + $high = $low; + } else { + printf STDERR "Cannot parse $opt{'a'} as article numbers\n"; + exit(1); + } + if (defined($low) and length($low) > 0) { + $sql_extraclause_artinfo .= " and artnum >= $low"; + } + if (defined($high) and length($high) > 0) { + $sql_extraclause_artinfo .= " and artnum <= $high"; + } +} + +# Grab information from the misc table. +load_settings(); + +# Pause the server if changes need being done, so that the overview is not +# updated by another process at the same time. +if (defined($opt{'F'})) { + if (system "$INN::Config::newsbin/ctlinnd -s pause '$pausemsg'") { + die "$0: failed to pause INN, aborting\n"; + } + $ispaused = 1; +} + +if (defined($opt{'A'}) or defined($opt{'F'})) { + # Run the checks, and fix them if -F given. + check_groupinfo_consistency(); +} elsif (defined($opt{'g'})) { + dump_overview(); +} elsif (defined($opt{'i'})) { + dump_groupinfo(); +} elsif (defined($opt{'o'})) { + dump_artinfo_clients(); +} elsif (defined($opt{'O'})) { + dump_artinfo_overchan(); +} + +# Close the connection properly. +$dbh->disconnect; + +exit(0); + +END { + # In case we bail out while being paused, make sure that the show goes on! + if ($ispaused) { + if (system "$INN::Config::newsbin/ctlinnd -s go '$pausemsg'") { + die "$0: failed to resume INN with " + . "\"ctlinnd -s go '$pausemsg'\" command " + . "=> please check why and *manually* resume it\n"; + } + } +} + +# Grab compression settings from the database. +sub load_settings { + my $getsetting; + + $getsetting = $dbh->prepare("select value from misc where key = ?1"); + ($compress) = $dbh->selectrow_array($getsetting, undef, "compress"); + if ($compress > 0) { + ($basedict) = $dbh->selectrow_array($getsetting, undef, "basedict"); + defined($basedict) + or die "No basedict value found to decompress overview data\n"; + } +} + +# Return the ID of the newsgroup given as argument, or 0 if not found. +sub get_groupid { + my $groupname = shift; + my ($getgroupid, $groupid); + + $getgroupid = $dbh->prepare( + q{ +select groupid from groupinfo + where cast(groupname as text) = ?1 + and deleted = 0; +} + ); + ($groupid) = $dbh->selectrow_array($getgroupid, undef, $groupname); + + return defined($groupid) ? $groupid : 0; +} + +# Turn enforcement of foreign key constraints on or off, depending on the +# argument given to the function (either 1 for on, or 0 for off). +# The AutoCommit attribute needs being true so as not to start a transaction. +sub pragma_foreign_keys { + my $onoff = shift; + local $dbh->{AutoCommit} = 1; + $dbh->do("pragma foreign_keys = $onoff;"); +} + +# Return an array containing the length of the encoded length of decompressed +# overview data, and the length of actual decompressed overview data, or undef +# if corrupted. +# This function can be called even on uncompressed data. +# The expected argument is the overview data. +sub overview_length { + my $data = shift; + my ($lenlen, $len); + + if ($compress > 0) { + my $first; + + $first = vec($data, 0, 8); + $len = $first; + $lenlen = 1; + while (($first & 0x80) > 0) { + $len = $len << 8 | vec($data, $lenlen, 8); + $lenlen++; + $first <<= 1; + } + if ($lenlen > 5) { + return (undef, undef); + } + $len &= (1 << $lenlen * 7) - 1; + $len += $pack_length_bias[$lenlen - 1]; + } else { + # Uncompressed overview data. + $lenlen = 0; + $len = length($data); + } + return ($lenlen, $len); +} + +# Return decompressed overview data, or undef if a failure occurs. +# This function can be called even on uncompressed data. +# The expected arguments are the newsgroup name, the article number, and the +# associated overview data. +sub decompress_overview { + my ($groupname, $artnum, $data) = @_; + my $result; + + if ($compress > 0) { + my ($lenlen, $len) = overview_length($data); + if (!defined($lenlen)) { + warn "$groupname:$artnum: Corrupt overview data\n"; + return undef; + } + if ($len == 0) { + # No compression. + $result = substr($data, $lenlen); + } else { + my ($inflation, $status); + + ($inflation, $status) + = inflateInit(-Dictionary => "$basedict$groupname:$artnum\r\n"); + if ($status != Z_OK) { + warn + "$groupname:$artnum: inflateInit failed with code $status\n"; + return undef; + } + ($result, $status) = $inflation->inflate(substr($data, $lenlen)); + if ($status != Z_STREAM_END) { + warn "$groupname:$artnum: inflate failed with code $status\n"; + return undef; + } + if (length($result) != $len) { + warn "$groupname:$artnum: Corrupt overview data\n"; + return undef; + } + } + } else { + # Uncompressed overview data. + $result = $data; + } + return $result; +} + +# Perform consistency checks on low water marks, high water marks, and +# article counts in groupinfo. SQL commands were provided by Bo Lindbergh. +sub check_groupinfo_consistency { + my ($statement, $result); + + pragma_foreign_keys(0); + $dbh->do( + q{ +create table temp.repairs ( + groupid integer + primary key, + new_low integer + not null, + low_was_bad integer + not null, + new_high integer + not null, + high_was_bad integer + not null, + new_count integer + not null, + count_was_bad integer + not null, + expired integer + not null, + groupname blob + not null, + flag_alias blob + not null +); + +with new_stats (groupid, new_low, new_high, new_count) as + (select groupid, + coalesce(min(artnum), low), + coalesce(max(artnum), high), + count(artnum) + from groupinfo + natural left join artinfo + where not deleted + group by groupid) +insert into repairs + (groupid, + new_low, low_was_bad, + new_high, high_was_bad, + new_count, count_was_bad, + expired, groupname, flag_alias) + select groupid, + new_low, new_low != low as low_was_bad, + new_high, new_high != high as high_was_bad, + new_count, new_count != "count" as count_was_bad, + expired, groupname, flag_alias + from new_stats + natural join groupinfo + where low_was_bad + or high_was_bad + or count_was_bad; +} + ); + pragma_foreign_keys(1); + + $statement = $dbh->prepare("select count(*) from repairs;"); + ($result) = $dbh->selectrow_array($statement); + + if ($result > 0) { + printf STDERR ( + "%d groupinfo record%s incoherent\n", $result, + ($result > 1) ? "s" : "" + ); + + # Show incoherent records (L, H and C are for Low, High, Count). + $statement = $dbh->prepare( + q{ +select case when low_was_bad then 'L' else '_' end + || case when high_was_bad then 'H' else '_' end + || case when count_was_bad then 'C' else '_' end as bad, groupname + from repairs; +} + ); + $statement->execute(); + + while (my @row = $statement->fetchrow_array()) { + print STDERR " $row[0] for $row[1]\n"; + } + + if (defined($opt{'F'})) { + # Fix groupinfo table. + $result = $dbh->do( + q{ +insert or replace into groupinfo + (groupid, low, high, "count", expired, groupname, flag_alias) + select groupid, new_low, new_high, new_count, + expired, groupname, flag_alias + from repairs; +} + ); + $dbh->commit(); + + printf STDERR ( + "%d groupinfo record%s fixed\n", $result, + ($result > 1) ? "s" : "" + ); + } + + } +} + +# Dump overview information (-g option). +sub dump_overview { + my $statement; + + $statement = $dbh->prepare( + qq{ +select artnum, overview, arrived, expires, quote(token) + from artinfo $sql_extraclause_artinfo; +} + ); + $statement->execute(); + + while (my @row = $statement->fetchrow_array()) { + # quote(token) returns a string in the form "X'token'" without + # surrounding '@' characters. + my $len = (overview_length($row[1]))[1]; + if (!defined($len)) { + warn "$opt{'n'}:$row[0]: Corrupt overview data\n"; + $len = 0; + } + print "$row[0] $len $row[2] $row[3]"; + print " @" . substr($row[4], 2, -1) . "@\n"; + } +} + +# Dump newsgroup-related overview information (-i option). +sub dump_groupinfo { + my $statement; + + $statement = $dbh->prepare( + qq{ +select groupname, high, low, count, flag_alias, expired, deleted + from groupinfo $sql_extraclause_groupinfo; +} + ); + $statement->execute(); + + while (my @row = $statement->fetchrow_array()) { + print join(" ", @row) . "\n"; + } +} + +# Dump overview information in the format returned to clients (-o option). +sub dump_artinfo_clients { + my $statement; + + $statement = $dbh->prepare( + qq{ +select overview, artnum, quote(token), arrived, expires + from artinfo $sql_extraclause_artinfo; +} + ); + $statement->execute(); + + # To generate valid Date header fields. + setlocale(LC_TIME, 'C'); + + while (my @row = $statement->fetchrow_array()) { + my $overdata = decompress_overview($opt{'n'}, $row[1], $row[0]); + # Remove trailing CRLF from overview data. + $overdata =~ s/\r\n//g; + print "$overdata"; + print "\tArticle: $row[1]"; + print "\tToken: @" . substr($row[2], 2, -1) . "@"; + print "\tArrived: " + . strftime('%a, %d %b %Y %H:%M:%S %z (%Z)', localtime($row[3])); + print "\tExpires: " + . strftime('%a, %d %b %Y %H:%M:%S %z (%Z)', localtime($row[4])) + if $row[4] > 0; + print "\n"; + } +} + +# Dump overview information in the format used by overchan (-O option). +sub dump_artinfo_overchan { + my $statement; + + $statement = $dbh->prepare( + qq{ +select quote(token), arrived, expires, overview, artnum + from artinfo $sql_extraclause_artinfo; +} + ); + $statement->execute(); + + while (my @row = $statement->fetchrow_array()) { + print "@" . substr($row[0], 2, -1) . "@"; + my $overdata = decompress_overview($opt{'n'}, $row[4], $row[3]); + # Remove the first field (article number, not expected by overchan) + # and trailing CRLF from overview data. + $overdata =~ s/^\d+\t//; + $overdata =~ s/\r\n//; + print " $row[1] $row[2] $overdata\n"; + } +} diff --git a/support/mkmanifest b/support/mkmanifest index b4694caf0..484090be5 100755 --- a/support/mkmanifest +++ b/support/mkmanifest @@ -266,6 +266,7 @@ site/update storage/buffindexed/buffindexed_d storage/buildconfig storage/ovsqlite/ovsqlite-server +storage/ovsqlite/ovsqlite-util storage/ovsqlite/sqlite-helper-gen storage/tradindexed/tdx-util support/fixconfig
signature.asc
Description: PGP signature