Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package perl-Cavil-Matcher for openSUSE:Factory checked in at 2026-08-06 16:27:06 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/perl-Cavil-Matcher (Old) and /work/SRC/openSUSE:Factory/.perl-Cavil-Matcher.new.16738 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "perl-Cavil-Matcher" Thu Aug 6 16:27:06 2026 rev:2 rq:1370033 version:1.10.0 Changes: -------- --- /work/SRC/openSUSE:Factory/perl-Cavil-Matcher/perl-Cavil-Matcher.changes 2026-07-22 19:08:53.928854130 +0200 +++ /work/SRC/openSUSE:Factory/.perl-Cavil-Matcher.new.16738/perl-Cavil-Matcher.changes 2026-08-06 16:28:59.290628970 +0200 @@ -1,0 +2,21 @@ +Wed Aug 5 07:49:19 UTC 2026 - Tina Müller <[email protected]> + +- updated to 1.10.0 (1.01) + see /usr/share/doc/packages/perl-Cavil-Matcher/Changes + + 1.01 2026-08-04 + - Big indexing speedup: the scan path (load/attach) no longer re-checksums a segment's whole + payload on every open. A compiled segment is an immutable, atomically-published cache that is + already CRC-checked when written, so re-CRCing hundreds of megabytes in every forked index job + was almost the entire cost of a load (~97% in a 160 MB corpus: ~293ms -> ~8ms). Structural + validation still always runs, so bad segments are rejected and scanning stays memory-safe; + only per-scan detection of on-disk bit-rot is dropped. + - CRC is now verified where it is meaningful: once at publish (dump() fully verifies before the + atomic rename) and on demand via the new Engine verify($file) method (an fsck for a cache dir). + - Same treatment for the closest-match "bag of patterns" cache (Bag::load, used by analyze): the + load path trusts the payload CRC (the record parse still bounds-checks every read), with a new + Bag verify($file) method and a publish-time check. + - No on-disk format change: existing segments load as-is, no reindex, and old/new engines + interoperate on the same files (in-place upgrade is seamless). + +------------------------------------------------------------------- Old: ---- Cavil-Matcher-1.00.tar.gz New: ---- Cavil-Matcher-1.01.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ perl-Cavil-Matcher.spec ++++++ --- /var/tmp/diff_new_pack.LSAZqV/_old 2026-08-06 16:29:00.298664154 +0200 +++ /var/tmp/diff_new_pack.LSAZqV/_new 2026-08-06 16:29:00.298664154 +0200 @@ -18,14 +18,14 @@ %define cpan_name Cavil-Matcher Name: perl-Cavil-Matcher -Version: 1.0.0 +Version: 1.10.0 Release: 0 -# 1.00 -> normalize -> 1.0.0 -%define cpan_version 1.00 +# 1.01 -> normalize -> 1.10.0 +%define cpan_version 1.01 License: GPL-1.0-or-later Summary: Next-generation license pattern matcher for Cavil URL: https://metacpan.org/release/%{cpan_name} -Source0: Cavil-Matcher-1.00.tar.gz +Source0: https://cpan.metacpan.org/authors/id/K/KR/KRAIH/%{cpan_name}-%{cpan_version}.tar.gz Source1: cpanspec.yml Source100: README.md BuildRequires: perl ++++++ Cavil-Matcher-1.00.tar.gz -> Cavil-Matcher-1.01.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/Changes new/Cavil-Matcher-1.01/Changes --- old/Cavil-Matcher-1.00/Changes 2026-07-22 14:59:29.258187138 +0200 +++ new/Cavil-Matcher-1.01/Changes 2026-08-04 20:28:44.956802618 +0200 @@ -1,5 +1,20 @@ Revision history for Cavil::Matcher +1.01 2026-08-04 + - Big indexing speedup: the scan path (load/attach) no longer re-checksums a segment's whole + payload on every open. A compiled segment is an immutable, atomically-published cache that is + already CRC-checked when written, so re-CRCing hundreds of megabytes in every forked index job + was almost the entire cost of a load (~97% in a 160 MB corpus: ~293ms -> ~8ms). Structural + validation still always runs, so bad segments are rejected and scanning stays memory-safe; + only per-scan detection of on-disk bit-rot is dropped. + - CRC is now verified where it is meaningful: once at publish (dump() fully verifies before the + atomic rename) and on demand via the new Engine verify($file) method (an fsck for a cache dir). + - Same treatment for the closest-match "bag of patterns" cache (Bag::load, used by analyze): the + load path trusts the payload CRC (the record parse still bounds-checks every read), with a new + Bag verify($file) method and a publish-time check. + - No on-disk format change: existing segments load as-is, no reindex, and old/new engines + interoperate on the same files (in-place upgrade is seamless). + 1.00 2026-07-22 - Initial release: the next-generation license pattern matcher for Cavil. - Segmented, incrementally-updatable index: adding or removing a pattern no longer diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/META.json new/Cavil-Matcher-1.01/META.json --- old/Cavil-Matcher-1.00/META.json 2026-07-22 15:00:53.330011255 +0200 +++ new/Cavil-Matcher-1.01/META.json 2026-08-04 20:33:35.142118638 +0200 @@ -4,7 +4,7 @@ "Sebastian Riedel <[email protected]>" ], "dynamic_config" : 0, - "generated_by" : "ExtUtils::MakeMaker version 7.76, CPAN::Meta::Converter version 2.150010", + "generated_by" : "ExtUtils::MakeMaker version 7.78, CPAN::Meta::Converter version 2.150013", "license" : [ "gpl_2" ], @@ -60,6 +60,6 @@ "web" : "https://github.com/openSUSE/cavil-matcher"; } }, - "version" : "1.00", + "version" : "1.01", "x_serialization_backend" : "JSON::PP version 4.16" } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/META.yml new/Cavil-Matcher-1.01/META.yml --- old/Cavil-Matcher-1.00/META.yml 2026-07-22 15:00:53.296892369 +0200 +++ new/Cavil-Matcher-1.01/META.yml 2026-08-04 20:33:35.110368413 +0200 @@ -9,7 +9,7 @@ configure_requires: ExtUtils::MakeMaker: '7.12' dynamic_config: 0 -generated_by: 'ExtUtils::MakeMaker version 7.76, CPAN::Meta::Converter version 2.150010' +generated_by: 'ExtUtils::MakeMaker version 7.78, CPAN::Meta::Converter version 2.150013' license: gpl meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html @@ -28,5 +28,5 @@ bugtracker: https://github.com/openSUSE/cavil-matcher/issues license: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html repository: https://github.com/openSUSE/cavil-matcher.git -version: '1.00' +version: '1.01' x_serialization_backend: 'CPAN::Meta::YAML version 0.020' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/Matcher.xs new/Cavil-Matcher-1.01/Matcher.xs --- old/Cavil-Matcher-1.00/Matcher.xs 2026-07-21 21:43:04.197090308 +0200 +++ new/Cavil-Matcher-1.01/Matcher.xs 2026-08-04 20:27:15.064991701 +0200 @@ -91,6 +91,12 @@ OUTPUT: RETVAL +int verify(Cavil::Matcher::Engine self, const char *filename) + CODE: + RETVAL = matcher_verify(self, filename); + OUTPUT: + RETVAL + void set_tombstones(Cavil::Matcher::Engine self, AV *ids) CODE: matcher_set_tombstones(self, ids); @@ -152,3 +158,9 @@ RETVAL = bag_load(self, filename); OUTPUT: RETVAL + +int verify(Cavil::Matcher::Bag self, const char *filename) + CODE: + RETVAL = bag_verify(self, filename); + OUTPUT: + RETVAL diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/docs/Architecture.md new/Cavil-Matcher-1.01/docs/Architecture.md --- old/Cavil-Matcher-1.00/docs/Architecture.md 2026-07-21 21:08:46.064115638 +0200 +++ new/Cavil-Matcher-1.01/docs/Architecture.md 2026-08-04 20:20:04.757907627 +0200 @@ -112,12 +112,21 @@ ## The on-disk format Each segment file begins with a header identifying it, stating its format version, and carrying a checksum of -everything that follows. When a segment is opened, all of this is verified, and every internal reference in the -file is bounds-checked before it is ever used. A file that is truncated, corrupted, of the wrong version, or +everything that follows. Opening a segment always validates its **structure** — the header fields, the size, +and every internal reference — so a file that is truncated, of the wrong version, structurally impossible, or simply not a segment at all is rejected cleanly; it is never partially trusted and never able to send the -scanner off the end of the data. The manifest additionally records a checksum for each segment, so a segment -that has been damaged or swapped underneath a running system is detected and skipped rather than used. This -validated, versioned format is the deliberate replacement for the old engine's headerless, unchecked file. +scanner off the end of the data. This structural validation is cheap and always on, so memory safety never +depends on the checksum. + +The whole-payload **checksum** is a separate, corruption-detection concern, and it is deliberately *not* +recomputed on the hot scan path. A segment is Cavil's own derived cache: it is checksummed when compiled, +published atomically (written to the side and renamed into place), and thereafter immutable and regenerable +from the database. Re-checksumming a multi-hundred-megabyte payload on every one of the thousands of indexing +opens — where the mmap already shares one physical copy — is almost the entire cost of a load (measured at +~97%), and it guards against a corruption that atomic publishing already prevents. So the CRC is verified where +it is meaningful: once when the segment is written, and on demand through the engine's `verify` entry point (an +fsck for operators). The manifest additionally records a checksum for each segment for that on-demand check. +This validated, versioned format is the deliberate replacement for the old engine's headerless, unchecked file. ## Robustness diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/lib/Cavil/Matcher.pm new/Cavil-Matcher-1.01/lib/Cavil/Matcher.pm --- old/Cavil-Matcher-1.00/lib/Cavil/Matcher.pm 2026-07-22 14:20:47.936531607 +0200 +++ new/Cavil-Matcher-1.01/lib/Cavil/Matcher.pm 2026-08-04 20:28:57.000000000 +0200 @@ -6,7 +6,7 @@ use strict; use warnings; -our $VERSION = '1.00'; +our $VERSION = '1.01'; require XSLoader; XSLoader::load('Cavil::Matcher', $VERSION); @@ -142,12 +142,25 @@ =item dump($file) / load($file) Write the in-memory patterns to a compiled segment file / replace the engine's state with a single mmapped -segment file. +segment file. C<dump> fully verifies the compiled segment before publishing it, so a bad segment is never +written. + +C<load> (like C<attach>) validates the segment's structure but does not re-checksum its whole payload: a +compiled segment is an immutable, already-verified cache, and re-checksumming hundreds of megabytes on every +open is almost the entire cost of a load. Structure is always validated, so a malformed file is still rejected +and scanning is always memory-safe; only on-disk corruption of an otherwise well-formed file goes undetected +here (use L</verify> for that). =item attach($file) Memory-map an additional compiled segment into the active set. Returns false (without dying) on a missing or -invalid file. +structurally invalid file. Trusts the payload checksum like L</load>. + +=item verify($file) + +Fully check a segment file on disk — structure B<and> whole-payload checksum — and return true iff it is +intact. This is the corruption check that the scan path (C<load>/C<attach>) deliberately skips for speed; run +it on demand (an fsck for a cache directory) when you need to prove a segment has not rotted on disk. =item set_tombstones(\@pattern_ids) @@ -169,8 +182,10 @@ =head1 Cavil::Matcher::Bag tf-idf closest-match model from L</init_bag_of_patterns>: C<set_patterns(\%id_to_text)>, -C<best_for($text, $count)>, C<dump($file)> and C<load($file)>. C<dump> and C<load> return true on -success; a failed C<load> (missing/truncated file) leaves the model unchanged. +C<best_for($text, $count)>, C<dump($file)>, C<load($file)> and C<verify($file)>. C<dump> and C<load> +return true on success; a failed C<load> (missing/truncated/malformed file) leaves the model unchanged. +Like the engine, C<load> validates structure but trusts the payload checksum of this immutable, +already-verified cache; C<verify> is the on-demand full check (structure B<and> CRC) for an fsck. =head1 SEE ALSO diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/bag.cc new/Cavil-Matcher-1.01/src/bag.cc --- old/Cavil-Matcher-1.00/src/bag.cc 2026-07-21 15:26:34.843863847 +0200 +++ new/Cavil-Matcher-1.01/src/bag.cc 2026-08-04 20:26:55.145033312 +0200 @@ -166,6 +166,9 @@ bool ok = fwrite(&h, sizeof(h), 1, file) == 1; if (ok && !payload.empty()) ok = fwrite(payload.data(), payload.size(), 1, file) == 1; if (fclose(file) != 0) ok = false; + // Prove integrity once, at publish: fully re-read and check the temp (CRC + parse) before it becomes + // the live cache, so the load path can trust it without re-checksumming on every open. + if (ok && !verify(tmp)) ok = false; if (ok && rename(tmp.c_str(), path.c_str()) != 0) ok = false; if (!ok) remove(tmp.c_str()); // don't leave a partial temp behind return ok; @@ -193,7 +196,14 @@ }; } // namespace -bool Bag::load(const std::string& path) { +bool Bag::verify(const std::string& path) const { + // Full integrity check (CRC + parse) of a bag file, for an explicit fsck. The load path trusts the CRC + // of the immutable published cache; this is how it is proven on demand. + Bag tmp; + return tmp.load(path, /*verify_crc=*/true); +} + +bool Bag::load(const std::string& path, bool verify_crc) { FILE* file = fopen(path.c_str(), "rb"); if (!file) return false; @@ -225,7 +235,9 @@ const char* payload = buf.data() + sizeof(BagHeader); size_t payload_len = buf.size() - sizeof(BagHeader); - if (cavil_crc32(payload, payload_len) != h.crc32) return false; + // Corruption detection over the whole payload. Skipped on the trusted load path (see the header doc): + // the record parse below bounds-checks every read, so a malformed bag is rejected either way. + if (verify_crc && cavil_crc32(payload, payload_len) != h.crc32) return false; // Reject counts that could not possibly fit in the payload before allocating anything. if (h.idf_count > payload_len / 16) return false; // each idf entry is 16 bytes diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/bag.h new/Cavil-Matcher-1.01/src/bag.h --- old/Cavil-Matcher-1.00/src/bag.h 2026-07-21 02:38:02.943372287 +0200 +++ new/Cavil-Matcher-1.01/src/bag.h 2026-08-04 20:26:33.941077604 +0200 @@ -24,7 +24,12 @@ void set_patterns(const std::vector<std::pair<uint64_t, std::string>>& patterns); std::vector<Hit> best_for(const std::string& snippet, unsigned int count) const; bool dump(const std::string& path) const; - bool load(const std::string& path); + // verify_crc controls only the whole-payload CRC32. It defaults to true; the load path (bag_load) + // passes false. Parsing the records already bounds-checks every read and rejects a malformed file, so + // skipping the CRC on load costs no safety - only per-load detection of on-disk bit-rot, which the + // publish-time check and verify() cover. The bag is Cavil's own immutable, atomically-published cache. + bool load(const std::string& path, bool verify_crc = true); + bool verify(const std::string& path) const; // full CRC + parse check of a bag file private: struct TfIdf { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/bindings.cc new/Cavil-Matcher-1.01/src/bindings.cc --- old/Cavil-Matcher-1.00/src/bindings.cc 2026-07-21 21:59:06.236314690 +0200 +++ new/Cavil-Matcher-1.01/src/bindings.cc 2026-08-04 20:27:02.193018589 +0200 @@ -265,6 +265,7 @@ int matcher_dump(Matcher* m, const char* filename) { return m->dump(filename) ? 1 : 0; } int matcher_load(Matcher* m, const char* filename) { return m->load(filename) ? 1 : 0; } int matcher_attach(Matcher* m, const char* filename) { return m->attach(filename) ? 1 : 0; } +int matcher_verify(Matcher* m, const char* filename) { return m->verify(filename) ? 1 : 0; } void matcher_set_tombstones(Matcher* m, AV* ids) { dTHX; @@ -349,4 +350,5 @@ } int bag_dump(Bag* b, const char* filename) { return b->dump(filename) ? 1 : 0; } -int bag_load(Bag* b, const char* filename) { return b->load(filename) ? 1 : 0; } +int bag_load(Bag* b, const char* filename) { return b->load(filename, /*verify_crc=*/false) ? 1 : 0; } +int bag_verify(Bag* b, const char* filename) { return b->verify(filename) ? 1 : 0; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/bindings.h new/Cavil-Matcher-1.01/src/bindings.h --- old/Cavil-Matcher-1.00/src/bindings.h 2026-07-21 21:42:58.359155899 +0200 +++ new/Cavil-Matcher-1.01/src/bindings.h 2026-08-04 20:27:06.717009139 +0200 @@ -35,6 +35,7 @@ int matcher_dump(Matcher* m, const char* filename); int matcher_load(Matcher* m, const char* filename); int matcher_attach(Matcher* m, const char* filename); +int matcher_verify(Matcher* m, const char* filename); void matcher_set_tombstones(Matcher* m, AV* ids); void matcher_set_generation(Matcher* m, UV generation); UV matcher_generation(Matcher* m); @@ -46,5 +47,6 @@ AV* bag_best_for(Bag* b, const char* str, int count); int bag_dump(Bag* b, const char* filename); int bag_load(Bag* b, const char* filename); +int bag_verify(Bag* b, const char* filename); #endif diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/matcher.cc new/Cavil-Matcher-1.01/src/matcher.cc --- old/Cavil-Matcher-1.00/src/matcher.cc 2026-07-21 21:58:59.381353862 +0200 +++ new/Cavil-Matcher-1.01/src/matcher.cc 2026-08-04 19:40:44.653155107 +0200 @@ -64,15 +64,31 @@ auto mf = std::make_unique<MappedFile>(); if (!mf->map(path)) return false; auto seg = std::make_unique<Segment>(); - if (!seg->open(mf->data(), mf->size())) return false; + // Trusted scan path: skip the whole-payload CRC (verified at publish), keep structural validation. + if (!seg->open(mf->data(), mf->size(), /*verify_crc=*/false)) return false; _maps.push_back(std::move(mf)); _segments.push_back(std::move(seg)); return true; } +bool Matcher::verify(const std::string& path) const { + // Full integrity check (CRC + structure) of a segment file, for an explicit fsck. The scan path + // (load/attach) trusts the CRC of the immutable published cache; this is how it is proven on demand. + MappedFile mf; + if (!mf.map(path)) return false; + Segment seg; + return seg.open(mf.data(), mf.size(), /*verify_crc=*/true); +} + bool Matcher::dump(const std::string& path) { std::vector<char> buf = _build.compile(_generation); + // Publish is the one place integrity is proven: full-verify (CRC + structure) the freshly compiled + // buffer before it is written, so the scan path can safely trust this immutable file without re-CRCing + // it on every open. A buffer that fails here is never published. + Segment check; + if (!check.open(buf.data(), buf.size(), /*verify_crc=*/true)) return false; + // Write to a temp file and rename into place, so a crash/short write/full disk mid-dump cannot damage // an existing good cache at `path` (and a concurrent reader mmapping the old file keeps a valid inode // across the swap). The cache is disposable/rebuilt from the DB, so no fsync is needed. @@ -94,7 +110,8 @@ auto mf = std::make_unique<MappedFile>(); if (!mf->map(path)) return false; auto seg = std::make_unique<Segment>(); - if (!seg->open(mf->data(), mf->size())) return false; + // Trusted scan path: skip the whole-payload CRC (verified at publish), keep structural validation. + if (!seg->open(mf->data(), mf->size(), /*verify_crc=*/false)) return false; clear(); _maps.push_back(std::move(mf)); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/matcher.h new/Cavil-Matcher-1.01/src/matcher.h --- old/Cavil-Matcher-1.00/src/matcher.h 2026-07-21 21:42:39.523424438 +0200 +++ new/Cavil-Matcher-1.01/src/matcher.h 2026-08-04 19:40:39.133673080 +0200 @@ -51,6 +51,7 @@ // Segmented surface used by the Perl manifest layer. bool attach(const std::string& path); // add one mmapped segment to the active set + bool verify(const std::string& path) const; // full CRC+structure check of a segment file void set_tombstones(const std::vector<uint32_t>& ids); // pattern ids to drop before resolution void set_generation(uint64_t g) { _generation = g; } uint64_t generation() const { return _generation; } // the generation this engine was pinned to diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/segment.cc new/Cavil-Matcher-1.01/src/segment.cc --- old/Cavil-Matcher-1.00/src/segment.cc 2026-07-21 21:58:52.108264151 +0200 +++ new/Cavil-Matcher-1.01/src/segment.cc 2026-08-04 19:38:40.080716593 +0200 @@ -152,7 +152,7 @@ return open(_owned.data(), _owned.size()); } -bool Segment::open(const char* data, size_t len) { +bool Segment::open(const char* data, size_t len, bool verify_crc) { _valid = false; _base = data; _len = len; @@ -185,7 +185,11 @@ total += skip_bytes; if (total != len) return false; - if (cavil_crc32(data + sizeof(SegmentHeader), total - sizeof(SegmentHeader)) != h->payload_crc32) return false; + // Corruption detection over the whole payload. Skipped on the trusted scan path (see the header doc): + // structural validation below still runs, so a corrupt segment is rejected or safe regardless. + if (verify_crc + && cavil_crc32(data + sizeof(SegmentHeader), total - sizeof(SegmentHeader)) != h->payload_crc32) + return false; // Semantic header invariants. The CRC only proves the bytes are the ones the writer produced; it does // not prove a *malicious or buggy* writer produced sane values. These fields steer the scan (notably diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/src/segment.h new/Cavil-Matcher-1.01/src/segment.h --- old/Cavil-Matcher-1.00/src/segment.h 2026-07-21 21:58:18.730364439 +0200 +++ new/Cavil-Matcher-1.01/src/segment.h 2026-08-04 19:38:23.429846009 +0200 @@ -106,7 +106,15 @@ // Open a segment over an external, read-only buffer (e.g. an mmap). Returns false (and leaves the // segment unusable) if the buffer is not a valid segment. Never crashes on bad input. - bool open(const char* data, size_t len); + // + // verify_crc controls only the whole-payload CRC32 (corruption detection). It defaults to true - the + // compile/publish path and explicit integrity checks re-checksum everything. The scan path passes + // false: segments are Cavil's own immutable, atomically-published cache, already CRC-verified when + // written, so re-CRCing the entire payload on every mmap-open in every forked job is wasted work. + // The O(1) header/size checks and the full structural validation walk run either way, so a bad + // segment is always rejected or safe to scan - skipping the CRC never costs memory safety, only + // per-scan detection of on-disk bit-rot (which publish-time verification already covers). + bool open(const char* data, size_t len, bool verify_crc = true); // Take ownership of an in-memory compiled buffer. bool open_owned(std::vector<char>&& buf); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/t/09segment.t new/Cavil-Matcher-1.01/t/09segment.t --- old/Cavil-Matcher-1.00/t/09segment.t 2026-07-21 21:18:56.000000000 +0200 +++ new/Cavil-Matcher-1.01/t/09segment.t 2026-08-04 20:15:56.000000000 +0200 @@ -85,21 +85,29 @@ # --- Format safety: corrupt segments are rejected, never mis-read or crashed on ------------------- my $good = slurp($seg_a); -sub attach_bytes { - my $bytes = shift; - my $f = "$dir/corrupt.$$." . int(rand(1e9)); +# The scan path (attach/load) always validates a segment's shape and structure - that is what keeps a bad +# file from ever being mis-read or crashed on. It deliberately does NOT re-checksum the whole payload: +# segments are Cavil's own immutable, atomically-published cache, CRC-verified when written. The full CRC +# is available on demand via verify() (and runs at publish time). +sub attach_bytes { _try_bytes('attach', @_) } +sub verify_bytes { _try_bytes('verify', @_) } + +sub _try_bytes { + my ($method, $bytes) = @_; + my $f = "$dir/corrupt.$$." . int(rand(1e9)); open my $fh, '>', $f or die $!; binmode $fh; print {$fh} $bytes; close $fh; my $m = Cavil::Matcher::init_matcher(); - my $ok = $m->attach($f); + my $ok = $m->$method($f); unlink $f; return $ok; } is(attach_bytes($good), 1, 'a valid segment attaches'); +# Structural/shape corruption is always rejected, even on the trusted scan path. my $bad_magic = $good; substr($bad_magic, 0, 1) = 'X'; is(attach_bytes($bad_magic), 0, 'wrong magic rejected'); @@ -108,15 +116,34 @@ substr($bad_version, 8, 4) = pack('L', 999); is(attach_bytes($bad_version), 0, 'wrong format version rejected'); -my $bad_crc = $good; -substr($bad_crc, length($bad_crc) - 1, 1) = chr((ord(substr($bad_crc, length($bad_crc) - 1, 1)) ^ 0xFF)); -is(attach_bytes($bad_crc), 0, 'flipped payload byte fails CRC'); - is(attach_bytes(substr($good, 0, 20)), 0, 'truncated file rejected'); is(attach_bytes(''), 0, 'empty file rejected'); is(attach_bytes('not a segment at all, just random text bytes here'), 0, 'garbage file rejected'); is(attach_bytes($good . 'trailing junk'), 0, 'trailing bytes after payload rejected'); +# An out-of-range index is caught by the structural walk, which runs on the scan path regardless of the +# CRC - so memory safety never depends on the checksum. Packed layout is header(56) | nodes | children | +# skips; FlatNode is 20 bytes, and each FlatChild starts with an 8-byte hash then a 4-byte child_node. +my ($node_count, $child_count) = unpack('L L', substr($good, 32, 8)); +cmp_ok($child_count, '>', 0, 'segment A has children to corrupt'); +my $bad_index = $good; +my $child_node_off = 56 + $node_count * 20 + 8; +substr($bad_index, $child_node_off, 4) = pack('L', 0xFFFFFFFF); +is(attach_bytes($bad_index), 0, 'out-of-range child index rejected on the scan path (memory safety)'); + +# The whole-payload CRC is checked by verify(), not by the scan path. Corrupt only the stored CRC field +# (header offset 48) so the payload and structure stay valid: verify() rejects it, the scan path trusts it. +is(verify_bytes($good), 1, 'verify() passes a good segment'); +my $bad_stored_crc = $good; +substr($bad_stored_crc, 48, 4) = pack('L', unpack('L', substr($good, 48, 4)) ^ 0xFFFFFFFF); +is(verify_bytes($bad_stored_crc), 0, 'verify() catches a bad CRC'); +is(attach_bytes($bad_stored_crc), 1, 'scan path trusts a structurally-valid segment despite a bad CRC'); + +# A flipped payload byte is likewise caught by the full check. +my $bad_crc = $good; +substr($bad_crc, length($bad_crc) - 1, 1) = chr((ord(substr($bad_crc, length($bad_crc) - 1, 1)) ^ 0xFF)); +is(verify_bytes($bad_crc), 0, 'verify() catches a flipped payload byte'); + # A matcher whose only segment failed to attach simply finds nothing (no crash). my $none = Cavil::Matcher::init_matcher(); $none->attach("$dir/does-not-exist.seg"); @@ -302,12 +329,11 @@ ok($e->load($good_seg), 'load a good segment'); cmp_deeply($e->find_matches($probe), [[1, 1, 1]], 'the loaded segment matches'); - # A corrupt segment (flipped payload byte -> CRC failure) fails to load... - my $bad_bytes = slurp($good_seg); - substr($bad_bytes, length($bad_bytes) - 1, 1) = chr(ord(substr($bad_bytes, length($bad_bytes) - 1, 1)) ^ 0xFF); + # A structurally-invalid segment (truncated) fails to load - the scan path rejects malformed structure + # even though it trusts the CRC of a well-formed file... my $bad_seg = "$dir/loadbad.seg"; open my $bf, '>:raw', $bad_seg or die $!; - print {$bf} $bad_bytes; + print {$bf} substr(slurp($good_seg), 0, 20); close $bf; is($e->load($bad_seg), 0, 'loading a corrupt segment fails'); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/t/12bag.t new/Cavil-Matcher-1.01/t/12bag.t --- old/Cavil-Matcher-1.00/t/12bag.t 2026-07-21 15:29:00.000000000 +0200 +++ new/Cavil-Matcher-1.01/t/12bag.t 2026-08-04 20:31:34.083395197 +0200 @@ -71,8 +71,10 @@ is_deeply($bag->best_for('permission is hereby granted', 0), [], 'best_for(count=0) returns empty'); is_deeply($bag->best_for('permission is hereby granted', -1), [], 'best_for(negative count) returns empty'); -# The cache is a versioned, CRC-checked format: truncated / wrong-magic / wrong-version / corrupt -# files are all rejected, and a rejected load must leave any existing model intact (not wipe it). +# The load path validates structure (the record parse bounds-checks every read) but does NOT re-checksum +# the whole payload: the bag is Cavil's own immutable, atomically-published cache. Structural corruption +# is still rejected on load; whole-payload corruption is caught by verify() (and at publish time). A +# rejected load must leave any existing model intact (not wipe it). my $sample = slurp('t/fixtures/licenses/04license.1.txt'); my $before = $loaded->best_for($sample, 1); ok(@$before, 'model matches before a bad load'); @@ -80,17 +82,26 @@ sub write_bytes { my ($p, $b) = @_; open my $o, '>:raw', $p or die $!; print {$o} $b; close $o; $p } -# Truncated payload (valid header, partial body): CRC fails. +# Truncated payload (valid header, partial body): the record parse runs out and rejects it. is($loaded->load(write_bytes("$dir/truncated", substr($good, 0, 48))), 0, 'truncated bag rejected'); # Not a bag at all (wrong magic). is($loaded->load(write_bytes("$dir/garbage", 'not a bag file, just some bytes here at all')), 0, 'wrong-magic bag rejected'); -# Valid magic but a flipped payload byte fails the CRC. +# The whole-payload CRC is checked by verify(), not by the load path. Corrupt only the stored CRC field +# (header offset 12) so the payload stays valid: verify() rejects it, load() trusts it. +is($loaded->verify($file), 1, 'verify() passes a good bag'); +my $bad_stored_crc = $good; +substr($bad_stored_crc, 12, 4) = pack('V', unpack('V', substr($good, 12, 4)) ^ 0xFFFFFFFF); +is($loaded->verify(write_bytes("$dir/badcrc", $bad_stored_crc)), 0, 'verify() catches a bad CRC'); +is($loaded->load(write_bytes("$dir/badcrc", $bad_stored_crc)), 1, + 'load() trusts a valid-payload bag despite a bad CRC'); + +# A flipped payload byte is likewise caught by the full check. my $corrupt = $good; substr($corrupt, length($corrupt) - 1, 1) = chr(ord(substr($corrupt, length($corrupt) - 1, 1)) ^ 0xFF); -is($loaded->load(write_bytes("$dir/corrupt", $corrupt)), 0, 'flipped payload byte fails CRC'); +is($loaded->verify(write_bytes("$dir/corrupt", $corrupt)), 0, 'verify() catches a flipped payload byte'); # Strictness (like the segment reader): a CRC-valid file with trailing bytes after the declared records, # or with a pattern's tf_idfs not in the strictly-ascending order compare2 relies on, must be rejected - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Cavil-Matcher-1.00/t/15segment_header.t new/Cavil-Matcher-1.01/t/15segment_header.t --- old/Cavil-Matcher-1.00/t/15segment_header.t 2026-07-21 21:05:19.000000000 +0200 +++ new/Cavil-Matcher-1.01/t/15segment_header.t 2026-08-04 20:16:54.000000000 +0200 @@ -56,6 +56,13 @@ is(accepts($good), 1, 'the untampered segment attaches'); is(crc32(substr($good, 56)), unpack('V', substr($good, 48, 4)), 'pure-Perl CRC32 matches the stored payload CRC'); +# Guardrail: the on-disk format version must stay 2. The scan path trusts an already-published segment's +# CRC (it verifies structure but not the whole-payload checksum), which makes an in-place production +# upgrade seamless ONLY as long as the format is unchanged. Bumping this without a migration plan would +# let a new writer publish segments an old reader rejects (and vice versa) - so if this fails, the format +# changed and the upgrade is no longer drop-in. +is(unpack('V', substr($good, 8, 4)), 2, 'segment format version is still 2 (in-place upgrade stays seamless)'); + my $node_count = unpack('V', substr($good, 32, 4)); my $child_count = unpack('V', substr($good, 36, 4)); my $skip_count = unpack('V', substr($good, 40, 4)); ++++++ _scmsync.obsinfo ++++++ --- /var/tmp/diff_new_pack.LSAZqV/_old 2026-08-06 16:29:00.522671973 +0200 +++ /var/tmp/diff_new_pack.LSAZqV/_new 2026-08-06 16:29:00.526672113 +0200 @@ -1,6 +1,6 @@ -mtime: 1784733830 -commit: 48884f1377224ebc38ebf8246b04ad87a8bed51bf66dd79fb9ae0f21d88590e9 +mtime: 1785916160 +commit: a3fe64e68ac939b73fcd0b0e07207129b78d41c9f326f32c4073c47d6e5167fc url: https://src.opensuse.org/perl/perl-Cavil-Matcher -revision: 48884f1377224ebc38ebf8246b04ad87a8bed51bf66dd79fb9ae0f21d88590e9 +revision: a3fe64e68ac939b73fcd0b0e07207129b78d41c9f326f32c4073c47d6e5167fc projectscmsync: https://src.opensuse.org/perl/_ObsPrj ++++++ build.specials.obscpio ++++++ ++++++ build.specials.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/.gitignore new/.gitignore --- old/.gitignore 1970-01-01 01:00:00.000000000 +0100 +++ new/.gitignore 2026-08-05 09:49:20.000000000 +0200 @@ -0,0 +1,5 @@ +*.obscpio +*.osc +_build.* +_service:* +.pbuild
