This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-tck.git
The following commit(s) were added to refs/heads/main by this push:
new 66cd863 Unify snapshot updates and publish complete batches (#16)
66cd863 is described below
commit 66cd86373f0ff1f19f0a0778bc4e8ffa01d4522e
Author: tison <[email protected]>
AuthorDate: Mon Oct 5 09:03:03 2026 +0800
Unify snapshot updates and publish complete batches (#16)
Signed-off-by: tison <[email protected]>
---
.github/workflows/update-snapshots.yml | 27 +--
AGENTS.md | 8 +
cmd/tck/__snapshots__/output_test.snap | 12 +-
cmd/tck/__snapshots__/root_test.snap | 3 +-
cmd/tck/output.go | 87 +++++-----
cmd/tck/output_test.go | 8 +-
cmd/tck/snapshots.go | 180 +++++++-------------
docs/serialization-snapshots.md | 64 +++++---
internal/snapshots/config.go | 23 ---
internal/snapshots/diff_test.go | 5 +-
internal/snapshots/publish.go | 123 ++++++++++++++
internal/snapshots/publish_test.go | 106 ++++++++++++
internal/snapshots/reconcile.go | 290 ++++++++++++---------------------
internal/snapshots/reconcile_test.go | 265 ++++++++++++++++++++++++++++++
14 files changed, 774 insertions(+), 427 deletions(-)
diff --git a/.github/workflows/update-snapshots.yml
b/.github/workflows/update-snapshots.yml
index a00d33b..36c2a83 100644
--- a/.github/workflows/update-snapshots.yml
+++ b/.github/workflows/update-snapshots.yml
@@ -48,24 +48,19 @@ jobs:
- name: Update snapshots
id: update
shell: bash
+ run: mise run tck -- snapshots update --skip-unstable-only 2>&1 | tee
"$RUNNER_TEMP/snapshot-update.log"
+
+ - name: Summarize update
+ if: always() && steps.update.outcome != 'skipped'
run: |
- status=0
- for language in cpp go java; do
- if ! mise run tck -- snapshots update "$language" HEAD 2>&1 | tee
-a update.log; then
- status=1
- fi
- done
- echo "status=${status}" >> "$GITHUB_OUTPUT"
{
echo '## Snapshot update'
echo
echo '```'
- grep -E '^(Update |Summary:|Source revision:|✓|✗|Error:)'
update.log || true
+ grep -E '^(Update |Summary:|Source revision:|✓|✗|Warning:|Error:)'
"$RUNNER_TEMP/snapshot-update.log" || true
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- # Sources that updated successfully are published even when another
- # source failed, so one broken upstream does not hold back the others.
- name: Commit updated snapshots
run: |
if [[ -z "$(git status --porcelain -- config.toml
serialization_test_data)" ]]; then
@@ -78,13 +73,7 @@ jobs:
{
echo "chore: update snapshots to the latest upstream revisions"
echo
- grep -E '^(Update |Summary:|Source revision:)' update.log || true
- } > commit-message.txt
- git commit --quiet --file commit-message.txt
+ grep -E '^(Update |Summary:|Source revision:)'
"$RUNNER_TEMP/snapshot-update.log" || true
+ } > "$RUNNER_TEMP/snapshot-commit-message.txt"
+ git commit --quiet --file "$RUNNER_TEMP/snapshot-commit-message.txt"
git push origin HEAD:${{ github.ref_name }}
-
- - name: Report failed sources
- if: steps.update.outputs.status != '0'
- run: |
- grep -E '^(✗|Error:)' update.log || true
- exit 1
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..73faa48
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,8 @@
+# Repository Guidelines
+
+The global working agreements apply. Keep this file limited to constraints
specific to this repository.
+
+## Markdown Style
+
+- Keep each semantic prose paragraph and each list item on one source line; do
not hard-wrap within a paragraph or list item.
+- Format Markdown tables with aligned pipe columns.
diff --git a/cmd/tck/__snapshots__/output_test.snap
b/cmd/tck/__snapshots__/output_test.snap
index 7f29055..df92ef2 100644
--- a/cmd/tck/__snapshots__/output_test.snap
+++ b/cmd/tck/__snapshots__/output_test.snap
@@ -6,8 +6,8 @@
│ M │ unstable │ allow │ probabilistic.sk │ old 12 B · aaaaaaaaaaaa │
│ │ │ │ │ new 12 B · bbbbbbbbbbbb │
╰────────┴───────────┴───────┴──────────────────┴─────────────────────────╯
-Summary: 0 added · 1 modified (1 probabilistic) · 0 deleted
-! Snapshot set and stable contents are unchanged; 1 probabilistic content
modification is allowed.
+Summary: 0 added · 1 modified (1 unstable) · 0 deleted
+! Snapshot set and stable contents are unchanged; 1 unstable content
modification is allowed.
---
@@ -20,19 +20,19 @@ Summary: 0 added · 1 modified (1 probabilistic) · 0 deleted
│ │ │ │ │ new 16 B · bbbbbbbbbbbb │
│ D │ stable │ block │ deleted.sk │ old 12 B · aaaaaaaaaaaa │
╰────────┴───────────┴───────┴─────────────┴─────────────────────────╯
-Summary: 1 added · 1 modified (1 probabilistic) · 1 deleted
+Summary: 1 added · 1 modified (1 unstable) · 1 deleted
✗ serialization_test_data/go_generated_files has 2 blocking changes.
---
-[TestPrintResultReportsUnwrittenUpdate - 1]
+[TestPrintUpdateResultReportsSkippedChanges - 1]
╭────────┬───────────┬───────┬──────────────────┬─────────────────────────╮
│ CHANGE │ STABILITY │ CHECK │ SNAPSHOT │ CONTENT (SIZE · SHA256) │
├────────┼───────────┼───────┼──────────────────┼─────────────────────────┤
│ M │ unstable │ allow │ probabilistic.sk │ old 12 B · aaaaaaaaaaaa │
│ │ │ │ │ new 12 B · bbbbbbbbbbbb │
╰────────┴───────────┴───────┴──────────────────┴─────────────────────────╯
-Summary: 0 added · 1 modified (1 probabilistic) · 0 deleted
-✓ snapshots already matches its source revision; 1 probabilistic content
modification was not written.
+Summary: 0 added · 1 modified (1 unstable) · 0 deleted
+✓ Skipped snapshots: source revision unchanged; 1 unstable content
modification was not written.
---
diff --git a/cmd/tck/__snapshots__/root_test.snap
b/cmd/tck/__snapshots__/root_test.snap
index 7442f13..b0127ff 100644
--- a/cmd/tck/__snapshots__/root_test.snap
+++ b/cmd/tck/__snapshots__/root_test.snap
@@ -8,8 +8,7 @@ Usage:
Available Commands:
check Check the snapshot set and stable snapshot contents
- sync Synchronize snapshots from config.toml
- update Adopt one upstream revision and update its snapshots
+ update Update snapshots and source revisions together
Flags:
-h, --help help for snapshots
diff --git a/cmd/tck/output.go b/cmd/tck/output.go
index 4776f46..53cb32d 100644
--- a/cmd/tck/output.go
+++ b/cmd/tck/output.go
@@ -29,20 +29,56 @@ import (
"github.com/jedib0t/go-pretty/v6/text"
)
-func printResult(output io.Writer, root string, mode snapshots.Mode, result
snapshots.Result) error {
+func printCheckResult(output io.Writer, root string, result snapshots.Result)
error {
+ if err := printChanges(output, result); err != nil {
+ return err
+ }
target := displayPath(root, result.Target)
- if len(result.Changes) == 0 {
- if result.RevisionChanged() {
- if err := printRevisionChange(output, result); err !=
nil {
- return err
- }
- _, err := fmt.Fprintf(output, "✓ Updated %s.\n", target)
- return err
- }
+ switch {
+ case len(result.Changes) == 0:
_, err := fmt.Fprintf(output, "✓ %s is up to date.\n", target)
return err
+ case result.HasBlockingChanges():
+ count := result.BlockingChangeCount()
+ _, err := fmt.Fprintf(output, "✗ %s has %d blocking %s.\n",
target, count, plural(count, "change", "changes"))
+ return err
+ default:
+ count := len(result.Changes)
+ _, err := fmt.Fprintf(output, "! Snapshot set and stable
contents are unchanged; %d unstable content %s allowed.\n",
+ count, plural(count, "modification is", "modifications
are"))
+ return err
}
+}
+func printUpdateResult(output io.Writer, root string, result snapshots.Result,
skipUnstableOnly bool) error {
+ if err := printChanges(output, result); err != nil {
+ return err
+ }
+ target := displayPath(root, result.Target)
+ switch {
+ case skipUnstableOnly && result.UnstableOnly():
+ _, err := fmt.Fprintf(output, "✓ Skipped %s: source revision
unchanged; %d unstable content %s not written.\n",
+ target, len(result.Changes),
plural(len(result.Changes), "modification was", "modifications were"))
+ return err
+ case result.RevisionChanged() && len(result.Changes) == 0:
+ _, err := fmt.Fprintf(output, "✓ Updated source revision for
%s; snapshot contents are unchanged.\n", target)
+ return err
+ case len(result.Changes) == 0:
+ _, err := fmt.Fprintf(output, "✓ %s is up to date.\n", target)
+ return err
+ default:
+ _, err := fmt.Fprintf(output, "✓ Updated %s.\n", target)
+ return err
+ }
+}
+
+func printChanges(output io.Writer, result snapshots.Result) error {
+ if len(result.Changes) == 0 {
+ if result.RevisionChanged() {
+ return printRevisionChange(output, result)
+ }
+ return nil
+ }
counts := make(map[snapshots.ChangeStatus]int)
unstableModified := 0
changeTable := table.NewWriter()
@@ -70,7 +106,7 @@ func printResult(output io.Writer, root string, mode
snapshots.Mode, result snap
}
if _, err := fmt.Fprintf(
output,
- "Summary: %d added · %d modified (%d probabilistic) · %d
deleted\n",
+ "Summary: %d added · %d modified (%d unstable) · %d deleted\n",
counts[snapshots.ChangeAdded],
counts[snapshots.ChangeModified],
unstableModified,
@@ -83,36 +119,7 @@ func printResult(output io.Writer, root string, mode
snapshots.Mode, result snap
return err
}
}
-
- switch {
- case (mode == snapshots.ModeUpdate || mode == snapshots.ModeSync) &&
!result.Written:
- _, err := fmt.Fprintf(
- output,
- "✓ %s already matches its source revision; %d
probabilistic content %s not written.\n",
- target,
- unstableModified,
- plural(unstableModified, "modification was",
"modifications were"),
- )
- return err
- case mode == snapshots.ModeUpdate:
- _, err := fmt.Fprintf(output, "✓ Updated %s.\n", target)
- return err
- case mode == snapshots.ModeSync:
- _, err := fmt.Fprintf(output, "✓ Synced %s.\n", target)
- return err
- case result.HasBlockingChanges():
- count := result.BlockingChangeCount()
- _, err := fmt.Fprintf(output, "✗ %s has %d blocking %s.\n",
target, count, plural(count, "change", "changes"))
- return err
- default:
- _, err := fmt.Fprintf(
- output,
- "! Snapshot set and stable contents are unchanged; %d
probabilistic content %s allowed.\n",
- unstableModified,
- plural(unstableModified, "modification is",
"modifications are"),
- )
- return err
- }
+ return nil
}
func printRevisionChange(output io.Writer, result snapshots.Result) error {
diff --git a/cmd/tck/output_test.go b/cmd/tck/output_test.go
index 15f2d2c..3c529d4 100644
--- a/cmd/tck/output_test.go
+++ b/cmd/tck/output_test.go
@@ -52,7 +52,7 @@ func TestPrintResultIncludesDetailedFileChanges(t *testing.T)
{
}
var output bytes.Buffer
- require.NoError(t, printResult(&output, root, snapshots.ModeCheck,
result))
+ require.NoError(t, printCheckResult(&output, root, result))
snaps.WithConfig(snaps.Raw()).MatchSnapshot(t, output.String())
}
@@ -70,11 +70,11 @@ func TestPrintResultAllowsOnlyUnstableModifications(t
*testing.T) {
}
var output bytes.Buffer
- require.NoError(t, printResult(&output, ".", snapshots.ModeCheck,
result))
+ require.NoError(t, printCheckResult(&output, ".", result))
snaps.WithConfig(snaps.Raw()).MatchSnapshot(t, output.String())
}
-func TestPrintResultReportsUnwrittenUpdate(t *testing.T) {
+func TestPrintUpdateResultReportsSkippedChanges(t *testing.T) {
t.Parallel()
before := testMetadata(12, 0xaa)
@@ -90,7 +90,7 @@ func TestPrintResultReportsUnwrittenUpdate(t *testing.T) {
}
var output bytes.Buffer
- require.NoError(t, printResult(&output, ".", snapshots.ModeUpdate,
result))
+ require.NoError(t, printUpdateResult(&output, ".", result, true))
snaps.WithConfig(snaps.Raw()).MatchSnapshot(t, output.String())
}
diff --git a/cmd/tck/snapshots.go b/cmd/tck/snapshots.go
index 19c8aca..ab7db11 100644
--- a/cmd/tck/snapshots.go
+++ b/cmd/tck/snapshots.go
@@ -24,6 +24,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "slices"
"strings"
"github.com/apache/datasketches-tck/internal/snapshots"
@@ -43,152 +44,95 @@ func newSnapshotsCommand() *cobra.Command {
}
command.AddCommand(
newSnapshotCheckCommand(),
- newSnapshotSyncCommand(),
newSnapshotUpdateCommand(),
)
return command
}
func newSnapshotCheckCommand() *cobra.Command {
- validLanguages := append(snapshots.Languages(), "all")
return &cobra.Command{
Use: "check <cpp|go|java|all>",
- Short: reconcileDescription(snapshots.ModeCheck),
- Long: reconcileDetails(snapshots.ModeCheck),
+ Short: "Check the snapshot set and stable snapshot
contents",
+ Long: "Regenerate snapshots at the configured commits
without writing files. Additions, deletions, and stable content changes fail
the check; unstable content changes are allowed.",
Args: cobra.MatchAll(cobra.ExactArgs(1),
cobra.OnlyValidArgs),
- ValidArgs: validLanguages,
+ ValidArgs: append(snapshots.Languages(), "all"),
RunE: func(command *cobra.Command, args []string) error {
+ root, err := repositoryRoot(command.Context())
+ if err != nil {
+ return err
+ }
languages := []string{args[0]}
if args[0] == "all" {
languages = snapshots.Languages()
}
- return reconcileSnapshots(command, snapshots.ModeCheck,
languages, "")
- },
- }
-}
-
-func newSnapshotSyncCommand() *cobra.Command {
- return &cobra.Command{
- Use: "sync",
- Short: reconcileDescription(snapshots.ModeSync),
- Long: reconcileDetails(snapshots.ModeSync),
- Args: cobra.NoArgs,
- RunE: func(command *cobra.Command, _ []string) error {
- return reconcileSnapshots(command, snapshots.ModeSync,
snapshots.Languages(), "")
+ outOfDate := false
+ for _, language := range languages {
+ result, err :=
snapshots.Check(command.Context(), root, language, command.OutOrStdout(),
command.ErrOrStderr())
+ if err != nil {
+ return err
+ }
+ if _, err := fmt.Fprintf(command.OutOrStdout(),
"\nCheck %s snapshots\n", languageHeading(language)); err != nil {
+ return err
+ }
+ if err :=
printCheckResult(command.OutOrStdout(), root, result); err != nil {
+ return err
+ }
+ outOfDate = outOfDate ||
result.HasBlockingChanges()
+ }
+ if outOfDate {
+ return errSnapshotsOutOfDate
+ }
+ return nil
},
}
}
func newSnapshotUpdateCommand() *cobra.Command {
- return &cobra.Command{
- Use: "update <cpp|go|java> <revision>",
- Short: reconcileDescription(snapshots.ModeUpdate),
- Long: reconcileDetails(snapshots.ModeUpdate),
+ var skipUnstableOnly bool
+ command := &cobra.Command{
+ Use: "update [cpp|go|java] [revision]",
+ Short: "Update snapshots and source revisions together",
+ Long: "Update all sources to their remote HEADs, or select one
language and optionally a branch, tag, or commit. All selected sources are
generated before any snapshots or pins are published.",
Args: func(command *cobra.Command, args []string) error {
- if err := cobra.ExactArgs(2)(command, args); err != nil
{
+ if err := cobra.RangeArgs(0, 2)(command, args); err !=
nil {
return err
}
- for _, language := range snapshots.Languages() {
- if args[0] == language {
- return nil
- }
+ if len(args) > 0 &&
!slices.Contains(snapshots.Languages(), args[0]) {
+ return fmt.Errorf("unsupported snapshot
language %q", args[0])
}
- return fmt.Errorf("unsupported snapshot language %q",
args[0])
+ return nil
},
ValidArgs: snapshots.Languages(),
RunE: func(command *cobra.Command, args []string) error {
- return reconcileSnapshots(command,
snapshots.ModeUpdate, []string{args[0]}, args[1])
- },
- }
-}
-
-func reconcileSnapshots(
- command *cobra.Command,
- mode snapshots.Mode,
- languages []string,
- requestedRevision string,
-) error {
- root, err := repositoryRoot(command.Context())
- if err != nil {
- return err
- }
-
- outOfDate := false
- for index, language := range languages {
- if index > 0 {
- if _, err := fmt.Fprintln(command.OutOrStdout()); err
!= nil {
- return fmt.Errorf("separate snapshot results:
%w", err)
+ root, err := repositoryRoot(command.Context())
+ if err != nil {
+ return err
}
- }
- if _, err := fmt.Fprintf(
- command.OutOrStdout(),
- "%s %s snapshots\n",
- modeHeading(mode),
- languageHeading(language),
- ); err != nil {
- return fmt.Errorf("print snapshot heading: %w", err)
- }
-
- result, err := snapshots.Reconcile(
- command.Context(),
- root,
- language,
- mode,
- requestedRevision,
- command.OutOrStdout(),
- command.ErrOrStderr(),
- )
- if err != nil {
- return err
- }
- if err := printResult(command.OutOrStdout(), root, mode,
result); err != nil {
- return fmt.Errorf("print snapshot result: %w", err)
- }
- outOfDate = outOfDate || (mode == snapshots.ModeCheck &&
result.HasBlockingChanges())
- }
- if outOfDate {
- return errSnapshotsOutOfDate
- }
- return nil
-}
-
-func reconcileDescription(mode snapshots.Mode) string {
- switch mode {
- case snapshots.ModeCheck:
- return "Check the snapshot set and stable snapshot contents"
- case snapshots.ModeSync:
- return "Synchronize snapshots from config.toml"
- case snapshots.ModeUpdate:
- return "Adopt one upstream revision and update its snapshots"
- default:
- panic(fmt.Sprintf("unsupported snapshot mode %q", mode))
- }
-}
-
-func reconcileDetails(mode snapshots.Mode) string {
- switch mode {
- case snapshots.ModeCheck:
- return "Generate snapshots and verify that the file set and
stable contents match. Content changes to existing probabilistic snapshots are
reported but do not fail the check."
- case snapshots.ModeSync:
- return "Regenerate all snapshot directories from the
repositories and commits in config.toml without changing the config."
- case snapshots.ModeUpdate:
- return "Resolve an upstream commit, branch, or tag, record its
exact commit ID in config.toml, and regenerate that source's snapshots.
Snapshots are left untouched when the revision is unchanged and only
probabilistic snapshots differ."
- default:
- panic(fmt.Sprintf("unsupported snapshot mode %q", mode))
- }
-}
-
-func modeHeading(mode snapshots.Mode) string {
- switch mode {
- case snapshots.ModeCheck:
- return "Check"
- case snapshots.ModeSync:
- return "Sync"
- case snapshots.ModeUpdate:
- return "Update"
- default:
- panic(fmt.Sprintf("unsupported snapshot mode %q", mode))
+ language, revision := "", ""
+ if len(args) > 0 {
+ language = args[0]
+ }
+ if len(args) > 1 {
+ revision = args[1]
+ }
+ results, err := snapshots.Update(command.Context(),
root, language, revision, skipUnstableOnly, command.OutOrStdout(),
command.ErrOrStderr())
+ if err != nil {
+ return err
+ }
+ for _, result := range results {
+ if _, err := fmt.Fprintf(command.OutOrStdout(),
"\nUpdate %s\n", displayPath(root, result.Target)); err != nil {
+ return err
+ }
+ if err :=
printUpdateResult(command.OutOrStdout(), root, result, skipUnstableOnly); err
!= nil {
+ return err
+ }
+ }
+ return nil
+ },
}
+ command.Flags().BoolVar(&skipUnstableOnly, "skip-unstable-only", false,
+ "Skip a source when its revision is unchanged and only existing
unstable snapshots differ")
+ return command
}
func languageHeading(language string) string {
diff --git a/docs/serialization-snapshots.md b/docs/serialization-snapshots.md
index 3b4eda9..5d076ae 100644
--- a/docs/serialization-snapshots.md
+++ b/docs/serialization-snapshots.md
@@ -31,32 +31,52 @@ repository = "https://github.com/apache/datasketches-go.git";
commit = "730c0ca31e00b8becf8b70591ae8ca73954912d0"
```
-## Update to the latest snapshots
+## Update snapshots
-With the [toolchain installed](#set-up-the-toolchain), use `HEAD` to update
each source to its default branch:
+With the [toolchain installed](#set-up-the-toolchain), update every available
source to its remote default branch:
```shell
-mise run tck -- snapshots update cpp HEAD
-mise run tck -- snapshots update go HEAD
-mise run tck -- snapshots update java HEAD
+mise run tck -- snapshots update
```
-For each source, the command resolves the default branch to an exact commit
ID, writes it to `config.toml`, and regenerates
`serialization_test_data/<language>_generated_files`. Generation happens before
either file set is changed, so a source whose generation fails is left
untouched.
+Update is the only command that writes snapshots. It accepts an optional
language and, after that, an optional revision:
-When a source's commit has not changed and only probabilistic snapshots
differ, its snapshots are left as they are, so repeated updates do not add
binary churn to the history.
+| Command | Sources and revisions
|
+|------------------------------|------------------------------------------------------------|
+| `snapshots update` | All available languages at their remote
`HEAD` |
+| `snapshots update go` | Go at its remote `HEAD`
|
+| `snapshots update go HEAD` | The same explicit selection
|
+| `snapshots update go v0.2.0` | Go at the specified tag; branches and commit
IDs also work |
-The [Update snapshots](#github-actions) workflow runs these commands weekly
and commits the result to `main`. It continues with the other languages if one
fails, then reports the failure after committing successful updates.
+A revision can only be supplied after a language. To regenerate or repair
snapshots at an existing pin, pass the commit recorded in `config.toml`
explicitly. Updating to an older revision follows the same process as updating
to a newer one. If the upstream build or output layout changed, update the
corresponding adapter in `internal/snapshots/<language>.go` before running the
command.
-## Update one source
+Every selected source is generated in a temporary workspace and compared
before any repository files are replaced. A generation or staging failure
aborts the whole batch. Successful updates publish the selected snapshot
directories and their exact commits in `config.toml` together. A changed
revision with byte-identical snapshots only changes the pin, after generation
has verified those contents.
-Update a single source to its default branch, or to a specific branch, tag, or
commit:
+### Skip unstable-only changes
+
+`stable` snapshots have deterministic contents. `unstable` snapshots are known
to vary between generations, even at the same source commit. By default, update
adopts all generated differences, including unstable contents. Add
`--skip-unstable-only` to any update invocation to avoid rewriting a source
whose commit is unchanged and whose only differences are modifications to
existing unstable files:
```shell
-mise run tck -- snapshots update go HEAD
-mise run tck -- snapshots update go v0.2.0
+mise run tck -- snapshots update --skip-unstable-only
+mise run tck -- snapshots update go --skip-unstable-only
+mise run tck -- snapshots update go v0.2.0 --skip-unstable-only
```
-Pinning an older revision is useful for reproducing a release or for holding a
source back while an upstream problem is fixed. If the upstream build or output
layout changed, update the corresponding adapter in
`internal/snapshots/<language>.go` before running the command.
+The option applies independently to each selected language. It skips that
source's entire update; it never filters individual files from an adopted
snapshot set. A revision change, any added or deleted file, or any stable
content change causes the complete generated snapshot set to be adopted,
including its unstable files. The report identifies skipped sources and shows
observed differences even when they were not written.
+
+| Source revision | Snapshot differences | Default
| With `--skip-unstable-only` |
+|-----------------|------------------------------------------|------------------------------|------------------------------|
+| Unchanged | None | No changes
| No changes |
+| Unchanged | Only existing unstable contents | Replace
snapshots | Keep the existing snapshots |
+| Unchanged | Additions, deletions, or stable contents | Replace
snapshots | Replace snapshots |
+| Changed | None | Update the pin
| Update the pin |
+| Changed | Any | Update the pin
and snapshots | Update the pin and snapshots |
+
+### Publication and failures
+
+The publisher stages the complete batch on the repository filesystem and
retains backups until all replacements succeed. An installation error rolls
back earlier replacements, including those for other languages. If rollback
itself fails, the command reports the recovery directory and keeps its backups.
Failure to remove backups after a successful publication is a warning and does
not undo the published pins or snapshots.
+
+This is rollback protection for ordinary errors, not a crash-atomic filesystem
transaction or synchronization for concurrent local writers. Do not run
multiple updates against the same checkout. The command refuses to overwrite
config edits detected after generation. In CI, an update failure prevents the
commit step; a successful batch is published in one Git commit and a normal
push. A rejected push leaves the remote unchanged; a push publishes the
complete commit, never part of the batch.
## Review an update
@@ -90,16 +110,6 @@ Check mode does not modify the repository. It fails for
added or deleted files a
This command answers whether the repository matches its pin; it does not
determine whether the pin is the latest upstream commit. Use update mode for
that.
-## Synchronize configured snapshots
-
-Regenerate every snapshot directory from the repositories and commits
currently recorded in `config.toml`:
-
-```shell
-mise run tck -- snapshots sync
-```
-
-Sync may change the committed snapshot directories, but it never changes
`config.toml`.
-
## Set up the toolchain
Install [mise](https://mise.jdx.dev/), then install the pinned toolchain and
inspect the available snapshot commands:
@@ -111,7 +121,7 @@ mise run tck -- snapshots --help
Mise supplies Go, CMake and CTest, Java, and Maven. Git is required for every
source language, a C++ compiler is required for C++, and Make is required for
Go.
-Check mode accepts `cpp`, `go`, `java`, or `all`. Synchronization takes no
arguments. Update accepts one of `cpp`, `go`, or `java`, followed by a required
revision.
+Check accepts `cpp`, `go`, `java`, or `all` and is always read-only. Update
defaults to all languages at `HEAD`; specifying a language limits the update to
that source, and a second argument selects its revision. The former `snapshots
sync` command is replaced by `snapshots update <language> <recorded-commit>`.
## Use the corpus from an implementation
@@ -127,16 +137,16 @@ Some upstream generators contain randomness, so
byte-for-byte reproduction is no
Only modifications to existing probabilistic files are allowed in check mode.
Additions and deletions always block the check because they alter the
compatibility corpus, and modifications to deterministic files block because
they indicate either a compatibility change or a non-reproducible generator.
-The probabilistic classification only controls byte-level comparison in this
repository. Upstream generators remain responsible for constructing valid
sketches, and consumers remain responsible for algorithm-appropriate assertions.
+The stable/unstable classification controls check results and the optional
unstable-only update policy. It does not prove that an individual difference is
random or that an old snapshot is valid for a new source revision. Upstream
generators remain responsible for constructing valid sketches, and consumers
remain responsible for algorithm-appropriate assertions.
## GitHub Actions
`.github/workflows/check.yml` runs `mise run check` for pull requests and
pushes to `main`. It validates the Go implementation of the TCK tooling, but it
does not run the upstream snapshot generators or modify committed snapshots.
-`.github/workflows/update-snapshots.yml` runs `snapshots update <language>
HEAD` for `cpp`, `go`, and `java` every Monday and on manual dispatch, and
commits any updated pins and snapshots directly to `main`. It is not a required
check. The job summary lists each source's revision change and change counts.
If a source fails to generate, the other sources are still committed and the
run fails so that the broken upstream is noticed.
+`.github/workflows/update-snapshots.yml` runs `snapshots update
--skip-unstable-only` every Monday and on manual dispatch. It is not a required
check. The job summary lists each source's revision change, change counts, and
skipped updates. The workflow commits and pushes the batch only after the
command succeeds; any failed source prevents publication of the whole batch.
## Implementation notes
-For each requested language, the `tck` command reads the repository and commit
from `config.toml`, checks out that revision in a temporary workspace, and
invokes the source-specific adapter in `internal/snapshots/<language>.go`. It
then compares the generated output with
`serialization_test_data/<language>_generated_files`.
+For each requested language, the `tck` command reads the source repository and
existing pin from `config.toml`. Check generates at that pin; update resolves
the requested revision, defaulting to remote `HEAD`. Both invoke the
source-specific adapter in `internal/snapshots/<language>.go` in a temporary
workspace and compare its output with
`serialization_test_data/<language>_generated_files`. Update stages and
publishes the selected directories and revised config as one batch.
The command-line interface and change report live in `cmd/tck`. Reconciliation
and file comparison live in `internal/snapshots`, where `stability.go`
classifies deterministic and known probabilistic outputs.
diff --git a/internal/snapshots/config.go b/internal/snapshots/config.go
index 2746df4..78bb0bd 100644
--- a/internal/snapshots/config.go
+++ b/internal/snapshots/config.go
@@ -141,26 +141,3 @@ func encodeConfig(config repositoryConfig) ([]byte, error)
{
config.Snapshot.Java.Commit,
)), nil
}
-
-func replaceConfigFile(repositoryRoot string, content []byte) error {
- target := filepath.Join(repositoryRoot, configFilename)
- transaction, err := os.MkdirTemp(repositoryRoot, ".tck-config-")
- if err != nil {
- return fmt.Errorf("create config update transaction: %w", err)
- }
- defer func() { _ = os.RemoveAll(transaction) }()
-
- next := filepath.Join(transaction, "next")
- if err := os.WriteFile(next, content, 0o644); err != nil {
- return fmt.Errorf("stage TCK config: %w", err)
- }
- previous := filepath.Join(transaction, "previous")
- if err := os.Rename(target, previous); err != nil {
- return fmt.Errorf("preserve current TCK config: %w", err)
- }
- if err := os.Rename(next, target); err != nil {
- _ = os.Rename(previous, target)
- return fmt.Errorf("install TCK config: %w", err)
- }
- return nil
-}
diff --git a/internal/snapshots/diff_test.go b/internal/snapshots/diff_test.go
index be011c9..0530e76 100644
--- a/internal/snapshots/diff_test.go
+++ b/internal/snapshots/diff_test.go
@@ -19,6 +19,7 @@ package snapshots
import (
"crypto/sha256"
+ "io"
"os"
"path/filepath"
"testing"
@@ -83,7 +84,7 @@ func TestChangeIsBlocking(t *testing.T) {
}
}
-func TestReplaceDirectoryReplacesTheExactFileSet(t *testing.T) {
+func TestPublishReplacesTheExactFileSet(t *testing.T) {
t.Parallel()
root := t.TempDir()
@@ -94,7 +95,7 @@ func TestReplaceDirectoryReplacesTheExactFileSet(t
*testing.T) {
writeTestFile(t, filepath.Join(generated, "modified.sk"), "new")
writeTestFile(t, filepath.Join(generated, "nested", "added.sk"),
"added")
- require.NoError(t, replaceDirectory(target, generated))
+ require.NoError(t, publish(root, []replacement{{target: target, source:
generated}}, io.Discard))
requireTestFile(t, filepath.Join(target, "modified.sk"), "new")
requireTestFile(t, filepath.Join(target, "nested", "added.sk"), "added")
require.NoFileExists(t, filepath.Join(target, "deleted.sk"))
diff --git a/internal/snapshots/publish.go b/internal/snapshots/publish.go
new file mode 100644
index 0000000..70fad77
--- /dev/null
+++ b/internal/snapshots/publish.go
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package snapshots
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strconv"
+)
+
+type replacement struct {
+ target string
+ source string
+}
+
+// publish stages the complete batch on the destination filesystem, then keeps
+// backups until every replacement succeeds. Rollback failures preserve those
+// backups for recovery. Cleanup after success must never trigger a rollback.
+// This protects ordinary error paths, not concurrent readers or process
crashes.
+func publish(root string, replacements []replacement, stderr io.Writer) (err
error) {
+ if len(replacements) == 0 {
+ return nil
+ }
+ transaction, err := os.MkdirTemp(root, ".tck-update-")
+ if err != nil {
+ return fmt.Errorf("create update transaction: %w", err)
+ }
+ var undo []func() error
+ defer func() {
+ if err != nil {
+ var rollbackErr error
+ for index := len(undo) - 1; index >= 0; index-- {
+ rollbackErr = errors.Join(rollbackErr,
undo[index]())
+ }
+ if rollbackErr != nil {
+ err = fmt.Errorf("%w; rollback failed: %v;
recovery files remain at %s", err, rollbackErr, transaction)
+ return
+ }
+ }
+ if cleanupErr := os.RemoveAll(transaction); cleanupErr != nil {
+ _, _ = fmt.Fprintf(stderr, "Warning: could not remove
update backups at %s: %v\n", transaction, cleanupErr)
+ }
+ }()
+
+ for index, replacement := range replacements {
+ next := filepath.Join(transaction, "next-"+strconv.Itoa(index))
+ info, err := os.Stat(replacement.source)
+ if err != nil {
+ return err
+ }
+ if current, err := os.Lstat(replacement.target); err == nil {
+ if current.Mode()&os.ModeSymlink != 0 ||
current.IsDir() != info.IsDir() || (!current.IsDir() &&
!current.Mode().IsRegular()) {
+ return fmt.Errorf("cannot replace %s:
unexpected file type", replacement.target)
+ }
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ if info.IsDir() {
+ err = copyDirectory(replacement.source, next)
+ } else {
+ err = copyFile(replacement.source, next)
+ }
+ if err != nil {
+ return fmt.Errorf("stage %s: %w", replacement.target,
err)
+ }
+ }
+ for index, replacement := range replacements {
+ target := replacement.target
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return err
+ }
+ previous := filepath.Join(transaction,
"previous-"+strconv.Itoa(index))
+ if err := os.Rename(target, previous); err == nil {
+ undo = append(undo, func() error { return
os.Rename(previous, target) })
+ } else if !os.IsNotExist(err) {
+ return fmt.Errorf("preserve %s: %w", target, err)
+ }
+ next := filepath.Join(transaction, "next-"+strconv.Itoa(index))
+ if err := os.Rename(next, target); err != nil {
+ return fmt.Errorf("install %s: %w", target, err)
+ }
+ undo = append(undo, func() error { return os.RemoveAll(target)
})
+ }
+ return nil
+}
+
+func copyDirectory(source, destination string) error {
+ files, err := regularFiles(source)
+ if err != nil {
+ return err
+ }
+ if err := os.MkdirAll(destination, 0o755); err != nil {
+ return err
+ }
+ for relative, file := range files {
+ output := filepath.Join(destination,
filepath.FromSlash(relative))
+ if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil {
+ return err
+ }
+ if err := copyFile(file.filename, output); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/internal/snapshots/publish_test.go
b/internal/snapshots/publish_test.go
new file mode 100644
index 0000000..feaee0b
--- /dev/null
+++ b/internal/snapshots/publish_test.go
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package snapshots
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestPublishStagingFailureDoesNotChangeTargets(t *testing.T) {
+ root := t.TempDir()
+ target, next := filepath.Join(root, "config.toml"), filepath.Join(root,
"next")
+ writeTestFile(t, target, "old")
+ writeTestFile(t, next, "new")
+ err := publish(root, []replacement{
+ {target: target, source: next},
+ {target: filepath.Join(root, "snapshots"), source:
filepath.Join(root, "missing")},
+ }, io.Discard)
+ require.Error(t, err)
+ requireTestFile(t, target, "old")
+ require.NoDirExists(t, filepath.Join(root, "snapshots"))
+}
+
+func TestPublishRollsBackEarlierFilesAndDirectories(t *testing.T) {
+ root := t.TempDir()
+ nextFile, nextDir := filepath.Join(root, "next-config"),
filepath.Join(root, "next-snapshots")
+ writeTestFile(t, nextFile, "new config")
+ writeTestFile(t, filepath.Join(nextDir, "new.sk"), "new")
+ config, snapshots := filepath.Join(root, "config.toml"),
filepath.Join(root, "snapshots")
+ absent, locked := filepath.Join(root, "absent"), filepath.Join(root,
"locked", "target")
+ writeTestFile(t, config, "old config")
+ writeTestFile(t, filepath.Join(snapshots, "old.sk"), "old")
+ writeTestFile(t, locked, "old locked file")
+ protectDirectory(t, filepath.Dir(locked))
+ err := publish(root, []replacement{
+ {target: config, source: nextFile},
+ {target: snapshots, source: nextDir},
+ {target: absent, source: nextDir},
+ {target: locked, source: nextFile},
+ }, io.Discard)
+ require.ErrorContains(t, err, "preserve")
+ requireTestFile(t, config, "old config")
+ requireTestFile(t, filepath.Join(snapshots, "old.sk"), "old")
+ require.NoFileExists(t, filepath.Join(snapshots, "new.sk"))
+ require.NoDirExists(t, absent)
+ requireTestFile(t, locked, "old locked file")
+}
+
+func TestPublishCleanupFailureKeepsNewConfigAndSnapshots(t *testing.T) {
+ root := t.TempDir()
+ target, config := filepath.Join(root, "snapshots"), filepath.Join(root,
"config.toml")
+ nextDir, nextFile := filepath.Join(root, "next-snapshots"),
filepath.Join(root, "next-config")
+ writeTestFile(t, filepath.Join(target, "protected", "old.sk"), "old")
+ writeTestFile(t, config, "old revision")
+ writeTestFile(t, filepath.Join(nextDir, "new.sk"), "new")
+ writeTestFile(t, nextFile, "new revision")
+ protectDirectory(t, filepath.Join(target, "protected"))
+ // The protected directory moves into the transaction backup.
+ t.Cleanup(func() {
+ _ = filepath.Walk(root, func(path string, info os.FileInfo, err
error) error {
+ if err == nil && info.IsDir() {
+ _ = os.Chmod(path, 0o755)
+ }
+ return nil
+ })
+ })
+ var warnings bytes.Buffer
+ require.NoError(t, publish(root, []replacement{
+ {target: target, source: nextDir},
+ {target: config, source: nextFile},
+ }, &warnings))
+ require.Contains(t, warnings.String(), "Warning: could not remove
update backups")
+ requireTestFile(t, config, "new revision")
+ requireTestFile(t, filepath.Join(target, "new.sk"), "new")
+ require.NoDirExists(t, filepath.Join(target, "protected"))
+}
+
+func protectDirectory(t *testing.T, directory string) {
+ t.Helper()
+ if runtime.GOOS == "windows" || os.Geteuid() == 0 {
+ t.Skip("requires Unix permission enforcement")
+ }
+ require.NoError(t, os.Chmod(directory, 0o555))
+ t.Cleanup(func() { _ = os.Chmod(directory, 0o755) })
+}
diff --git a/internal/snapshots/reconcile.go b/internal/snapshots/reconcile.go
index 92b7316..4de015c 100644
--- a/internal/snapshots/reconcile.go
+++ b/internal/snapshots/reconcile.go
@@ -18,8 +18,8 @@
package snapshots
import (
+ "bytes"
"context"
- "errors"
"fmt"
"io"
"os"
@@ -27,21 +27,13 @@ import (
"path/filepath"
)
-type Mode string
-
-const (
- ModeCheck Mode = "check"
- ModeSync Mode = "sync"
- ModeUpdate Mode = "update"
-)
-
+// Result describes the generated revision and its differences from the
existing
+// snapshot directory, including differences skipped by the update policy.
type Result struct {
Target string
PreviousRevision string
Revision string
Changes []Change
- // Written reports whether the snapshot directory was replaced.
- Written bool
}
func (result Result) RevisionChanged() bool {
@@ -62,214 +54,140 @@ func (result Result) BlockingChangeCount() int {
return count
}
-func Reconcile(
- ctx context.Context,
- repositoryRoot, language string,
- mode Mode,
- requestedRevision string,
- stdout, stderr io.Writer,
-) (Result, error) {
- if mode != ModeCheck && mode != ModeSync && mode != ModeUpdate {
- return Result{}, fmt.Errorf("unsupported reconciliation mode
%q", mode)
- }
- generator, found := generators[language]
- if !found {
- return Result{}, fmt.Errorf("unsupported snapshot language %q",
language)
- }
- if mode == ModeUpdate && requestedRevision == "" {
- return Result{}, fmt.Errorf("a source revision is required in
update mode")
- }
- if mode != ModeUpdate && requestedRevision != "" {
- return Result{}, fmt.Errorf("a source revision can only be
selected in update mode")
- }
- config, originalConfig, err := loadConfig(repositoryRoot)
+// UnstableOnly reports whether the only differences are content changes to
+// existing unstable snapshots. A revision change always requires an update.
+func (result Result) UnstableOnly() bool {
+ return !result.RevisionChanged() && len(result.Changes) > 0 &&
!result.HasBlockingChanges()
+}
+
+// Check regenerates a source at its configured commit without writing
repository
+// files. Snapshot differences are returned in Result rather than as errors.
+func Check(ctx context.Context, root, language string, stdout, stderr
io.Writer) (Result, error) {
+ config, _, err := loadConfig(root)
if err != nil {
return Result{}, err
}
source, found := config.source(language)
if !found {
- return Result{}, fmt.Errorf("snapshot source for unsupported
language %q", language)
- }
- sourceRevision := source.Commit
- if requestedRevision != "" {
- sourceRevision = requestedRevision
- }
- for _, requirement := range generator.requirements {
- if _, err := exec.LookPath(requirement); err != nil {
- return Result{}, fmt.Errorf("required command %q is not
installed or not on PATH", requirement)
- }
+ return Result{}, fmt.Errorf("unsupported snapshot language %q",
language)
}
-
- workspace, err := os.MkdirTemp("", "datasketches-tck-"+language+"-")
+ workspace, err := os.MkdirTemp("", "datasketches-tck-check-")
if err != nil {
- return Result{}, fmt.Errorf("create generator workspace: %w",
err)
+ return Result{}, err
}
defer func() { _ = os.RemoveAll(workspace) }()
+ return generate(ctx, root, workspace, language, source, source.Commit,
commandRunner{stdout: stdout, stderr: stderr})
+}
- generated := filepath.Join(workspace, "generated")
- runner := commandRunner{stdout: stdout, stderr: stderr}
- resolvedRevision, err := generator.run(
- ctx,
- workspace,
- generated,
- source.Repository,
- sourceRevision,
- runner,
- )
+// Update generates every selected source before publishing any changes. Empty
+// language selects all sources; empty revision selects each remote's HEAD.
+// With skipUnstableOnly, unchanged revisions with only unstable content
changes
+// retain their entire snapshot directory. Other sources are updated in full.
+func Update(ctx context.Context, root, language, revision string,
skipUnstableOnly bool, stdout, stderr io.Writer) ([]Result, error) {
+ languages := Languages()
+ if language != "" {
+ languages = []string{language}
+ }
+ if revision == "" {
+ revision = "HEAD"
+ }
+ config, originalConfig, err := loadConfig(root)
if err != nil {
- return Result{}, fmt.Errorf("generate %s snapshots: %w",
language, err)
+ return nil, err
}
-
- target := snapshotDirectory(repositoryRoot, language)
- changes, err := compareDirectories(target, generated,
generator.stability)
+ workspace, err := os.MkdirTemp("", "datasketches-tck-update-")
if err != nil {
- return Result{}, err
+ return nil, err
}
+ defer func() { _ = os.RemoveAll(workspace) }()
- result := Result{
- Target: target,
- PreviousRevision: source.Commit,
- Revision: resolvedRevision,
- Changes: changes,
- }
- if mode == ModeCheck {
- return result, nil
- }
- if mode == ModeSync {
- if len(changes) > 0 {
- if err := replaceDirectory(target, generated); err !=
nil {
- return Result{}, err
+ var results []Result
+ var replacements []replacement
+ revisionChanged := false
+ for _, language := range languages {
+ source, found := config.source(language)
+ if !found {
+ return nil, fmt.Errorf("unsupported snapshot language
%q", language)
+ }
+ directory := filepath.Join(workspace, language)
+ result, err := generate(ctx, root, directory, language, source,
revision, commandRunner{stdout: stdout, stderr: stderr})
+ if err != nil {
+ return nil, fmt.Errorf("update aborted before
publishing: %w", err)
+ }
+ results = append(results, result)
+ if skipUnstableOnly && result.UnstableOnly() {
+ continue
+ }
+ if result.RevisionChanged() {
+ if err := config.setCommit(language, result.Revision);
err != nil {
+ return nil, err
}
- result.Written = true
+ revisionChanged = true
+ }
+ if len(result.Changes) > 0 {
+ replacements = append(replacements, replacement{
+ target: result.Target,
+ source: filepath.Join(directory, "generated"),
+ })
}
- return result, nil
- }
-
- revisionChanged := result.RevisionChanged()
- if !revisionChanged && !result.HasBlockingChanges() {
- // Nothing new upstream. Rewriting the probabilistic snapshots
would only
- // add binary churn to the history without adopting anything.
- return result, nil
}
if revisionChanged {
- if err := config.setCommit(language, resolvedRevision); err !=
nil {
- return Result{}, err
- }
- updatedConfig, err := encodeConfig(config)
+ content, err := encodeConfig(config)
if err != nil {
- return Result{}, err
+ return nil, err
}
- if err := replaceConfigFile(repositoryRoot, updatedConfig); err
!= nil {
- return Result{}, err
+ nextConfig := filepath.Join(workspace, configFilename)
+ if err := os.WriteFile(nextConfig, content, 0o644); err != nil {
+ return nil, err
}
+ replacements = append(replacements, replacement{target:
filepath.Join(root, configFilename), source: nextConfig})
}
- if len(changes) > 0 {
- if err := replaceDirectory(target, generated); err != nil {
- if !revisionChanged {
- return Result{}, err
- }
- if restoreErr := replaceConfigFile(repositoryRoot,
originalConfig); restoreErr != nil {
- return Result{}, fmt.Errorf(
- "%w; restoring the previous config also
failed: %v",
- err,
- restoreErr,
- )
- }
- return Result{}, err
- }
- result.Written = true
- }
- return result, nil
-}
-
-// snapshotDirectory returns where a language's snapshots are committed. The
-// serialization_test_data/<language>_generated_files layout matches the
-// DataSketches implementation repositories, so a TCK checkout can be used
-// directly as their test data root.
-func snapshotDirectory(repositoryRoot, language string) string {
- return filepath.Join(repositoryRoot, "serialization_test_data",
language+"_generated_files")
-}
-
-func replaceDirectory(target, generated string) error {
- parent := filepath.Dir(target)
- if err := os.MkdirAll(parent, 0o755); err != nil {
- return fmt.Errorf("create snapshot parent directory: %w", err)
+ if err := ctx.Err(); err != nil {
+ return nil, err
}
-
- transaction, err := os.MkdirTemp(parent,
"."+filepath.Base(target)+"-tck-")
+ // A long-running generation must not overwrite a pin edited in the
meantime.
+ currentConfig, err := os.ReadFile(filepath.Join(root, configFilename))
if err != nil {
- return fmt.Errorf("create snapshot update transaction: %w", err)
+ return nil, err
}
- removeTransaction := true
- defer func() {
- if removeTransaction {
- _ = os.RemoveAll(transaction)
- }
- }()
-
- next := filepath.Join(transaction, "next")
- if err := copyDirectory(generated, next); err != nil {
- return fmt.Errorf("stage generated snapshots: %w", err)
+ if !bytes.Equal(currentConfig, originalConfig) {
+ return nil, fmt.Errorf("config.toml changed during generation;
no updates were published")
}
-
- previous := filepath.Join(transaction, "previous")
- hadTarget := false
- info, err := os.Lstat(target)
- switch {
- case err == nil:
- if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
- return fmt.Errorf("replace snapshots: %s is not a
directory", target)
- }
- hadTarget = true
- if err := os.Rename(target, previous); err != nil {
- return fmt.Errorf("preserve current snapshots: %w", err)
- }
- case os.IsNotExist(err):
- case err != nil:
- return fmt.Errorf("inspect snapshots before update: %w", err)
+ if err := publish(root, replacements, stderr); err != nil {
+ return nil, err
}
+ return results, nil
+}
- if err := os.Rename(next, target); err != nil {
- if !hadTarget {
- return fmt.Errorf("install generated snapshots: %w",
err)
- }
- if restoreErr := os.Rename(previous, target); restoreErr == nil
{
- return fmt.Errorf("install generated snapshots: %w",
err)
- } else {
- removeTransaction = false
- return fmt.Errorf(
- "install generated snapshots: %w; restoring
previous snapshots also failed: %v; previous files remain at %s",
- err,
- restoreErr,
- previous,
- )
- }
+func generate(ctx context.Context, root, workspace, language string, source
snapshotSource, revision string, runner commandRunner) (Result, error) {
+ definition, found := generators[language]
+ if !found {
+ return Result{}, fmt.Errorf("unsupported snapshot language %q",
language)
}
-
- if hadTarget {
- if err := os.RemoveAll(previous); err != nil && !errors.Is(err,
os.ErrNotExist) {
- return fmt.Errorf("remove previous snapshots after
update: %w", err)
+ for _, requirement := range definition.requirements {
+ if _, err := exec.LookPath(requirement); err != nil {
+ return Result{}, fmt.Errorf("required command %q is not
installed or not on PATH", requirement)
}
}
- return nil
-}
-
-func copyDirectory(source, destination string) error {
- files, err := regularFiles(source)
- if err != nil {
- return err
+ if err := os.MkdirAll(workspace, 0o755); err != nil {
+ return Result{}, err
}
- if err := os.MkdirAll(destination, 0o755); err != nil {
- return err
+ if _, err := fmt.Fprintf(runner.stdout, "Generate %s snapshots\n",
language); err != nil {
+ return Result{}, err
}
- for relative, file := range files {
- output := filepath.Join(destination,
filepath.FromSlash(relative))
- if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil {
- return err
- }
- if err := copyFile(file.filename, output); err != nil {
- return err
- }
+ generated := filepath.Join(workspace, "generated")
+ resolvedRevision, err := definition.run(ctx, workspace, generated,
source.Repository, revision, runner)
+ if err != nil {
+ return Result{}, fmt.Errorf("generate %s snapshots: %w",
language, err)
}
- return nil
+ target := snapshotDirectory(root, language)
+ changes, err := compareDirectories(target, generated,
definition.stability)
+ if err != nil {
+ return Result{}, err
+ }
+ return Result{Target: target, PreviousRevision: source.Commit,
Revision: resolvedRevision, Changes: changes}, nil
+}
+
+func snapshotDirectory(root, language string) string {
+ return filepath.Join(root, "serialization_test_data",
language+"_generated_files")
}
diff --git a/internal/snapshots/reconcile_test.go
b/internal/snapshots/reconcile_test.go
new file mode 100644
index 0000000..17f4f2b
--- /dev/null
+++ b/internal/snapshots/reconcile_test.go
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package snapshots
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestUpdateGeneratesEntireBatchBeforePublishing(t *testing.T) {
+ root, oldRevision, newRevision := updateFixture(t)
+ calls := 0
+ for language, definition := range generators {
+ definition.generate = func(_ context.Context, paths
generationPaths, _ commandRunner) error {
+ config, _, err := loadConfig(root)
+ require.NoError(t, err)
+ for _, selected := range Languages() {
+ source, _ := config.source(selected)
+ require.Equal(t, oldRevision, source.Commit)
+ requireTestFile(t,
filepath.Join(snapshotDirectory(root, selected), "stable.sk"), "old")
+ }
+ calls++
+ writeTestFile(t, filepath.Join(paths.destination,
"stable.sk"), "new")
+ return nil
+ }
+ generators[language] = definition
+ }
+ results, err := Update(t.Context(), root, "", "", false, io.Discard,
io.Discard)
+ require.NoError(t, err)
+ require.Len(t, results, len(Languages()))
+ require.Equal(t, len(Languages()), calls)
+ config, _, err := loadConfig(root)
+ require.NoError(t, err)
+ for _, language := range Languages() {
+ source, _ := config.source(language)
+ require.Equal(t, newRevision, source.Commit)
+ requireTestFile(t, filepath.Join(snapshotDirectory(root,
language), "stable.sk"), "new")
+ }
+}
+
+func TestUpdateGenerationFailureLeavesEntireBatchUntouched(t *testing.T) {
+ root, oldRevision, _ := updateFixture(t)
+ _, before, err := loadConfig(root)
+ require.NoError(t, err)
+ definition := generators["java"]
+ definition.generate = func(context.Context, generationPaths,
commandRunner) error {
+ return errors.New("upstream generation failed")
+ }
+ generators["java"] = definition
+ results, err := Update(t.Context(), root, "", "", false, io.Discard,
io.Discard)
+ require.ErrorContains(t, err, "update aborted before publishing")
+ require.Nil(t, results)
+ config, after, err := loadConfig(root)
+ require.NoError(t, err)
+ require.Equal(t, before, after)
+ for _, language := range Languages() {
+ source, _ := config.source(language)
+ require.Equal(t, oldRevision, source.Commit)
+ requireTestFile(t, filepath.Join(snapshotDirectory(root,
language), "stable.sk"), "old")
+ }
+}
+
+func TestUpdateSkipsUnstableSourceWithinBatch(t *testing.T) {
+ root, _, newRevision := updateFixture(t)
+ config, _, err := loadConfig(root)
+ require.NoError(t, err)
+ require.NoError(t, config.setCommit("go", newRevision))
+ content, err := encodeConfig(config)
+ require.NoError(t, err)
+ writeTestFile(t, filepath.Join(root, configFilename), string(content))
+ target := snapshotDirectory(root, "go")
+ writeTestFile(t, filepath.Join(target, "stable.sk"), "new")
+ writeTestFile(t, filepath.Join(target, "unstable.sk"), "old")
+ definition := generators["go"]
+ generate := definition.generate
+ definition.generate = func(ctx context.Context, paths generationPaths,
runner commandRunner) error {
+ if err := generate(ctx, paths, runner); err != nil {
+ return err
+ }
+ writeTestFile(t, filepath.Join(paths.destination,
"unstable.sk"), "new")
+ return nil
+ }
+ definition.stability = func(path string) Stability {
+ if path == "unstable.sk" {
+ return Unstable
+ }
+ return Stable
+ }
+ generators["go"] = definition
+ _, err = Update(t.Context(), root, "", "", true, io.Discard, io.Discard)
+ require.NoError(t, err)
+ config, _, err = loadConfig(root)
+ require.NoError(t, err)
+ for _, language := range Languages() {
+ source, _ := config.source(language)
+ require.Equal(t, newRevision, source.Commit)
+ requireTestFile(t, filepath.Join(snapshotDirectory(root,
language), "stable.sk"), "new")
+ }
+ requireTestFile(t, filepath.Join(target, "unstable.sk"), "old")
+}
+
+func TestUpdateUnstableOnlyPolicy(t *testing.T) {
+ tests := []struct {
+ name string
+ skip bool
+ advance bool
+ generated map[string]string
+ keepOld bool
+ }{
+ {name: "default refreshes unstable contents", generated:
map[string]string{"stable.sk": "old", "unstable.sk": "new"}},
+ {name: "skip retains unchanged source", skip: true, generated:
map[string]string{"stable.sk": "old", "unstable.sk": "new"}, keepOld: true},
+ {name: "new revision includes unstable contents", skip: true,
advance: true, generated: map[string]string{"stable.sk": "old", "unstable.sk":
"new"}},
+ {name: "stable change includes unstable contents", skip: true,
generated: map[string]string{"stable.sk": "new", "unstable.sk": "new"}},
+ {name: "unstable addition is adopted", skip: true, generated:
map[string]string{"stable.sk": "old", "unstable.sk": "new", "added.sk": "new"}},
+ {name: "unstable deletion is adopted", skip: true, generated:
map[string]string{"stable.sk": "old"}},
+ {name: "revision only", skip: true, advance: true, generated:
map[string]string{"stable.sk": "old", "unstable.sk": "old"}},
+ {name: "no changes", skip: true, generated:
map[string]string{"stable.sk": "old", "unstable.sk": "old"}},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ root, oldRevision, newRevision := updateFixture(t)
+ target := snapshotDirectory(root, "go")
+ writeTestFile(t, filepath.Join(target, "unstable.sk"),
"old")
+ definition := generators["go"]
+ definition.generate = func(_ context.Context, paths
generationPaths, _ commandRunner) error {
+ for name, content := range test.generated {
+ writeTestFile(t,
filepath.Join(paths.destination, name), content)
+ }
+ return nil
+ }
+ definition.stability = func(path string) Stability {
+ if path == "stable.sk" {
+ return Stable
+ }
+ return Unstable
+ }
+ generators["go"] = definition
+ revision := oldRevision
+ if test.advance {
+ revision = newRevision
+ }
+ results, err := Update(t.Context(), root, "go",
revision, test.skip, io.Discard, io.Discard)
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ config, _, err := loadConfig(root)
+ require.NoError(t, err)
+ require.Equal(t, revision, config.Snapshot.Go.Commit)
+ require.Equal(t, oldRevision,
config.Snapshot.CPP.Commit)
+ require.Equal(t, oldRevision,
config.Snapshot.Java.Commit)
+ expected := test.generated
+ if test.keepOld {
+ expected = map[string]string{"stable.sk":
"old", "unstable.sk": "old"}
+ }
+ entries, err := os.ReadDir(target)
+ require.NoError(t, err)
+ require.Len(t, entries, len(expected))
+ for name, content := range expected {
+ requireTestFile(t, filepath.Join(target, name),
content)
+ }
+ })
+ }
+}
+
+func TestCheckUsesPinnedRevisionWithoutWriting(t *testing.T) {
+ root, oldRevision, _ := updateFixture(t)
+ _, before, err := loadConfig(root)
+ require.NoError(t, err)
+ result, err := Check(t.Context(), root, "go", io.Discard, io.Discard)
+ require.NoError(t, err)
+ require.Equal(t, oldRevision, result.Revision)
+ require.Empty(t, result.Changes)
+ _, after, err := loadConfig(root)
+ require.NoError(t, err)
+ require.Equal(t, before, after)
+ requireTestFile(t, filepath.Join(snapshotDirectory(root, "go"),
"stable.sk"), "old")
+}
+
+func TestUpdatePreservesConcurrentConfigEdit(t *testing.T) {
+ root, _, _ := updateFixture(t)
+ _, before, err := loadConfig(root)
+ require.NoError(t, err)
+ edited := append(bytes.Clone(before), []byte("\n# Maintainer
edit\n")...)
+ definition := generators["go"]
+ generate := definition.generate
+ definition.generate = func(ctx context.Context, paths generationPaths,
runner commandRunner) error {
+ require.NoError(t, os.WriteFile(filepath.Join(root,
configFilename), edited, 0o644))
+ return generate(ctx, paths, runner)
+ }
+ generators["go"] = definition
+ _, err = Update(t.Context(), root, "go", "", false, io.Discard,
io.Discard)
+ require.ErrorContains(t, err, "config.toml changed during generation")
+ after, err := os.ReadFile(filepath.Join(root, configFilename))
+ require.NoError(t, err)
+ require.Equal(t, edited, after)
+ requireTestFile(t, filepath.Join(snapshotDirectory(root, "go"),
"stable.sk"), "old")
+}
+
+func updateFixture(t *testing.T) (string, string, string) {
+ t.Helper()
+ source := t.TempDir()
+ git := func(args ...string) string {
+ t.Helper()
+ args = append([]string{"-C", source, "-c", "user.name=TCK
test", "-c", "[email protected]", "-c", "commit.gpgsign=false"},
args...)
+ output, err := exec.Command("git", args...).CombinedOutput()
+ require.NoError(t, err, "%s", output)
+ return strings.TrimSpace(string(output))
+ }
+ git("init", "--quiet")
+ writeTestFile(t, filepath.Join(source, "stable.sk"), "old")
+ git("add", "--force", "stable.sk")
+ git("commit", "--quiet", "-m", "Initial snapshot")
+ oldRevision := git("rev-parse", "HEAD")
+ writeTestFile(t, filepath.Join(source, "stable.sk"), "new")
+ git("commit", "--quiet", "-am", "Update snapshot")
+ newRevision := git("rev-parse", "HEAD")
+
+ original := generators
+ t.Cleanup(func() { generators = original })
+ generators = make(map[string]generator)
+ for language := range original {
+ generators[language] = generator{
+ requirements: []string{"git"},
+ generate: func(_ context.Context, paths
generationPaths, _ commandRunner) error {
+ if err := os.MkdirAll(paths.destination,
0o755); err != nil {
+ return err
+ }
+ return copyFile(filepath.Join(paths.source,
"stable.sk"), filepath.Join(paths.destination, "stable.sk"))
+ },
+ stability: func(string) Stability { return Stable },
+ }
+ }
+ root := t.TempDir()
+ pin := snapshotSource{Repository: source, Commit: oldRevision}
+ content, err := encodeConfig(repositoryConfig{Snapshot:
snapshotSources{CPP: pin, Go: pin, Java: pin}})
+ require.NoError(t, err)
+ writeTestFile(t, filepath.Join(root, configFilename), string(content))
+ for _, language := range Languages() {
+ writeTestFile(t, filepath.Join(snapshotDirectory(root,
language), "stable.sk"), "old")
+ }
+ return root, oldRevision, newRevision
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
