This is an automated email from the ASF dual-hosted git repository. smiklosovic pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit b4e9399865eb51077f51d34c4c383b8418a9e6ed Author: Stefan Miklosovic <[email protected]> AuthorDate: Wed Dec 18 14:25:29 2024 +0100 Create manifest upon loading where it does not exist or enrich it When snapshot manifest does not exist, no snapshots would be found, because even they would be loaded, since CASSANDRA-18111, snapshot's presence is logically determined by the existence of at least one manifest.json file in any data dir. That would result in snapshots not being shown e.g. in the output of nodetool listsnapshots. The fix consists of creating a manifest in each snapshot directory when not present. When a manifest.json exists, its older format has not contained created_at field (pre-16789) When this is detected, we will proceed to enrich such manifest by overwriting it with a manifest of new format. patch by Stefan Miklosovic; reviewed by Jordan West, Cheng Wang, Bernardo Botella for CASSANDRA-20150 --- CHANGES.txt | 1 + .../config/CassandraRelevantProperties.java | 6 + .../cassandra/service/snapshot/TableSnapshot.java | 146 +++++++++++++++++- .../cassandra/distributed/test/SnapshotsTest.java | 167 ++++++++++++++++++++- .../org/apache/cassandra/db/DirectoriesTest.java | 10 +- .../service/snapshot/SnapshotLoaderTest.java | 57 +++++-- 6 files changed, 364 insertions(+), 23 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b20e0f8b1f..ee4e28789e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Create manifest upon loading where it does not exist or enrich it (CASSANDRA-20150) * Propagate true size of snapshot in SnapshotDetailsTabularData to not call JMX twice in nodetool listsnapshots (CASSANDRA-20149) * Implementation of CEP-43 (CASSANDRA-19964) * Periodically disconnect roles that are revoked or have LOGIN=FALSE set (CASSANDRA-19385) diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java index 8e5b0c6dd9..70a6ea702e 100644 --- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java +++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java @@ -502,6 +502,12 @@ public enum CassandraRelevantProperties SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS("cassandra.snapshot.ttl_cleanup_initial_delay_seconds", "5"), /** snapshots ttl cleanup period in seconds */ SNAPSHOT_CLEANUP_PERIOD_SECONDS("cassandra.snapshot.ttl_cleanup_period_seconds", "60"), + /** + * When there is a snapshot with old / basic format (basically pre-CASSANDRA-16789), + * it will enrich it with more metadata upon snapshot's loading at startup. + * Defaults to true, when set to false, no enriching will be done. + * */ + SNAPSHOT_MANIFEST_ENRICH_OR_CREATE_ENABLED("cassandra.snapshot.enrich.or.create.enabled", "true"), /** minimum allowed TTL for snapshots */ SNAPSHOT_MIN_ALLOWED_TTL_SECONDS("cassandra.snapshot.min_allowed_ttl_seconds", "60"), SSL_ENABLE("ssl.enable"), diff --git a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java index 549dbb6db6..7698ea1e3d 100644 --- a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java +++ b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java @@ -19,10 +19,13 @@ package org.apache.cassandra.service.snapshot; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,6 +38,7 @@ import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; @@ -42,10 +46,13 @@ import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableId; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.LocalizeString; import org.apache.cassandra.utils.Throwables; @@ -262,7 +269,8 @@ public class TableSnapshot } catch (Throwable t) { - logger.error("Could not list directory content {}", dir, t); + if (!(t instanceof NoSuchFileException)) + logger.error("Could not list directory content {}", dir, t); } return paths; @@ -490,8 +498,144 @@ public class TableSnapshot TableSnapshot build() { + maybeCreateOrEnrichManifest(); return new TableSnapshot(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral); } + + private List<File> getSnapshotDirsWithoutManifest() + { + List<File> snapshotDirNotHavingManifests = new ArrayList<>(); + for (File snapshotDir : snapshotDirs) + { + if (!new File(snapshotDir.toPath().resolve("manifest.json")).exists()) + snapshotDirNotHavingManifests.add(snapshotDir); + } + + return snapshotDirNotHavingManifests; + } + + private void maybeCreateOrEnrichManifest() + { + if (!CassandraRelevantProperties.SNAPSHOT_MANIFEST_ENRICH_OR_CREATE_ENABLED.getBoolean()) + return; + + // this is caused by not reading any manifest or that snapshot had a basic manifest just with sstables + // enumerated (pre CASSANDRA-16789), so we just go ahead and enrich it in each snapshot dir + + List<File> snapshotDirsWithoutManifests = getSnapshotDirsWithoutManifest(); + if (createdAt != null && snapshotDirsWithoutManifests.isEmpty()) + return; + + if (createdAt == null && snapshotDirsWithoutManifests.isEmpty()) + logger.info("Manifest in the old format for snapshot {} for {}.{} was detected, going to enrich it.", + tag, keyspaceName, tableName); + + if (!snapshotDirsWithoutManifests.isEmpty()) + logger.info("There is no manifest for snapshot {} for {}.{} at paths {}, going to create it.", + tag, keyspaceName, tableName, snapshotDirsWithoutManifests); + + long lastModified = createdAt == null ? -1 : createdAt.toEpochMilli(); + + if (lastModified == -1) + { + for (File snapshotDir : snapshotDirs) + { + // we will consider time of the creation the oldest last modified + // timestamp on any snapshot directory + long currentLastModified = snapshotDir.lastModified(); + if ((currentLastModified < lastModified || lastModified == -1) && currentLastModified > 0) + lastModified = currentLastModified; + } + } + + List<String> allDataFiles = new ArrayList<>(); + for (File snapshotDir : snapshotDirs) + { + List<File> dataFiles = new ArrayList<>(); + try + { + List<File> indicesDirs = new ArrayList<>(); + File[] snapshotFiles = snapshotDir.list(file -> { + if (file.isDirectory() && file.name().startsWith(".")) + { + indicesDirs.add(file); + return false; + } + else + { + return file.name().endsWith('-' + SSTableFormat.Components.DATA.type.repr); + } + }); + + Collections.addAll(dataFiles, snapshotFiles); + + for (File indexDir : indicesDirs) + dataFiles.addAll(Arrays.asList(indexDir.list(file -> file.name().endsWith('-' + SSTableFormat.Components.DATA.type.repr)))); + } + catch (IOException ex) + { + logger.error("Unable to list a directory for data components: {}", snapshotDir); + } + + for (File dataFile : dataFiles) + { + Descriptor descriptor = SSTable.tryDescriptorFromFile(dataFile); + if (descriptor != null) + { + String relativeDataFileName = descriptor.relativeFilenameFor(SSTableFormat.Components.DATA); + allDataFiles.add(relativeDataFileName); + } + } + } + + // in any case of not being able to resolve it, set it to current time + if (lastModified < 0) + createdAt = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()); + else + createdAt = Instant.ofEpochMilli(lastModified); + + for (File snapshotDir : snapshotDirs) + { + SnapshotManifest snapshotManifest = new SnapshotManifest(allDataFiles, null, createdAt, ephemeral); + File manifestFile = new File(snapshotDir, "manifest.json"); + if (snapshotDirsWithoutManifests.contains(snapshotDir)) + { + writeManifest(snapshotManifest, manifestFile); + } + else + { + SnapshotManifest existingManifest = readManifest(manifestFile); + if (existingManifest != null && existingManifest.createdAt == null) + writeManifest(snapshotManifest, manifestFile); + } + } + } + + private SnapshotManifest readManifest(File manifestFile) + { + try + { + return SnapshotManifest.deserializeFromJsonFile(manifestFile); + } + catch (IOException ex) + { + logger.error("Unable to read a manifest.json file in {}", manifestFile.absolutePath()); + } + + return null; + } + + private void writeManifest(SnapshotManifest snapshotManifest, File manifestFile) + { + try + { + snapshotManifest.serializeToJsonFile(manifestFile); + } + catch (IOException ex) + { + logger.error("Unable to create a manifest.json file in {}", manifestFile.absolutePath()); + } + } } protected static String buildSnapshotId(String keyspaceName, String tableName, UUID tableId, String tag) diff --git a/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java b/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java index 2c57654695..f04a3403ca 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java @@ -33,32 +33,40 @@ import org.junit.Test; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableCallable; import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.distributed.shared.WithProperties; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.FBUtilities; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; +import static org.apache.cassandra.config.CassandraRelevantProperties.SNAPSHOT_MANIFEST_ENRICH_OR_CREATE_ENABLED; import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.apache.cassandra.schema.SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES; +import static org.apache.cassandra.schema.SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeThat; import static oshi.PlatformEnum.MACOS; public class SnapshotsTest extends TestBaseImpl { - public static final Integer SNAPSHOT_CLEANUP_PERIOD_SECONDS = 1; + public static final Integer SNAPSHOT_CLEANUP_PERIOD_SECONDS = 2; public static final Integer FIVE_SECONDS = 5; public static final Integer TEN_SECONDS = 10; private static final WithProperties properties = new WithProperties(); @@ -130,10 +138,117 @@ public class SnapshotsTest extends TestBaseImpl } } + @Test + public void testLocalOrReplicatedSystemTablesSnapshotsDoNotHaveSchema() + { + cluster.get(1) + .nodetoolResult("snapshot", "-t", "snapshot_with_local_or_replicated") + .asserts() + .success(); + + String[] dataDirs = (String[]) cluster.get(1).config().get("data_file_directories"); + String[] paths = getSnapshotPaths(true); + + for (String dataDir : dataDirs) + { + for (String path : paths) + { + Path snapshotDir = Paths.get(dataDir) + .resolve(path) + .resolve("snapshots") + .resolve("snapshot_with_local_or_replicated"); + + if (snapshotDir.toFile().exists()) + assertFalse(new File(snapshotDir, "schema.cql").exists()); + } + } + } + + @Test + public void testMissingManifestIsCreatedOnStartupWithEnrichmentEnabled() + { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))")); + populate(cluster); + + cluster.get(1) + .nodetoolResult("snapshot", "-t", "snapshot_with_local_or_replicated") + .asserts() + .success(); + + String[] dataDirs = (String[]) cluster.get(1).config().get("data_file_directories"); + String[] paths = getSnapshotPaths(false); + + assertManifestsPresence(dataDirs, paths, true); + + // remove all manifest files + removeAllManifests(dataDirs, paths); + assertManifestsPresence(dataDirs, paths, false); + + // restart which should add them back + cluster.get(1).shutdown(true); + cluster.get(1).startup(); + + assertManifestsPresence(dataDirs, paths, true); + + // remove them again and reload by mbean, that should create them too + removeAllManifests(dataDirs, paths); + assertManifestsPresence(dataDirs, paths, false); + cluster.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true)); + assertManifestsPresence(dataDirs, paths, true); + + cluster.get(1).shutdown(true); + + // remove manifest only in the first data dir + removeAllManifests(new String[] {dataDirs[0]}, paths); + + // they will be still created for that first dir + cluster.get(1).startup(); + assertManifestsPresence(dataDirs, paths, true); + } + + @Test + public void testMissingManifestIsNotCreatedOnStartupWithEnrichmentDisabled() + { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))")); + populate(cluster); + + cluster.get(1) + .nodetoolResult("snapshot", "-t", "snapshot_with_local_or_replicated") + .asserts() + .success(); + + String[] dataDirs = (String[]) cluster.get(1).config().get("data_file_directories"); + String[] paths = getSnapshotPaths(false); + + assertManifestsPresence(dataDirs, paths, true); + + cluster.get(1).shutdown(true); + + // remove all manifest files + removeAllManifests(dataDirs, paths); + assertManifestsPresence(dataDirs, paths, false); + + try (WithProperties ignored = new WithProperties().set(SNAPSHOT_MANIFEST_ENRICH_OR_CREATE_ENABLED, false)) + { + // restart which should NOT add them back because we disabled it by system property + cluster.get(1).startup(); + + // no manifests created + assertManifestsPresence(dataDirs, paths, false); + + cluster.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true)); + + // no manifests created, even after restart of SnapshotManager + assertManifestsPresence(dataDirs, paths, false); + } + } + @Test public void testSnapshotsCleanupByTTL() { - cluster.get(1).nodetoolResult("snapshot", "--ttl", format("%ds", FIVE_SECONDS), + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))")); + populate(cluster, withKeyspace("%s"), "tbl"); + cluster.get(1).nodetoolResult("snapshot", "-kt", withKeyspace("%s.tbl"), "--ttl", format("%ds", FIVE_SECONDS), "-t", "basic").asserts().success(); waitForSnapshotPresent("basic"); waitForSnapshotCleared("basic"); @@ -469,4 +584,52 @@ public class SnapshotsTest extends TestBaseImpl waitForSnapshot(tag, true, true); } } + + private void assertManifestsPresence(String[] dataDirs, String[] paths, boolean shouldExist) + { + for (String dataDir : dataDirs) + { + for (String path : paths) + { + Path snapshotDir = Paths.get(dataDir).resolve(path).resolve("snapshots").resolve("snapshot_with_local_or_replicated"); + assertEquals(shouldExist, new File(snapshotDir, "manifest.json").exists()); + } + } + } + + private void removeAllManifests(String[] dataDirs, String[] paths) + { + for (String dataDir : dataDirs) + { + for (String path : paths) + { + Path snapshotDir = Paths.get(dataDir).resolve(path).resolve("snapshots").resolve("snapshot_with_local_or_replicated"); + + File manifest = new File(snapshotDir, "manifest.json"); + assertTrue(manifest.exists()); + manifest.delete(); + assertFalse(manifest.exists()); + } + } + } + + private String[] getSnapshotPaths(boolean forSystemKeyspaces) + { + return cluster.get(1).applyOnInstance((IIsolatedExecutor.SerializableFunction<Boolean, String[]>) forSystem -> { + List<String> result = new ArrayList<>(); + + for (Keyspace keyspace : Keyspace.all()) + { + if (forSystem && !LOCAL_SYSTEM_KEYSPACE_NAMES.contains(keyspace.getName()) && !REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(keyspace.getName())) + continue; + else if (LOCAL_SYSTEM_KEYSPACE_NAMES.contains(keyspace.getName()) || REPLICATED_SYSTEM_KEYSPACE_NAMES.contains(keyspace.getName())) + continue; + + for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores()) + result.add(format("%s/%s-%s", keyspace.getName(), cfs.name, cfs.metadata().id.toHexString())); + } + + return result.toArray(new String[0]); + }, forSystemKeyspaces); + } } diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index b6d4126fc9..06db7e4a02 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -351,11 +351,11 @@ public class DirectoriesTest // Create snapshot with and without manifest FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false); - FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false); - // ephemeral without manifst - FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true); + FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, true, false); + // ephemeral without manifest + FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, true, true); - // Both snapshots should be present + // All snapshots should be present Map<String, TableSnapshot> snapshots = listSnapshots(directories); assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3)); assertThat(snapshots.get(SNAPSHOT1)).isEqualTo(snapshot1.asTableSnapshot()); @@ -459,7 +459,7 @@ public class DirectoriesTest } }); // CASSANDRA-17357: include indexes when computing true size of parent table - assertEquals(70L, parentSnapshotDetail.get("test").computeTrueSizeBytes(files)); + assertEquals(70L, parentSnapshotDetail.get("test").computeTrueSizeBytes(files) - parentSnapshotDetail.get("test").getManifestsSize()); // check backup directory File parentBackupDirectory = Directories.getBackupsDirectory(parentDesc); diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java index 54ecf19d98..ee76cedd63 100644 --- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java +++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java @@ -23,6 +23,7 @@ import java.nio.file.Paths; import java.time.Instant; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; @@ -37,6 +38,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.Directories; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.utils.Clock; import org.assertj.core.util.Lists; import static org.apache.cassandra.service.snapshot.SnapshotLoader.SNAPSHOT_DIR_PATTERN; @@ -110,9 +112,9 @@ public class SnapshotLoaderTest @Test public void testSnapshotsWithoutManifests() throws IOException { - Set<File> tag1Files = new HashSet<>(); - Set<File> tag2Files = new HashSet<>(); - Set<File> tag3Files = new HashSet<>(); + Set<File> firstSnapshotDirs = new HashSet<>(); + Set<File> secondSnapshotDirs = new HashSet<>(); + Set<File> thirdSnapshotDirs = new HashSet<>(); // Create one snapshot per table - without manifests: // - ks1.t1 : tag1 @@ -121,20 +123,26 @@ public class SnapshotLoaderTest File baseDir = new File(tmpDir.newFolder()); for (String dataDir : DATA_DIRS) { - tag1Files.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1)); - tag2Files.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE2_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2)); - tag3Files.add(createDir(baseDir, dataDir, KEYSPACE_2, tableDirName(TABLE3_NAME, TABLE3_ID), Directories.SNAPSHOT_SUBDIR, TAG3)); + firstSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1)); + secondSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE2_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2)); + thirdSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_2, tableDirName(TABLE3_NAME, TABLE3_ID), Directories.SNAPSHOT_SUBDIR, TAG3)); } + Instant createdAt = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()); + + createManifests(firstSnapshotDirs, createdAt); + createManifests(secondSnapshotDirs, createdAt); + createManifests(thirdSnapshotDirs, createdAt); + // Verify all 3 snapshots are found correctly from data directories SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1), Paths.get(baseDir.toString(), DATA_DIR_2), Paths.get(baseDir.toString(), DATA_DIR_3))); Set<TableSnapshot> snapshots = loader.loadSnapshots(); assertThat(snapshots).hasSize(3); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false)); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files, false)); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files, false)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, createdAt, null, firstSnapshotDirs, false)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, createdAt, null, secondSnapshotDirs, false)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, createdAt, null, thirdSnapshotDirs, false)); // Verify snapshot loading for a specific keyspace loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1), @@ -143,21 +151,37 @@ public class SnapshotLoaderTest snapshots = loader.loadSnapshots(KEYSPACE_1); assertThat(snapshots).hasSize(2); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false)); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files, false)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, createdAt, null, firstSnapshotDirs, false)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, createdAt, null, secondSnapshotDirs, false)); loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1), Paths.get(baseDir.toString(), DATA_DIR_2), Paths.get(baseDir.toString(), DATA_DIR_3))); snapshots = loader.loadSnapshots(KEYSPACE_2); assertThat(snapshots).hasSize(1); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files, false)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, createdAt, null, thirdSnapshotDirs, false)); + } + + private void createManifests(Set<File> snapshotDirs, Instant createdAt) + { + SnapshotManifest snapshotManifest = new SnapshotManifest(List.of(), null, createdAt, false); + for (File snapshotDir : snapshotDirs) + { + try + { + snapshotManifest.serializeToJsonFile(new File(snapshotDir, "manifest.json")); + } + catch (Throwable t) + { + throw new RuntimeException("Unable to write manifest file", t); + } + } } @Test public void testEphemeralSnapshotWithoutManifest() throws IOException { - Set<File> tag1Files = new HashSet<>(); + Set<File> snapshotDirs = new HashSet<>(); // Create one snapshot per table - without manifests: // - ks1.t1 : tag1 @@ -166,7 +190,7 @@ public class SnapshotLoaderTest for (String dataDir : DATA_DIRS) { File dir = createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1); - tag1Files.add(dir); + snapshotDirs.add(dir); if (!ephemeralFileCreated) { createEphemeralMarkerFile(dir); @@ -179,9 +203,12 @@ public class SnapshotLoaderTest Paths.get(baseDir.toString(), DATA_DIR_2), Paths.get(baseDir.toString(), DATA_DIR_3))); + Instant createdAt = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()); + createManifests(snapshotDirs, createdAt); + Set<TableSnapshot> snapshots = loader.loadSnapshots(); assertThat(snapshots).hasSize(1); - assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, true)); + assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, createdAt, null, snapshotDirs, true)); Assert.assertTrue(snapshots.stream().findFirst().get().isEphemeral()); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
