This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch cassandra-3.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 4f11cb9feebdb0017de1e1063cbf72b840567518 Merge: 979a71b 257fb03 Author: adelapena <a.penya.gar...@gmail.com> AuthorDate: Fri Jul 10 18:04:18 2020 +0100 Merge branch 'cassandra-2.2' into cassandra-3.0 # Conflicts: # CHANGES.txt # src/java/org/apache/cassandra/db/ColumnFamilyStore.java # test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java CHANGES.txt | 3 ++ .../org/apache/cassandra/db/ColumnFamilyStore.java | 12 +++---- src/java/org/apache/cassandra/db/Directories.java | 23 ++++++++----- .../apache/cassandra/io/sstable/Descriptor.java | 7 +++- .../apache/cassandra/db/ColumnFamilyStoreTest.java | 40 ++++++++++++++++++++++ 5 files changed, 69 insertions(+), 16 deletions(-) diff --cc CHANGES.txt index 16974c4,02de7c1..5681007 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -26,71 -4,24 +26,74 @@@ Merged from 2.2 * Remove duplicated tools binaries from tarballs (CASSANDRA-15768) * Duplicate results with DISTINCT queries in mixed mode (CASSANDRA-15501) * Disable JMX rebinding (CASSANDRA-15653) - * Fix Commit log replays when static column clustering keys are collections (CASSANDRA-14365) - * Fix Red Hat init script on newer systemd versions (CASSANDRA-15273) - * Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567) Merged from 2.1: + * Fix writing of snapshot manifest when the table has table-backed secondary indexes (CASSANDRA-10968) * Fix parse error in cqlsh COPY FROM and formatting for map of blobs (CASSANDRA-15679) + * Fix Commit log replays when static column clustering keys are collections (CASSANDRA-14365) + * Fix Red Hat init script on newer systemd versions (CASSANDRA-15273) + * Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567) + -2.2.16 +3.0.20 + * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506) + * Include updates to static column in mutation size calculations (CASSANDRA-15293) + * Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292) + * GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306) + * Fix sstabledump's position key value when partitions have multiple rows (CASSANDRA-14721) + * Avoid over-scanning data directories in LogFile.verify() (CASSANDRA-15364) + * Bump generations and document changes to system_distributed and system_traces in 3.0, 3.11 + (CASSANDRA-15441) + * Fix system_traces creation timestamp; optimise system keyspace upgrades (CASSANDRA-15398) + * Fix various data directory prefix matching issues (CASSANDRA-13974) + * Minimize clustering values in metadata collector (CASSANDRA-15400) + * Avoid over-trimming of results in mixed mode clusters (CASSANDRA-15405) + * validate value sizes in LegacyLayout (CASSANDRA-15373) + * Ensure that tracing doesn't break connections in 3.x/4.0 mixed mode by default (CASSANDRA-15385) + * Make sure index summary redistribution does not start when compactions are paused (CASSANDRA-15265) + * Ensure legacy rows have primary key livenessinfo when they contain illegal cells (CASSANDRA-15365) + * Fix race condition when setting bootstrap flags (CASSANDRA-14878) + * Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426) +Merged from 2.2: * Fix SELECT JSON output for empty blobs (CASSANDRA-15435) * In-JVM DTest: Set correct internode message version for upgrade test (CASSANDRA-15371) - * In-JVM DTest: Support NodeTool in dtest + * In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429) + -2.2.15 +3.0.19 + * Add ability to cap max negotiable protocol version (CASSANDRA-15193) + * Gossip tokens on startup if available (CASSANDRA-15335) + * Fix resource leak in CompressedSequentialWriter (CASSANDRA-15340) + * Fix merge which reverted CASSANDRA-14993 (CASSANDRA-15289) + * Fix LegacyLayout RangeTombstoneList IndexOutOfBoundsException when upgrading and RangeTombstone bounds are asymmetric (CASSANDRA-15172) + * Fix NPE when using allocate_tokens_for_keyspace on new DC/rack (CASSANDRA-14952) + * Filter sstables earlier when running cleanup (CASSANDRA-15100) + * Use mean row count instead of mean column count for index selectivity calculation (CASSANDRA-15259) + * Avoid updating unchanged gossip states (CASSANDRA-15097) + * Prevent recreation of previously dropped columns with a different kind (CASSANDRA-14948) + * Prevent client requests from blocking on executor task queue (CASSANDRA-15013) + * Toughen up column drop/recreate type validations (CASSANDRA-15204) + * LegacyLayout should handle paging states that cross a collection column (CASSANDRA-15201) + * Prevent RuntimeException when username or password is empty/null (CASSANDRA-15198) + * Multiget thrift query returns null records after digest mismatch (CASSANDRA-14812) + * Skipping illegal legacy cells can break reverse iteration of indexed partitions (CASSANDRA-15178) + * Handle paging states serialized with a different version than the session's (CASSANDRA-15176) + * Throw IOE instead of asserting on unsupporter peer versions (CASSANDRA-15066) + * Update token metadata when handling MOVING/REMOVING_TOKEN events (CASSANDRA-15120) + * Add ability to customize cassandra log directory using $CASSANDRA_LOG_DIR (CASSANDRA-15090) + * Skip cells with illegal column names when reading legacy sstables (CASSANDRA-15086) + * Fix assorted gossip races and add related runtime checks (CASSANDRA-15059) + * Fix mixed mode partition range scans with limit (CASSANDRA-15072) + * cassandra-stress works with frozen collections: list and set (CASSANDRA-14907) + * For nodetool listsnapshots output, put spaces between columns, and increase snapshot padding (CASSANDRA-14876) + * Fix handling FS errors on writing and reading flat files - LogTransaction and hints (CASSANDRA-15053) + * Avoid double closing the iterator to avoid overcounting the number of requests (CASSANDRA-15058) + * Improve `nodetool status -r` speed (CASSANDRA-14847) + * Improve merkle tree size and time on heap (CASSANDRA-14096) + * Add missing commands to nodetool-completion (CASSANDRA-14916) + * Anti-compaction temporarily corrupts sstable state for readers (CASSANDRA-15004) + Merged from 2.2: * Catch non-IOException in FileUtils.close to make sure that all resources are closed (CASSANDRA-15225) * Handle exceptions during authentication/authorization (CASSANDRA-15041) - * Fix JDK7 compatibility broken in cassandra-2.2 (CASSANDRA-15050) * Support cross version messaging in in-jvm upgrade dtests (CASSANDRA-15078) * Fix index summary redistribution cancellation (CASSANDRA-15045) * Refactor Circle CI configuration (CASSANDRA-14806) diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java index a9bd54a,7f625e1..70c14c0 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@@ -1640,10 -2426,10 +1640,10 @@@ public class ColumnFamilyStore implemen public Set<SSTableReader> snapshotWithoutFlush(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral) { Set<SSTableReader> snapshottedSSTables = new HashSet<>(); + final JSONArray filesJSONArr = new JSONArray(); for (ColumnFamilyStore cfs : concatWithIndexes()) { - final JSONArray filesJSONArr = new JSONArray(); - try (RefViewFragment currentView = cfs.selectAndReference(CANONICAL_SSTABLES)) + try (RefViewFragment currentView = cfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate == null || predicate.apply(x)))) { for (SSTableReader ssTable : currentView.sstables) { @@@ -1655,13 -2444,9 +1655,13 @@@ logger.trace("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory); snapshottedSSTables.add(ssTable); } - - writeSnapshotManifest(filesJSONArr, snapshotName); - - if (!Schema.isLocalSystemKeyspace(metadata.ksName) && !Schema.isReplicatedSystemKeyspace(metadata.ksName)) - writeSnapshotSchema(snapshotName); } } + writeSnapshotManifest(filesJSONArr, snapshotName); ++ ++ if (!Schema.isLocalSystemKeyspace(metadata.ksName) && !Schema.isReplicatedSystemKeyspace(metadata.ksName)) ++ writeSnapshotSchema(snapshotName); ++ if (ephemeral) createEphemeralSnapshotMarkerFile(snapshotName); return snapshottedSSTables; diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 6840e2b,4d452c6..f7152ff --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@@ -19,38 -19,98 +19,44 @@@ package org.apache.cassandra.db; import java.io.File; + import java.io.FileReader; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; - -import org.apache.cassandra.db.index.PerRowSecondaryIndexTest; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; +import java.util.*; + +import org.junit.Before; import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.cassandra.OrderedJUnit4ClassRunner; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.Util; ++import org.json.simple.JSONArray; ++import org.json.simple.JSONObject; ++import org.json.simple.parser.JSONParser; ++ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.Iterators; +import org.apache.cassandra.*; import org.apache.cassandra.config.*; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.columniterator.IdentityQueryFilter; -import org.apache.cassandra.db.composites.CellName; -import org.apache.cassandra.db.composites.CellNameType; -import org.apache.cassandra.db.composites.CellNames; -import org.apache.cassandra.db.composites.Composites; -import org.apache.cassandra.db.filter.ColumnSlice; -import org.apache.cassandra.db.filter.ExtendedFilter; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.QueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.db.marshal.LexicalUUIDType; -import org.apache.cassandra.db.marshal.LongType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.dht.Bounds; -import org.apache.cassandra.dht.ExcludingBounds; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.IncludingExcludingBounds; -import org.apache.cassandra.dht.Range; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.ClearableHistogram; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.thrift.SlicePredicate; -import org.apache.cassandra.thrift.SliceRange; -import org.apache.cassandra.thrift.ThriftValidation; +import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.WrappedRunnable; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; - -import static org.apache.cassandra.Util.cellname; -import static org.apache.cassandra.Util.column; -import static org.apache.cassandra.Util.dk; -import static org.apache.cassandra.Util.rp; -import static org.apache.cassandra.utils.ByteBufferUtil.bytes; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static junit.framework.Assert.assertNotNull; + @RunWith(OrderedJUnit4ClassRunner.class) public class ColumnFamilyStoreTest { @@@ -512,30 -2362,37 +518,64 @@@ } @Test + public void testSnapshotWithoutFlushWithSecondaryIndexes() throws Exception + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1); + cfs.truncateBlocking(); + - List<Mutation> rms = new LinkedList<>(); - Mutation rm; - - rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes("k1")); - rm.add(CF_INDEX1, cellname("birthdate"), ByteBufferUtil.bytes(1L), 0); - rm.add(CF_INDEX1, cellname("nobirthdate"), ByteBufferUtil.bytes(1L), 0); - rms.add(rm); - Util.writeColumnFamily(rms); ++ UpdateBuilder builder = UpdateBuilder.create(cfs.metadata, "key") ++ .newRow() ++ .add("birthdate", 1L) ++ .add("notbirthdate", 2L); ++ new Mutation(builder.build()).applyUnsafe(); ++ cfs.forceBlockingFlush(); + + String snapshotName = "newSnapshot"; + cfs.snapshotWithoutFlush(snapshotName); + - File snapshotManifestFile = cfs.directories.getSnapshotManifestFile(snapshotName); ++ File snapshotManifestFile = cfs.getDirectories().getSnapshotManifestFile(snapshotName); + JSONParser parser = new JSONParser(); + JSONObject manifest = (JSONObject) parser.parse(new FileReader(snapshotManifestFile)); + JSONArray files = (JSONArray) manifest.get("files"); + + // Keyspace1-Indexed1 and the corresponding index + assert files.size() == 2; + + // Snapshot of the secondary index is stored in the subfolder with the same file name + String baseTableFile = (String) files.get(0); + String indexTableFile = (String) files.get(1); + assert !baseTableFile.equals(indexTableFile); + assert Directories.isSecondaryIndexFolder(new File(indexTableFile).getParentFile()); + assert indexTableFile.endsWith(baseTableFile); + } ++ ++ ++ @Test + public void testScrubDataDirectories() throws Throwable + { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + + ColumnFamilyStore.scrubDataDirectories(cfs.metadata); + + new RowUpdateBuilder(cfs.metadata, 2, "key").clustering("name").add("val", "2").build().applyUnsafe(); + cfs.forceBlockingFlush(); + + // Nuke the metadata and reload that sstable + Collection<SSTableReader> ssTables = cfs.getLiveSSTables(); + assertEquals(1, ssTables.size()); + SSTableReader ssTable = ssTables.iterator().next(); + + String dataFileName = ssTable.descriptor.filenameFor(Component.DATA); + String tmpDataFileName = ssTable.descriptor.tmpFilenameFor(Component.DATA); + new File(dataFileName).renameTo(new File(tmpDataFileName)); + + ssTable.selfRef().release(); + + ColumnFamilyStore.scrubDataDirectories(cfs.metadata); + + List<File> ssTableFiles = new Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles(); + assertNotNull(ssTableFiles); + assertEquals(0, ssTableFiles.size()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org