This is an automated email from the ASF dual-hosted git repository. brandonwilliams pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 90f68a5f9439ed41e585e4946381c91ab1de6599 Merge: 78de6ac c190ab9 Author: Brandon Williams <brandonwilli...@apache.org> AuthorDate: Mon Apr 27 11:17:55 2020 -0500 Merge branch 'cassandra-3.11' into trunk CHANGES.txt | 1 + .../cassandra/io/compress/CompressionMetadata.java | 6 +- .../compress/CompressedRandomAccessReaderTest.java | 83 ++++++++++++++++------ 3 files changed, 69 insertions(+), 21 deletions(-) diff --cc CHANGES.txt index 11bc713,b3c593b..afd8c24 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,21 -1,7 +1,22 @@@ -3.11.7 - * Allow sstableloader to use SSL on the native port (CASSANDRA-14904) +4.0-alpha5 + * Avoid race condition when completing stream sessions (CASSANDRA-15666) + * Flush with fast compressors by default (CASSANDRA-15379) + * Fix CqlInputFormat regression from the switch to system.size_estimates (CASSANDRA-15637) + * Allow sending Entire SSTables over SSL (CASSANDRA-15740) + * Fix CQLSH UTF-8 encoding issue for Python 2/3 compatibility (CASSANDRA-15739) + * Fix batch statement preparation when multiple tables and parameters are used (CASSANDRA-15730) + * Fix regression with traceOutgoingMessage printing message size (CASSANDRA-15687) + * Ensure repaired data tracking reads a consistent amount of data across replicas (CASSANDRA-15601) + * Fix CQLSH to avoid arguments being evaluated (CASSANDRA-15660) + * Correct Visibility and Improve Safety of Methods in LatencyMetrics (CASSANDRA-15597) + * Allow cqlsh to run with Python2.7/Python3.6+ (CASSANDRA-15659,CASSANDRA-15573) + * Improve logging around incremental repair (CASSANDRA-15599) + * Do not check cdc_raw_directory filesystem space if CDC disabled (CASSANDRA-15688) + * Replace array iterators with get by index (CASSANDRA-15394) + * Minimize BTree iterator allocations (CASSANDRA-15389) +Merged from 3.11: Merged from 3.0: + * Fix chunk index overflow due to large sstable with small chunk length (CASSANDRA-15595) * Allow selecting static column only when querying static index (CASSANDRA-14242) * cqlsh return non-zero status when STDIN CQL fails (CASSANDRA-15623) * Don't skip sstables in slice queries based only on local min/max/deletion timestamp (CASSANDRA-15690) diff --cc test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java index 5153310,c718147..d3d81f0 --- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java @@@ -24,6 -25,6 +25,7 @@@ import java.io.RandomAccessFile import java.util.Arrays; import java.util.Random; ++import org.assertj.core.api.Assertions; import org.junit.BeforeClass; import org.junit.Test; @@@ -35,8 -36,9 +37,9 @@@ import org.apache.cassandra.io.sstable. import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.util.*; import org.apache.cassandra.schema.CompressionParams; -import org.apache.cassandra.utils.ChecksumType; import org.apache.cassandra.utils.SyncUtil; ++import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; @@@ -136,33 -125,52 +139,46 @@@ public class CompressedRandomAccessRead } } - private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize, double minCompressRatio) throws IOException + /** + * JIRA: CASSANDRA-15595 verify large position with small chunk length won't overflow chunk index + */ + @Test + public void testChunkIndexOverflow() throws IOException { - final String filename = f.getAbsolutePath(); - MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); - try(SequentialWriter writer = compressed - ? new CompressedSequentialWriter(f, filename + ".metadata", - null, SequentialWriterOption.DEFAULT, - CompressionParams.snappy(), sstableMetadataCollector) - : new SequentialWriter(f)) + File file = File.createTempFile("chunk_idx_overflow", "1"); + String filename = file.getAbsolutePath(); + int chunkLength = 4096; // 4k + + try { - writer.write("The quick ".getBytes()); - DataPosition mark = writer.mark(); - writer.write("blue fox jumps over the lazy dog".getBytes()); + writeSSTable(file, CompressionParams.snappy(chunkLength), 10); - CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), ChecksumType.CRC32); ++ CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), true); - // write enough to be sure to change chunk - for (int i = 0; i < junkSize; ++i) - { - writer.write((byte) 1); - } + long chunks = 2761628520L; + long midPosition = (chunks / 2L) * chunkLength; + int idx = 8 * (int) (midPosition / chunkLength); // before patch + assertTrue("Expect integer overflow", idx < 0); - writer.resetAndTruncate(mark); - writer.write("brown fox jumps over the lazy dog".getBytes()); - writer.finish(); - try - { - metadata.chunkFor(midPosition); - fail("Expected to throw EOF exception with chunk idx larger than total number of chunks in the sstable"); - } - catch (CorruptSSTableException e) - { - assertTrue("Expect EOF, but got " + e.getCause(), e.getCause() instanceof EOFException); - } ++ Throwable thrown = Assertions.catchThrowable(() -> metadata.chunkFor(midPosition)); ++ assertThat(thrown).isInstanceOf(CorruptSSTableException.class) ++ .hasCauseInstanceOf(EOFException.class); } - assert f.exists(); + finally + { + if (file.exists()) + assertTrue(file.delete()); + File metadata = new File(filename + ".metadata"); + if (metadata.exists()) + metadata.delete(); + } + } + - private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize) throws IOException ++ private static void testResetAndTruncate(File f, boolean compressed, boolean usemmap, int junkSize, double minCompressRatio) throws IOException + { + final String filename = f.getAbsolutePath(); + writeSSTable(f, compressed ? CompressionParams.snappy() : null, junkSize); - CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) : null; + CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), true) : null; try (FileHandle.Builder builder = new FileHandle.Builder(filename).mmapped(usemmap).withCompressionMetadata(compressionMetadata); FileHandle fh = builder.complete(); RandomAccessReader reader = fh.createReader()) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org