This is an automated email from the ASF dual-hosted git repository.
brandonwilliams pushed a commit to branch cassandra-3.11
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit c190ab9fcbc8a993e570e6cdafc2b90219f80109
Merge: 377ceb6 04be736
Author: Brandon Williams <brandonwilli...@apache.org>
AuthorDate: Mon Apr 27 11:15:30 2020 -0500
Merge branch 'cassandra-3.0' into cassandra-3.11
CHANGES.txt | 1 +
.../cassandra/io/compress/CompressionMetadata.java | 6 +-
.../compress/CompressedRandomAccessReaderTest.java | 86 +++++++++++++++++-----
3 files changed, 73 insertions(+), 20 deletions(-)
diff --cc CHANGES.txt
index fd7ad6a,99f540c..b3c593b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,7 -1,5 +1,8 @@@
-3.0.21
+3.11.7
+ * Allow sstableloader to use SSL on the native port (CASSANDRA-14904)
+Merged from 3.0:
+ * Fix chunk index overflow due to large sstable with small chunk length
(CASSANDRA-15595)
+ * Allow selecting static column only when querying static index
(CASSANDRA-14242)
* cqlsh return non-zero status when STDIN CQL fails (CASSANDRA-15623)
* Don't skip sstables in slice queries based only on local min/max/deletion
timestamp (CASSANDRA-15690)
* Memtable memory allocations may deadlock (CASSANDRA-15367)
diff --cc
test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
index 0d2a9fb,34ff94f..c718147
---
a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
+++
b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
@@@ -123,42 -118,77 +125,61 @@@ public class CompressedRandomAccessRead
}
}
- private static void testResetAndTruncate(File f, boolean compressed,
boolean usemmap, int junkSize) throws IOException
+ /**
+ * JIRA: CASSANDRA-15595 verify large position with small chunk length
won't overflow chunk index
+ */
+ @Test
+ public void testChunkIndexOverflow() throws IOException
{
- final String filename = f.getAbsolutePath();
- MetadataCollector sstableMetadataCollector = new
MetadataCollector(new ClusteringComparator(BytesType.instance));
- try(SequentialWriter writer = compressed
- ? new CompressedSequentialWriter(f, filename + ".metadata",
- null, SequentialWriterOption.DEFAULT,
- CompressionParams.snappy(), sstableMetadataCollector)
- : new SequentialWriter(f))
+ File file = File.createTempFile("chunk_idx_overflow", "1");
+ String filename = file.getAbsolutePath();
+ int chunkLength = 4096; // 4k
+
+ try
{
- writer.write("The quick ".getBytes());
- DataPosition mark = writer.mark();
- writer.write("blue fox jumps over the lazy dog".getBytes());
+ writeSSTable(file, CompressionParams.snappy(chunkLength), 10);
+ CompressionMetadata metadata = new CompressionMetadata(filename +
".metadata", file.length(), ChecksumType.CRC32);
- // write enough to be sure to change chunk
- for (int i = 0; i < junkSize; ++i)
+ long chunks = 2761628520L;
+ long midPosition = (chunks / 2L) * chunkLength;
+ int idx = 8 * (int) (midPosition / chunkLength); // before patch
+ assertTrue("Expect integer overflow", idx < 0);
+
+ try
{
- writer.write((byte) 1);
+ metadata.chunkFor(midPosition);
+ fail("Expected to throw EOF exception with chunk idx larger
than total number of chunks in the sstable");
+ }
+ catch (CorruptSSTableException e)
+ {
+ assertTrue("Expect EOF, but got " + e.getCause(),
e.getCause() instanceof EOFException);
}
-
- writer.resetAndTruncate(mark);
- writer.write("brown fox jumps over the lazy dog".getBytes());
- writer.finish();
}
- assert f.exists();
+ finally
+ {
+ if (file.exists())
+ assertTrue(file.delete());
+ File metadata = new File(filename + ".metadata");
+ if (metadata.exists())
+ metadata.delete();
+ }
+ }
+
+ private static void testResetAndTruncate(File f, boolean compressed,
boolean usemmap, int junkSize) throws IOException
+ {
+ final String filename = f.getAbsolutePath();
- try(ChannelProxy channel = new ChannelProxy(f))
- {
- writeSSTable(f, compressed ? CompressionParams.snappy() : null,
junkSize);
-
- CompressionMetadata compressionMetadata = compressed ? new
CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) :
null;
- RandomAccessReader.Builder builder = compressed
- ? new
CompressedRandomAccessReader.Builder(channel, compressionMetadata)
- : new
RandomAccessReader.Builder(channel);
-
- if (usemmap)
- {
- if (compressed)
- builder.regions(MmappedRegions.map(channel,
compressionMetadata));
- else
- builder.regions(MmappedRegions.map(channel, f.length()));
- }
++ writeSSTable(f, compressed ? CompressionParams.snappy() : null,
junkSize);
- try(RandomAccessReader reader = builder.build())
- {
- String expected = "The quick brown fox jumps over the lazy
dog";
- assertEquals(expected.length(), reader.length());
- byte[] b = new byte[expected.length()];
- reader.readFully(b);
- assert new String(b).equals(expected) : "Expecting '" +
expected + "', got '" + new String(b) + '\'';
- }
-
- if (usemmap)
- builder.regions.close();
+ CompressionMetadata compressionMetadata = compressed ? new
CompressionMetadata(filename + ".metadata", f.length(), ChecksumType.CRC32) :
null;
+ try (FileHandle.Builder builder = new
FileHandle.Builder(filename).mmapped(usemmap).withCompressionMetadata(compressionMetadata);
+ FileHandle fh = builder.complete();
+ RandomAccessReader reader = fh.createReader())
+ {
+ String expected = "The quick brown fox jumps over the lazy dog";
+ assertEquals(expected.length(), reader.length());
+ byte[] b = new byte[expected.length()];
+ reader.readFully(b);
+ assert new String(b).equals(expected) : "Expecting '" + expected
+ "', got '" + new String(b) + '\'';
}
finally
{
@@@ -170,6 -200,31 +191,33 @@@
}
}
+ private static void writeSSTable(File f, CompressionParams params, int
junkSize) throws IOException
+ {
+ final String filename = f.getAbsolutePath();
+ MetadataCollector sstableMetadataCollector = new
MetadataCollector(new ClusteringComparator(BytesType.instance));
+ try(SequentialWriter writer = params != null
- ? new CompressedSequentialWriter(f,
filename + ".metadata", params, sstableMetadataCollector)
- : SequentialWriter.open(f))
++ ? new CompressedSequentialWriter(f, filename + ".metadata",
++ null,
SequentialWriterOption.DEFAULT,
++ params,
sstableMetadataCollector)
++ : new SequentialWriter(f))
+ {
+ writer.write("The quick ".getBytes());
+ DataPosition mark = writer.mark();
+ writer.write("blue fox jumps over the lazy dog".getBytes());
+
+ // write enough to be sure to change chunk
+ for (int i = 0; i < junkSize; ++i)
+ {
+ writer.write((byte) 1);
+ }
+
+ writer.resetAndTruncate(mark);
+ writer.write("brown fox jumps over the lazy dog".getBytes());
+ writer.finish();
+ }
+ assert f.exists();
+ }
+
/**
* If the data read out doesn't match the checksum, an exception should
be thrown
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
