Author: slebresne
Date: Mon Oct 10 13:56:27 2011
New Revision: 1180970
URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
Log:
Fix places where uncompressed sstable size is used in place of the compressed
one.
patch by slebresne; reviewed by jbellis for CASSANDRA-3338
Modified:
cassandra/branches/cassandra-1.0.0/CHANGES.txt
cassandra/branches/cassandra-1.0.0/build.xml
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
@@ -12,6 +12,8 @@
* run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
* default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
* add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
+ * Fix places where uncompressed size of sstables was use in place of the
+ compressed one (CASSANDRA-3338)
Fixes merged from 0.8 below:
* Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
* Force flush of status table when removing/updating token (CASSANDRA-3243)
Modified: cassandra/branches/cassandra-1.0.0/build.xml
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/build.xml (original)
+++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
@@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
<license name="The Apache Software License, Version 2.0"
url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
<scm connection="${scm.connection}"
developerConnection="${scm.developerConnection}" url="${scm.url}"/>
<dependencyManagement>
- <dependency groupId="org.xerial.snappy" artifactId="snappy-java"
version="1.0.3.3"/>
+ <dependency groupId="org.xerial.snappy" artifactId="snappy-java"
version="1.0.3"/>
<dependency groupId="com.ning" artifactId="compress-lzf"
version="0.8.4"/>
<dependency groupId="com.google.guava" artifactId="guava"
version="r08"/>
<dependency groupId="commons-cli" artifactId="commons-cli"
version="1.1"/>
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Oct 10 13:56:27 2011
@@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
long expectedFileSize = 0;
for (SSTableReader sstable : sstables)
{
- long size = sstable.length();
+ long size = sstable.onDiskLength();
expectedFileSize = expectedFileSize + size;
}
return expectedFileSize;
@@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
SSTableReader maxFile = null;
for (SSTableReader sstable : sstables)
{
- if (sstable.length() > maxSize)
+ if (sstable.onDiskLength() > maxSize)
{
- maxSize = sstable.length();
+ maxSize = sstable.onDiskLength();
maxFile = sstable;
}
}
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Mon Oct 10 13:56:27 2011
@@ -472,7 +472,7 @@ public class CompactionManager implement
boolean isCommutative =
cfs.metadata.getDefaultValidator().isCommutative();
// Calculate the expected compacted filesize
- String compactionFileLocation =
cfs.table.getDataFileLocation(sstable.length());
+ String compactionFileLocation =
cfs.table.getDataFileLocation(sstable.onDiskLength());
if (compactionFileLocation == null)
throw new IOException("disk full");
int expectedBloomFilterSize =
Math.max(DatabaseDescriptor.getIndexInterval(),
@@ -765,8 +765,8 @@ public class CompactionManager implement
String format = "Cleaned up to %s. %,d to %,d (~%d%% of
original) bytes for %,d keys. Time: %,dms.";
long dTime = System.currentTimeMillis() - startTime;
- long startsize = sstable.length();
- long endsize = newSstable.length();
+ long startsize = sstable.onDiskLength();
+ long endsize = newSstable.onDiskLength();
double ratio = (double)endsize / (double)startsize;
logger.info(String.format(format, writer.getFilename(),
startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
}
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Mon Oct 10 13:56:27 2011
@@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
{
List<Pair<SSTableReader, Long>> tableLengthPairs = new
ArrayList<Pair<SSTableReader, Long>>();
for(SSTableReader table: collection)
- tableLengthPairs.add(new Pair<SSTableReader, Long>(table,
table.length()));
+ tableLengthPairs.add(new Pair<SSTableReader, Long>(table,
table.onDiskLength()));
return tableLengthPairs;
}
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Mon Oct 10 13:56:27 2011
@@ -257,7 +257,7 @@ public abstract class SSTable
long sum = 0;
for (SSTableReader sstable : sstables)
{
- sum += sstable.length();
+ sum += sstable.onDiskLength();
}
return sum;
}
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Mon Oct 10 13:56:27 2011
@@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
long right = getPosition(new DecoratedKey(range.right, null),
Operator.GT);
if (right == -1 || Range.isWrapAround(range.left, range.right))
// right is past the end of the file, or it wraps
- right = length();
+ right = uncompressedLength();
if (left == right)
// empty range
continue;
@@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
}
/**
- * @return The length in bytes of the data file for this SSTable.
+ * @return The length in bytes of the data for this SSTable. For
+ * compressed files, this is not the same thing as the on disk size (see
+ * onDiskLength())
*/
- public long length()
+ public long uncompressedLength()
{
return dfile.length;
}
+ /**
+ * @return The length in bytes of the on disk size for this SSTable. For
+ * compressed files, this is not the same thing as the data length (see
+ * length())
+ */
+ public long onDiskLength()
+ {
+ return dfile.onDiskLength;
+ }
+
public boolean acquireReference()
{
while (true)
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
Mon Oct 10 13:56:27 2011
@@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
public CompressedSegmentedFile(String path, CompressionMetadata metadata)
{
- super(path, metadata.dataLength);
+ super(path, metadata.dataLength, metadata.compressedFileLength);
this.metadata = metadata;
}
Modified:
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
(original)
+++
cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
Mon Oct 10 13:56:27 2011
@@ -42,13 +42,23 @@ public abstract class SegmentedFile
public final String path;
public final long length;
+ // This differs from length for compressed files (but we still need length
for
+ // SegmentIterator because offsets in the file are relative to the
uncompressed size)
+ public final long onDiskLength;
+
/**
* Use getBuilder to get a Builder to construct a SegmentedFile.
*/
SegmentedFile(String path, long length)
{
+ this(path, length, length);
+ }
+
+ protected SegmentedFile(String path, long length, long onDiskLength)
+ {
this.path = path;
this.length = length;
+ this.onDiskLength = onDiskLength;
}
/**
