Author: slebresne Date: Wed Nov 9 14:05:50 2011 New Revision: 1199764 URL: http://svn.apache.org/viewvc?rev=1199764&view=rev Log: merge from 1.0
Modified: cassandra/trunk/ (props changed) cassandra/trunk/CHANGES.txt cassandra/trunk/contrib/ (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java (props changed) cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java (props changed) cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java Propchange: cassandra/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0:1125021-1130369 /cassandra/branches/cassandra-0.8.1:1101014-1125018 -/cassandra/branches/cassandra-1.0:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1:1102511-1125020 Modified: cassandra/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=1199764&r1=1199763&r2=1199764&view=diff ============================================================================== --- cassandra/trunk/CHANGES.txt (original) +++ cassandra/trunk/CHANGES.txt Wed Nov 9 14:05:50 2011 @@ -13,6 +13,7 @@ * (CQL) fix handling of rows with no columns (CASSANDRA-3424, 3473) * fix querying supercolumns by name returning only a subset of subcolumns or old subcolumn versions (CASSANDRA-3446) + * automatically compute sha1 sum for uncompressed data files (CASSANDRA-3456) Merged from 0.8: * Make counter shard merging thread safe (CASSANDRA-3178) * fix updating CF row_cache_provider (CASSANDRA-3414) Propchange: cassandra/trunk/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018 -/cassandra/branches/cassandra-1.0/contrib:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0/contrib:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0/contrib:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/contrib:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/contrib:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1102511-1125020 Propchange: cassandra/trunk/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Nov 9 14:05:50 2011 @@ -4,7 +4,7 @@ /cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1198724,1198726-1199259,1199284 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018 -/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1199374,1199379 +/cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167085-1199374,1199379,1199763 /cassandra/branches/cassandra-1.0.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167104-1167229,1167232-1181093,1181741,1181816,1181820,1182951,1183243 /cassandra/tags/cassandra-0.7.0-rc3/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1051699-1053689 /cassandra/tags/cassandra-0.8.0-rc1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1102511-1125020 Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java?rev=1199764&r1=1199763&r2=1199764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/Component.java Wed Nov 9 14:05:50 2011 @@ -54,7 +54,9 @@ public class Component // statistical metadata about the content of the sstable STATS("Statistics.db"), // a bitmap secondary index: many of these may exist per sstable - BITMAP_INDEX("Bitidx.db"); + BITMAP_INDEX("Bitidx.db"), + // holds sha1 sum of the data file (to be checked by sha1sum) + DIGEST("Digest.sha1"); final String repr; Type(String repr) @@ -78,6 +80,7 @@ public class Component public final static Component COMPACTED_MARKER = new Component(Type.COMPACTED_MARKER, -1); public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO, -1); public final static Component STATS = new Component(Type.STATS, -1); + public final static Component DIGEST = new Component(Type.DIGEST, -1); public final Type type; public final int id; @@ -108,6 +111,7 @@ public class Component case COMPACTED_MARKER: case COMPRESSION_INFO: case STATS: + case DIGEST: return type.repr; case BITMAP_INDEX: return String.format("%d%c%s", id, separator, type.repr); @@ -145,6 +149,7 @@ public class Component case COMPACTED_MARKER: component = Component.COMPACTED_MARKER; break; case COMPRESSION_INFO: component = Component.COMPRESSION_INFO; break; case STATS: component = Component.STATS; break; + case DIGEST: component = Component.DIGEST; break; case BITMAP_INDEX: component = new Component(type, id); break; Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1199764&r1=1199763&r2=1199764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Wed Nov 9 14:05:50 2011 @@ -58,6 +58,7 @@ public abstract class SSTable public static final String COMPONENT_INDEX = Component.Type.PRIMARY_INDEX.repr; public static final String COMPONENT_FILTER = Component.Type.FILTER.repr; public static final String COMPONENT_STATS = Component.Type.STATS.repr; + public static final String COMPONENT_DIGEST = Component.Type.DIGEST.repr; public static final String TEMPFILE_MARKER = "tmp"; Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199764&r1=1199763&r2=1199764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Wed Nov 9 14:05:50 2011 @@ -37,9 +37,7 @@ import org.apache.cassandra.dht.IPartiti import org.apache.cassandra.io.compress.CompressedSequentialWriter; import org.apache.cassandra.io.util.*; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.BloomFilter; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.*; public class SSTableWriter extends SSTable { @@ -66,6 +64,10 @@ public class SSTableWriter extends SSTab Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.STATS)); if (metadata.compressionParameters().sstableCompressor != null) components.add(Component.COMPRESSION_INFO); + else + // it would feel safer to actually add this component later in maybeWriteDigest(), + // but the components are unmodifiable after construction + components.add(Component.DIGEST); return components; } @@ -93,6 +95,7 @@ public class SSTableWriter extends SSTab { dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode()); dataFile = SequentialWriter.open(new File(getFilename()), true); + dataFile.setComputeDigest(); } this.sstableMetadataCollector = sstableMetadataCollector; @@ -299,6 +302,7 @@ public class SSTableWriter extends SSTab // write sstable statistics SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata(); writeMetadata(descriptor, sstableMetadata); + maybeWriteDigest(); // remove the 'tmp' marker from all components final Descriptor newdesc = rename(descriptor, components); @@ -323,6 +327,21 @@ public class SSTableWriter extends SSTab return sstable; } + private void maybeWriteDigest() throws IOException + { + byte[] digest = dataFile.digest(); + if (digest == null) + return; + + SequentialWriter out = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_DIGEST)), true); + // Writting output compatible with sha1sum + Descriptor newdesc = descriptor.asTemporary(false); + String[] tmp = newdesc.filenameFor(SSTable.COMPONENT_DATA).split(new Character(File.separatorChar).toString()); + String dataFileName = tmp[tmp.length - 1]; + out.write(String.format("%s %s", Hex.bytesToHex(digest), dataFileName).getBytes()); + out.close(); + } + private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata) throws IOException { SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)), true); Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java?rev=1199764&r1=1199763&r2=1199764&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/util/SequentialWriter.java Wed Nov 9 14:05:50 2011 @@ -20,6 +20,8 @@ package org.apache.cassandra.io.util; import java.io.*; import java.nio.channels.ClosedChannelException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import org.apache.cassandra.utils.CLibrary; @@ -47,6 +49,7 @@ public class SequentialWriter extends Ou private long ioCacheStartOffset = 0, bytesSinceCacheFlush = 0; public final DataOutputStream stream; + private MessageDigest digest; public SequentialWriter(File file, int bufferSize, boolean skipIOCache) throws IOException { @@ -199,6 +202,8 @@ public class SequentialWriter extends Ou protected void flushData() throws IOException { out.write(buffer, 0, validBufferBytes); + if (digest != null) + digest.update(buffer, 0, validBufferBytes); } public long getFilePointer() @@ -286,6 +291,41 @@ public class SequentialWriter extends Ou } /** + * Turn on digest computation on this writer. + * This can only be called before any data is written to this write, + * otherwise an IllegalStateException is thrown. + */ + public void setComputeDigest() + { + if (current != 0) + throw new IllegalStateException(); + + try + { + digest = MessageDigest.getInstance("SHA-1"); + } + catch (NoSuchAlgorithmException e) + { + // SHA-1 is standard in java 6 + throw new RuntimeException(e); + } + } + + /** + * Return the digest associated to this file or null if no digest was + * created. + * This can only be called once the file is fully created, i.e. after + * close() has been called. Otherwise an IllegalStateException is thrown. + */ + public byte[] digest() + { + if (buffer != null) + throw new IllegalStateException(); + + return digest == null ? null : digest.digest(); + } + + /** * Class to hold a mark to the position of the file */ protected static class BufferedFileWriterMark implements FileMark