merge from 1.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/070c1d8d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/070c1d8d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/070c1d8d Branch: refs/heads/trunk Commit: 070c1d8dd3c7b674034acfbc421555d658aabb60 Parents: 10372c2 1ad710b Author: Jonathan Ellis <jbel...@apache.org> Authored: Thu Jul 26 17:16:39 2012 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Thu Jul 26 17:16:39 2012 -0500 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../db/index/PerColumnSecondaryIndex.java | 8 + .../cassandra/db/index/PerRowSecondaryIndex.java | 7 + .../apache/cassandra/db/index/SecondaryIndex.java | 3 + .../cassandra/db/index/SecondaryIndexManager.java | 19 +- .../apache/cassandra/thrift/ThriftValidation.java | 14 +- .../cassandra/db/SecondaryIndexColumnSizeTest.java | 215 +++++++++++++++ .../cassandra/thrift/ThriftValidationTest.java | 52 ---- 8 files changed, 258 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 84db73d,79278aa..bcf56b7 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,154 -1,6 +1,155 @@@ -1.0.11 +1.1.3 + * (JMX) rename getRangeKeySample to sampleKeyRange to avoid returning + multi-MB results as an attribute (CASSANDRA-4452) + * flush based on data size, not throughput; overwritten columns no + longer artificially inflate liveRatio (CASSANDRA-4399) + * update default commitlog segment size to 32MB and total commitlog + size to 32/1024 MB for 32/64 bit JVMs, respectively (CASSANDRA-4422) + * avoid using global partitioner to estimate ranges in index sstables + (CASSANDRA-4403) + * restore pre-CASSANDRA-3862 approach to removing expired tombstones + from row cache during compaction (CASSANDRA-4364) + * (stress) support for CQL prepared statements (CASSANDRA-3633) + * Correctly catch exception when Snappy cannot be loaded (CASSANDRA-4400) + * (cql3) Support ORDER BY when IN condition is given in WHERE clause (CASSANDRA-4327) + * (cql3) delete "component_index" column on DROP TABLE call (CASSANDRA-4420) + * change nanoTime() to currentTimeInMillis() in schema related code (CASSANDRA-4432) + * add a token generation tool (CASSANDRA-3709) + * Fix LCS bug with sstable containing only 1 row (CASSANDRA-4411) + * fix "Can't Modify Index Name" problem on CF update (CASSANDRA-4439) + * Fix assertion error in getOverlappingSSTables during repair (CASSANDRA-4456) + * fix nodetool's setcompactionthreshold command (CASSANDRA-4455) + * Ensure compacted files are never used, to avoid counter overcount (CASSANDRA-4436) +Merged from 1.0: ++ * Push the validation of secondary index values to the SecondaryIndexManager (CASSANDRA-4240) * allow dropping columns shadowed by not-yet-expired supercolumn or row tombstones in PrecompactedRow (CASSANDRA-4396) + * fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195) + * Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419) + * Bootstraps that fail are detected upon restart and will retry safely without + needing to delete existing data first (CASSANDRA-4427) + * (cqlsh) add a COPY TO command to copy a CF to a CSV file (CASSANDRA-4434) + + +1.1.2 + * Fix cleanup not deleting index entries (CASSANDRA-4379) + * Use correct partitioner when saving + loading caches (CASSANDRA-4331) + * Check schema before trying to export sstable (CASSANDRA-2760) + * Raise a meaningful exception instead of NPE when PFS encounters + an unconfigured node + no default (CASSANDRA-4349) + * fix bug in sstable blacklisting with LCS (CASSANDRA-4343) + * LCS no longer promotes tiny sstables out of L0 (CASSANDRA-4341) + * skip tombstones during hint replay (CASSANDRA-4320) + * fix NPE in compactionstats (CASSANDRA-4318) + * enforce 1m min keycache for auto (CASSANDRA-4306) + * Have DeletedColumn.isMFD always return true (CASSANDRA-4307) + * (cql3) exeption message for ORDER BY constraints said primary filter can be + an IN clause, which is misleading (CASSANDRA-4319) + * (cql3) Reject (not yet supported) creation of 2ndardy indexes on tables with + composite primary keys (CASSANDRA-4328) + * Set JVM stack size to 160k for java 7 (CASSANDRA-4275) + * cqlsh: add COPY command to load data from CSV flat files (CASSANDRA-4012) + * CFMetaData.fromThrift to throw ConfigurationException upon error (CASSANDRA-4353) + * Use CF comparator to sort indexed columns in SecondaryIndexManager + (CASSANDRA-4365) + * add strategy_options to the KSMetaData.toString() output (CASSANDRA-4248) + * (cql3) fix range queries containing unqueried results (CASSANDRA-4372) + * (cql3) allow updating column_alias types (CASSANDRA-4041) + * (cql3) Fix deletion bug (CASSANDRA-4193) + * Fix computation of overlapping sstable for leveled compaction (CASSANDRA-4321) + * Improve scrub and allow to run it offline (CASSANDRA-4321) + * Fix assertionError in StorageService.bulkLoad (CASSANDRA-4368) + * (cqlsh) add option to authenticate to a keyspace at startup (CASSANDRA-4108) + * (cqlsh) fix ASSUME functionality (CASSANDRA-4352) + * Fix ColumnFamilyRecordReader to not return progress > 100% (CASSANDRA-3942) +Merged from 1.0: + * Set gc_grace on index CF to 0 (CASSANDRA-4314) + + +1.1.1 + * allow larger cache capacities than 2GB (CASSANDRA-4150) + * add getsstables command to nodetool (CASSANDRA-4199) + * apply parent CF compaction settings to secondary index CFs (CASSANDRA-4280) + * preserve commitlog size cap when recycling segments at startup + (CASSANDRA-4201) + * (Hadoop) fix split generation regression (CASSANDRA-4259) + * ignore min/max compactions settings in LCS, while preserving + behavior that min=max=0 disables autocompaction (CASSANDRA-4233) + * log number of rows read from saved cache (CASSANDRA-4249) + * calculate exact size required for cleanup operations (CASSANDRA-1404) + * avoid blocking additional writes during flush when the commitlog + gets behind temporarily (CASSANDRA-1991) + * enable caching on index CFs based on data CF cache setting (CASSANDRA-4197) + * warn on invalid replication strategy creation options (CASSANDRA-4046) + * remove [Freeable]Memory finalizers (CASSANDRA-4222) + * include tombstone size in ColumnFamily.size, which can prevent OOM + during sudden mass delete operations by yielding a nonzero liveRatio + (CASSANDRA-3741) + * Open 1 sstableScanner per level for leveled compaction (CASSANDRA-4142) + * Optimize reads when row deletion timestamps allow us to restrict + the set of sstables we check (CASSANDRA-4116) + * add support for commitlog archiving and point-in-time recovery + (CASSANDRA-3690) + * avoid generating redundant compaction tasks during streaming + (CASSANDRA-4174) + * add -cf option to nodetool snapshot, and takeColumnFamilySnapshot to + StorageService mbean (CASSANDRA-556) + * optimize cleanup to drop entire sstables where possible (CASSANDRA-4079) + * optimize truncate when autosnapshot is disabled (CASSANDRA-4153) + * update caches to use byte[] keys to reduce memory overhead (CASSANDRA-3966) + * add column limit to cli (CASSANDRA-3012, 4098) + * clean up and optimize DataOutputBuffer, used by CQL compression and + CompositeType (CASSANDRA-4072) + * optimize commitlog checksumming (CASSANDRA-3610) + * identify and blacklist corrupted SSTables from future compactions + (CASSANDRA-2261) + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037) + * Expose API to repair a user provided range (CASSANDRA-3912) + * Add way to force the cassandra-cli to refresh its schema (CASSANDRA-4052) + * Avoid having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889) + * (cql3) Backwards compatibility for composite comparators in non-cql3-aware + clients (CASSANDRA-4093) + * (cql3) Fix order by for reversed queries (CASSANDRA-4160) + * (cql3) Add ReversedType support (CASSANDRA-4004) + * (cql3) Add timeuuid type (CASSANDRA-4194) + * (cql3) Minor fixes (CASSANDRA-4185) + * (cql3) Fix prepared statement in BATCH (CASSANDRA-4202) + * (cql3) Reduce the list of reserved keywords (CASSANDRA-4186) + * (cql3) Move max/min compaction thresholds to compaction strategy options + (CASSANDRA-4187) + * Fix exception during move when localhost is the only source (CASSANDRA-4200) + * (cql3) Allow paging through non-ordered partitioner results (CASSANDRA-3771) + * (cql3) Fix drop index (CASSANDRA-4192) + * (cql3) Don't return range ghosts anymore (CASSANDRA-3982) + * fix re-creating Keyspaces/ColumnFamilies with the same name as dropped + ones (CASSANDRA-4219) + * fix SecondaryIndex LeveledManifest save upon snapshot (CASSANDRA-4230) + * fix missing arrayOffset in FBUtilities.hash (CASSANDRA-4250) + * (cql3) Add name of parameters in CqlResultSet (CASSANDRA-4242) + * (cql3) Correctly validate order by queries (CASSANDRA-4246) + * rename stress to cassandra-stress for saner packaging (CASSANDRA-4256) + * Fix exception on colum metadata with non-string comparator (CASSANDRA-4269) + * Check for unknown/invalid compression options (CASSANDRA-4266) + * (cql3) Adds simple access to column timestamp and ttl (CASSANDRA-4217) + * (cql3) Fix range queries with secondary indexes (CASSANDRA-4257) + * Better error messages from improper input in cli (CASSANDRA-3865) + * Try to stop all compaction upon Keyspace or ColumnFamily drop (CASSANDRA-4221) + * (cql3) Allow keyspace properties to contain hyphens (CASSANDRA-4278) + * (cql3) Correctly validate keyspace access in create table (CASSANDRA-4296) + * Avoid deadlock in migration stage (CASSANDRA-3882) + * Take supercolumn names and deletion info into account in memtable throughput + (CASSANDRA-4264) + * Add back backward compatibility for old style replication factor (CASSANDRA-4294) + * Preserve compatibility with pre-1.1 index queries (CASSANDRA-4262) +Merged from 1.0: + * Fix super columns bug where cache is not updated (CASSANDRA-4190) + * fix maxTimestamp to include row tombstones (CASSANDRA-4116) + * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129) + * Avoids possible deadlock during bootstrap (CASSANDRA-4159) + * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128) + * stress tool to return appropriate exit code on failure (CASSANDRA-4188) + * fix compaction NPE when out of disk space and assertions disabled + (CASSANDRA-3985) * synchronize LCS getEstimatedTasks to avoid CME (CASSANDRA-4255) * ensure unique streaming session id's (CASSANDRA-4223) * kick off background compaction when min/max thresholds change @@@ -156,44 -8,15 +157,46 @@@ * improve ability of STCS.getBuckets to deal with 100s of 1000s of sstables, such as when convertinb back from LCS (CASSANDRA-4287) * Oversize integer in CQL throws NumberFormatException (CASSANDRA-4291) - * Set gc_grace on index CF to 0 (CASSANDRA-4314) + * fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195) + * Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419) - * Push the validation of secondary index values to the SecondaryIndexManager (CASSANDRA-4240) -1.0.10 - * fix maxTimestamp to include row tombstones (CASSANDRA-4116) + +1.1.0-final + * average a reduced liveRatio estimate with the previous one (CASSANDRA-4065) + * Allow KS and CF names up to 48 characters (CASSANDRA-4157) + * fix stress build (CASSANDRA-4140) + * add time remaining estimate to nodetool compactionstats (CASSANDRA-4167) + * (cql) fix NPE in cql3 ALTER TABLE (CASSANDRA-4163) + * (cql) Add support for CL.TWO and CL.THREE in CQL (CASSANDRA-4156) + * (cql) Fix type in CQL3 ALTER TABLE preventing update (CASSANDRA-4170) + * (cql) Throw invalid exception from CQL3 on obsolete options (CASSANDRA-4171) + * (cqlsh) fix recognizing uppercase SELECT keyword (CASSANDRA-4161) + * Pig: wide row support (CASSANDRA-3909) +Merged from 1.0: * avoid streaming empty files with bulk loader if sstablewriter errors out (CASSANDRA-3946) + + +1.1-rc1 + * Include stress tool in binary builds (CASSANDRA-4103) + * (Hadoop) fix wide row iteration when last row read was deleted + (CASSANDRA-4154) + * fix read_repair_chance to really default to 0.1 in the cli (CASSANDRA-4114) + * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042) + * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087) + * fix KEYS index from skipping results (CASSANDRA-3996) + * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076) + * make loadNewSStable preserve sstable version (CASSANDRA-4077) + * Respect 1.0 cache settings as much as possible when upgrading + (CASSANDRA-4088) + * relax path length requirement for sstable files when upgrading on + non-Windows platforms (CASSANDRA-4110) + * fix terminination of the stress.java when errors were encountered + (CASSANDRA-4128) + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037) + * Fix get_paged_slice (CASSANDRA-4136) + * CQL3: Support slice with exclusive start and stop (CASSANDRA-3785) +Merged from 1.0: * support PropertyFileSnitch in bulk loader (CASSANDRA-4145) * add auto_snapshot option allowing disabling snapshot before drop/truncate (CASSANDRA-3710) http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java index ba7360d,a7bd3fb..05fa70b --- a/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/PerColumnSecondaryIndex.java @@@ -55,9 -57,15 +57,15 @@@ public abstract class PerColumnSecondar * @param col all the column info */ public abstract void updateColumn(DecoratedKey<?> valueKey, ByteBuffer rowKey, IColumn col) throws IOException; - + public String getNameForSystemTable(ByteBuffer column) { - return getIndexName(); + return getIndexName(); } + + @Override + public boolean validate(Column column) + { + return column.value.remaining() < FBUtilities.MAX_UNSIGNED_SHORT; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/PerRowSecondaryIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/SecondaryIndex.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndex.java index 64a696d,3e38f28..3732086 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java @@@ -327,7 -319,9 +328,9 @@@ public abstract class SecondaryInde index.addColumnDef(cdef); index.validateOptions(); index.setBaseCfs(baseCfs); - + return index; } + + public abstract boolean validate(Column column); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java index ba066e2,ecdf8b1..e923575 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java @@@ -20,7 -20,7 +20,14 @@@ package org.apache.cassandra.db.index import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; --import java.util.concurrent.*; ++import java.util.concurrent.ConcurrentNavigableMap; ++import java.util.concurrent.ConcurrentSkipListMap; ++import java.util.concurrent.ExecutionException; ++import java.util.concurrent.Future; ++ ++import org.apache.commons.lang.StringUtils; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.ConfigurationException; @@@ -31,11 -31,13 +38,9 @@@ import org.apache.cassandra.dht.Abstrac import org.apache.cassandra.dht.LocalToken; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.io.sstable.SSTableReader; + import org.apache.cassandra.thrift.Column; -import org.apache.cassandra.thrift.IndexClause; import org.apache.cassandra.thrift.IndexExpression; import org.apache.cassandra.utils.ByteBufferUtil; --import org.apache.commons.lang.StringUtils; --import org.slf4j.Logger; --import org.slf4j.LoggerFactory; /** * Manages all the indexes associated with a given CFS @@@ -590,20 -556,14 +595,26 @@@ public class SecondaryIndexManage //We currently don't support searching across multiple index types if (indexSearchers.size() > 1) throw new RuntimeException("Unable to search across multiple secondary index types"); - - - return indexSearchers.get(0).search(clause, range, dataFilter); + + + return indexSearchers.get(0).search(clause, range, maxResults, dataFilter, maxIsColumns); + } + + public void setIndexBuilt(Collection<ByteBuffer> indexes) + { + for (ByteBuffer colName : indexes) + indexesByColumn.get(colName).setIndexBuilt(colName); + } + + public void setIndexRemoved(Collection<ByteBuffer> indexes) + { + for (ByteBuffer colName : indexes) + indexesByColumn.get(colName).setIndexRemoved(colName); } + + public boolean validate(Column column) + { + SecondaryIndex index = getIndexForColumn(column.name); + return index != null ? index.validate(column) : true; + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/src/java/org/apache/cassandra/thrift/ThriftValidation.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/070c1d8d/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java index 9c131f9,c025037..af0e668 --- a/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java +++ b/test/unit/org/apache/cassandra/thrift/ThriftValidationTest.java @@@ -21,23 -21,18 +21,20 @@@ package org.apache.cassandra.thrift */ - import java.nio.ByteBuffer; - import org.junit.Test; -import org.apache.cassandra.CleanupHelper; +import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ConfigurationException; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.KSMetaData; import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.locator.NetworkTopologyStrategy; - import org.apache.cassandra.utils.FBUtilities; -public class ThriftValidationTest extends CleanupHelper +public class ThriftValidationTest extends SchemaLoader { @Test(expected=InvalidRequestException.class) public void testValidateCommutativeWithStandard() throws InvalidRequestException