add nodetool rebuild_index patch by Vijay; reviewed by jbellis for CASSANDRA-3583
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/12d26f86 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/12d26f86 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/12d26f86 Branch: refs/heads/cassandra-1.0 Commit: 12d26f869d966a842e6b01b13e338051c9c85f68 Parents: e5be952 Author: Jonathan Ellis <jbel...@apache.org> Authored: Thu Feb 9 17:18:18 2012 -0600 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Thu Feb 9 17:18:18 2012 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/db/ColumnFamilyStore.java | 34 +++++++++++++-- src/java/org/apache/cassandra/db/Table.java | 5 ++ .../apache/cassandra/db/filter/QueryFilter.java | 2 +- .../apache/cassandra/db/index/SecondaryIndex.java | 10 ++++ .../cassandra/db/index/SecondaryIndexManager.java | 24 ++++++++++- .../apache/cassandra/service/StorageService.java | 5 ++ .../cassandra/service/StorageServiceMBean.java | 6 ++- src/java/org/apache/cassandra/tools/NodeCmd.java | 11 +++++ src/java/org/apache/cassandra/tools/NodeProbe.java | 5 ++ 10 files changed, 96 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 8b97c73..63323e4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.1-dev + * add nodetool rebuild_index (CASSANDRA-3583) * add nodetool rangekeysample (CASSANDRA-2917) * Fix streaming too much data during move operations (CASSANDRA-3639) * Nodetool and CLI connect to localhost by default (CASSANDRA-3568) http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 85698d6..bf4a000 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -28,10 +28,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import javax.management.*; -import com.google.common.collect.AbstractIterator; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; +import com.google.common.collect.*; import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.service.CacheService; @@ -512,6 +509,35 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean logger.info("Done loading load new SSTables for " + table.name + "/" + columnFamily); } + public static void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames) + { + ColumnFamilyStore cfs = Table.open(ksName).getColumnFamilyStore(cfName); + + SortedSet<ByteBuffer> indexes = new TreeSet<ByteBuffer>(cfs.metadata.comparator); + if (idxNames.length == 0) + indexes.addAll(cfs.indexManager.getIndexedColumns()); + for (String idxName : idxNames) + indexes.add(cfs.indexManager.getColumnByIdxName(idxName)); + + Collection<SSTableReader> sstables = cfs.getSSTables(); + try + { + cfs.indexManager.setIndexRemoved(indexes); + SSTableReader.acquireReferences(sstables); + logger.info(String.format("User Requested secondary index re-build for %s/%s indexes", ksName, cfName)); + cfs.indexManager.maybeBuildSecondaryIndexes(sstables, indexes); + cfs.indexManager.setIndexBuilt(indexes); + } + catch (IOException e) + { + throw new IOError(e); + } + finally + { + SSTableReader.releaseReferences(sstables); + } + } + /** * @return the name of the column family */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/db/Table.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Table.java b/src/java/org/apache/cassandra/db/Table.java index 1fdb3ae..ce38217 100644 --- a/src/java/org/apache/cassandra/db/Table.java +++ b/src/java/org/apache/cassandra/db/Table.java @@ -486,6 +486,11 @@ public class Table return replicationStrategy; } + /** + * @param key row to index + * @param cfs ColumnFamily to index row in + * @param indexedColumns columns to index, in comparator order + */ public static void indexRow(DecoratedKey<?> key, ColumnFamilyStore cfs, SortedSet<ByteBuffer> indexedColumns) { if (logger.isDebugEnabled()) http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/db/filter/QueryFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/QueryFilter.java b/src/java/org/apache/cassandra/db/filter/QueryFilter.java index b50e5d1..eb9813f 100644 --- a/src/java/org/apache/cassandra/db/filter/QueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/QueryFilter.java @@ -180,7 +180,7 @@ public class QueryFilter * @return a QueryFilter object that will return columns matching the given names * @param key the row to slice * @param path path to the level to slice at (CF or SuperColumn) - * @param columns the column names to restrict the results to + * @param columns the column names to restrict the results to, sorted in comparator order */ public static QueryFilter getNamesFilter(DecoratedKey<?> key, QueryPath path, SortedSet<ByteBuffer> columns) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/db/index/SecondaryIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java index 5f7d7c6..9571073 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java @@ -96,6 +96,16 @@ public abstract class SecondaryIndex return SystemTable.isIndexBuilt(baseCfs.table.name, getNameForSystemTable(columnName)); } + public void setIndexBuilt(ByteBuffer columnName) + { + SystemTable.setIndexBuilt(baseCfs.table.name, getNameForSystemTable(columnName)); + } + + public void setIndexRemoved(ByteBuffer columnName) + { + SystemTable.setIndexRemoved(baseCfs.table.name, getNameForSystemTable(columnName)); + } + /** * Called at query time * Creates a implementation specific searcher instance for this index type http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java index 610bd2a..3758e9b 100644 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java @@ -103,7 +103,7 @@ public class SecondaryIndexManager * Caller must acquire and release references to the sstables used here. * * @param sstables the data to build from - * @param columns the list of columns to index + * @param columns the list of columns to index, ordered by comparator * @throws IOException */ public void maybeBuildSecondaryIndexes(Collection<SSTableReader> sstables, SortedSet<ByteBuffer> columns) throws IOException @@ -309,6 +309,16 @@ public class SecondaryIndexManager return indexList; } + public ByteBuffer getColumnByIdxName(String idxName) + { + for (Map.Entry<ByteBuffer, SecondaryIndex> entry : indexesByColumn.entrySet()) + { + if (entry.getValue().getIndexName().equals(idxName)) + return entry.getKey(); + } + throw new RuntimeException("Unknown Index Name: " + idxName); + } + /** * @return all CFS from indexes which use a backing CFS internally (KEYS) */ @@ -578,4 +588,16 @@ public class SecondaryIndexManager return indexSearchers.get(0).search(clause, range, maxResults, dataFilter, maxIsColumns); } + + public void setIndexBuilt(Collection<ByteBuffer> indexes) + { + for (ByteBuffer colName : indexes) + indexesByColumn.get(colName).setIndexBuilt(colName); + } + + public void setIndexRemoved(Collection<ByteBuffer> indexes) + { + for (ByteBuffer colName : indexes) + indexesByColumn.get(colName).setIndexBuilt(colName); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 4c79373..c1681b9 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -2904,4 +2904,9 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe sampledKeys.add(key.getToken().toString()); return sampledKeys; } + + public void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames) + { + ColumnFamilyStore.rebuildSecondaryIndex(ksName, cfName, idxNames); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/service/StorageServiceMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index e8f0dd7..6af63b7 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -373,5 +373,9 @@ public interface StorageServiceMBean * @return set of Tokens as Strings */ public List<String> getRangeKeySample(); - + + /** + * rebuild the specified indexes + */ + public void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/tools/NodeCmd.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java index 25645c4..2b19074 100644 --- a/src/java/org/apache/cassandra/tools/NodeCmd.java +++ b/src/java/org/apache/cassandra/tools/NodeCmd.java @@ -120,6 +120,7 @@ public class NodeCmd VERSION, DESCRIBERING, RANGEKEYSAMPLE, + REBUILD_INDEX, } @@ -173,6 +174,7 @@ public class NodeCmd addCmdHelp(header, "getcompactionthreshold <keyspace> <cfname>", "Print min and max compaction thresholds for a given column family"); addCmdHelp(header, "cfhistograms <keyspace> <cfname>", "Print statistic histograms for a given column family"); addCmdHelp(header, "refresh <keyspace> <cf-name>", "Load newly placed SSTables to the system without restart."); + addCmdHelp(header, "rebuild_index <keyspace> <cf-name> <idx1,idx1>", "a full rebuilds of native secondry index for a given column family. IndexNameExample: Standard3.IdxName,Standard3.IdxName1"); // Three args addCmdHelp(header, "getendpoints <keyspace> <cf> <key>", "Print the end points that owns the key"); @@ -771,6 +773,15 @@ public class NodeCmd probe.loadNewSSTables(arguments[0], arguments[1]); break; + case REBUILD_INDEX: + if (arguments.length < 2) { badUse("rebuild_index requires ks and cf args"); } + if (arguments.length >= 3) + probe.rebuildIndex(arguments[0], arguments[1], arguments[2].split(",")); + else + probe.rebuildIndex(arguments[0], arguments[1]); + + break; + case GOSSIPINFO : nodeCmd.printGossipInfo(System.out); break; case STOP: http://git-wip-us.apache.org/repos/asf/cassandra/blob/12d26f86/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 469c2a5..46d4c63 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -619,6 +619,11 @@ public class NodeProbe ssProxy.loadNewSSTables(ksName, cfName); } + public void rebuildIndex(String ksName, String cfName, String... idxNames) + { + ssProxy.rebuildSecondaryIndex(ksName, cfName, idxNames); + } + public String getGossipInfo() { return fdProxy.getAllEndpointStates();