add --migrate optionto sstableupgrade and sstablescrub patch by Tyler Hobbs; reviewed by Jeremiah Jordan for CASSANDRA-5831
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d45d2bd Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d45d2bd Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d45d2bd Branch: refs/heads/trunk Commit: 7d45d2bde3724a15a9e0f68bb047a41853ccd340 Parents: 27943b1 Author: Jonathan Ellis <jbel...@apache.org> Authored: Wed Aug 7 18:03:29 2013 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Wed Aug 7 18:03:29 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/db/SystemTable.java | 52 +++++++++++++++----- .../cassandra/tools/StandaloneScrubber.java | 19 ++++++- .../cassandra/tools/StandaloneUpgrader.java | 22 ++++++++- 4 files changed, 80 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 25c0217..859e7f9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.2.9 + * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831) * fix bulk-loading compressed sstables (CASSANDRA-5820) * (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter (CASSANDRA-5824) http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/db/SystemTable.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/SystemTable.java b/src/java/org/apache/cassandra/db/SystemTable.java index 4c74df5..318b34d 100644 --- a/src/java/org/apache/cassandra/db/SystemTable.java +++ b/src/java/org/apache/cassandra/db/SystemTable.java @@ -147,6 +147,7 @@ public class SystemTable ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF); if (oldStatusCfs.getSSTables().size() > 0) { + logger.info("Old system data found in {}.{}; migrating to new format in {}.{}", Table.SYSTEM_KS, OLD_STATUS_CF, Table.SYSTEM_KS, LOCAL_CF); SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance); cols.add(ByteBufferUtil.bytes("ClusterName")); cols.add(ByteBufferUtil.bytes("Token")); @@ -428,24 +429,51 @@ public class SystemTable ex.initCause(err); throw ex; } - ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF); - String req = "SELECT cluster_name FROM system.%s WHERE key='%s'"; - UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY)); + String savedClusterName; - if (result.isEmpty() || !result.one().has("cluster_name")) + // See if there is still data in System.LocationInfo, indicating that the system data has not yet been + // upgraded by SystemTable.upgradeSystemData() + ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF); + if (oldStatusCfs.getSSTables().size() > 0) + { + logger.debug("Detected system data in {}.{}, checking saved cluster name", Table.SYSTEM_KS, OLD_STATUS_CF); + SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance); + cols.add(ByteBufferUtil.bytes("ClusterName")); + QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes("L")), new QueryPath(OLD_STATUS_CF), cols); + ColumnFamily oldCf = oldStatusCfs.getColumnFamily(filter); + try + { + savedClusterName = ByteBufferUtil.string(oldCf.getColumn(ByteBufferUtil.bytes("ClusterName")).value()); + } + catch (CharacterCodingException e) + { + throw new RuntimeException(e); + } + } + else { - // this is a brand new node - if (!cfs.getSSTables().isEmpty()) - throw new ConfigurationException("Found system table files, but they couldn't be loaded!"); + ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF); - // no system files. this is a new node. - req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')"; - processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName())); - return; + String req = "SELECT cluster_name FROM system.%s WHERE key='%s'"; + UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY)); + + if (result.isEmpty() || !result.one().has("cluster_name")) + { + + // this is a brand new node + if (!cfs.getSSTables().isEmpty()) + throw new ConfigurationException("Found system table files, but they couldn't be loaded!"); + + // no system files. this is a new node. + req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')"; + processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName())); + return; + } + + savedClusterName = result.one().getString("cluster_name"); } - String savedClusterName = result.one().getString("cluster_name"); if (!DatabaseDescriptor.getClusterName().equals(savedClusterName)) throw new ConfigurationException("Saved cluster name " + savedClusterName + " != configured name " + DatabaseDescriptor.getClusterName()); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneScrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index 0542756..d59a7d1 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -49,15 +49,30 @@ public class StandaloneScrubber private static final String DEBUG_OPTION = "debug"; private static final String HELP_OPTION = "help"; private static final String MANIFEST_CHECK_OPTION = "manifest-check"; + private static final String MIGRATE_OPTION = "migrate"; public static void main(String args[]) throws IOException { Options options = Options.parseArgs(args); try { + OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug); + // Migrate sstables from pre-#2749 to the correct location if (Directories.sstablesNeedsMigration()) + { + if (!options.migrate) + { + System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " + + "must be performed to the 1.1+ format for directories and filenames. Re-run " + + TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " + + "migrate *all* keyspaces and column families to the new layout."); + System.exit(1); + } + handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " + + "will be migrated to the 1.1+ format."); Directories.migrateSSTables(); + } // load keyspace descriptions. DatabaseDescriptor.loadSchemas(); @@ -72,7 +87,6 @@ public class StandaloneScrubber ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cfName); String snapshotName = "pre-scrub-" + System.currentTimeMillis(); - OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug); Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true); List<SSTableReader> sstables = new ArrayList<SSTableReader>(); @@ -184,6 +198,7 @@ public class StandaloneScrubber public boolean debug; public boolean verbose; public boolean manifestCheckOnly; + public boolean migrate; private Options(String tableName, String cfName) { @@ -222,6 +237,7 @@ public class StandaloneScrubber opts.debug = cmd.hasOption(DEBUG_OPTION); opts.verbose = cmd.hasOption(VERBOSE_OPTION); opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION); + opts.migrate = cmd.hasOption(MIGRATE_OPTION); return opts; } @@ -246,6 +262,7 @@ public class StandaloneScrubber options.addOption("v", VERBOSE_OPTION, "verbose output"); options.addOption("h", HELP_OPTION, "display this help message"); options.addOption("m", MANIFEST_CHECK_OPTION, "only check and repair the leveled manifest, without actually scrubbing the sstables"); + options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure"); return options; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java index 357e99c..16bf588 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java +++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java @@ -47,12 +47,30 @@ public class StandaloneUpgrader private static final String TOOL_NAME = "sstableupgrade"; private static final String DEBUG_OPTION = "debug"; private static final String HELP_OPTION = "help"; + private static final String MIGRATE_OPTION = "migrate"; public static void main(String args[]) throws IOException { Options options = Options.parseArgs(args); try { + OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug); + // Migrate sstables from pre-#2749 to the correct location + if (Directories.sstablesNeedsMigration()) + { + if (!options.migrate) + { + System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " + + "must be performed to the 1.1+ format for directories and filenames. Re-run " + + TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " + + "migrate *all* keyspaces and column families to the new layout."); + System.exit(1); + } + handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " + + "will be migrated to the 1.1+ format."); + Directories.migrateSSTables(); + } + // load keyspace descriptions. DatabaseDescriptor.loadSchemas(); @@ -64,7 +82,6 @@ public class StandaloneUpgrader Table table = Table.openWithoutSSTables(options.keyspace); ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cf); - OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug); Directories.SSTableLister lister = cfs.directories.sstableLister(); if (options.snapshot != null) lister.onlyBackups(true).snapshots(options.snapshot); @@ -137,6 +154,7 @@ public class StandaloneUpgrader public final String snapshot; public boolean debug; + public boolean migrate; private Options(String keyspace, String cf, String snapshot) { @@ -176,6 +194,7 @@ public class StandaloneUpgrader Options opts = new Options(keyspace, cf, snapshot); opts.debug = cmd.hasOption(DEBUG_OPTION); + opts.migrate = cmd.hasOption(MIGRATE_OPTION); return opts; } @@ -197,6 +216,7 @@ public class StandaloneUpgrader { CmdLineOptions options = new CmdLineOptions(); options.addOption(null, DEBUG_OPTION, "display stack traces"); + options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure"); options.addOption("h", HELP_OPTION, "display this help message"); return options; }