Updated Branches: refs/heads/1.4.5-SNAPSHOT 0d0bc4643 -> 3b08c717e refs/heads/1.5.1-SNAPSHOT 7eb838e3c -> dc70a30d6 refs/heads/1.6.0-SNAPSHOT 02275812b -> a425f6a50 refs/heads/master 403815f55 -> bdac06f65
ACCUMULO-1978 fixed compaction and flush of metadata table and added test Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/02275812 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/02275812 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/02275812 Branch: refs/heads/master Commit: 02275812b6ed7cec6c4923183f2f75011f4c4f4a Parents: 07cdc57 Author: Keith Turner <ktur...@apache.org> Authored: Thu Dec 12 13:08:34 2013 -0500 Committer: Keith Turner <ktur...@apache.org> Committed: Thu Dec 12 13:08:34 2013 -0500 ---------------------------------------------------------------------- .../java/org/apache/accumulo/master/Master.java | 11 ++++-- .../accumulo/master/tableOps/CompactRange.java | 15 +++++--- .../accumulo/test/functional/MetadataIT.java | 38 ++++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/02275812/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index eb1a492..bd6ee27 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -74,6 +74,7 @@ import org.apache.accumulo.core.master.thrift.TabletServerStatus; import org.apache.accumulo.core.master.thrift.TabletSplit; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.security.Authorizations; @@ -708,12 +709,18 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt try { Connector conn = getConnector(); - Scanner scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)); + Scanner scanner; + if (tableId.equals(MetadataTable.ID)) { + scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY)); + scanner.setRange(MetadataSchema.TabletsSection.getRange()); + } else { + scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)); + scanner.setRange(new KeyExtent(new Text(tableId), null, ByteBufferUtil.toText(startRow)).toMetadataRange()); + } TabletsSection.ServerColumnFamily.FLUSH_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.fetchColumnFamily(LogColumnFamily.NAME); - scanner.setRange(new KeyExtent(new Text(tableId), null, ByteBufferUtil.toText(startRow)).toMetadataRange()); RowIterator ri = new RowIterator(scanner); http://git-wip-us.apache.org/repos/asf/accumulo/blob/02275812/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java index e8f65d2..df474b3 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java @@ -43,6 +43,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.master.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.MapCounter; @@ -97,14 +98,18 @@ class CompactionDriver extends MasterRepo { MapCounter<TServerInstance> serversToFlush = new MapCounter<TServerInstance>(); Connector conn = master.getConnector(); - Scanner scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)); - Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange(); + Scanner scanner; - if (tableId.equals(MetadataTable.ID)) - range = range.clip(new Range(RootTable.EXTENT.getMetadataEntry(), false, null, true)); + if (tableId.equals(MetadataTable.ID)) { + scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY)); + scanner.setRange(MetadataSchema.TabletsSection.getRange()); + } else { + scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)); + Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange(); + scanner.setRange(range); + } - scanner.setRange(range); TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); http://git-wip-us.apache.org/repos/asf/accumulo/blob/02275812/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java index 92e0d44..6c9f27a 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java @@ -20,7 +20,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.util.Collections; +import java.util.HashSet; import java.util.Map.Entry; +import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -36,11 +38,47 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.hadoop.io.Text; +import org.junit.Assert; import org.junit.Test; public class MetadataIT extends SimpleMacIT { @Test(timeout = 60 * 1000) + public void testFlushAndCompact() throws Exception { + Connector c = getConnector(); + + // create a table to write some data to metadata table + c.tableOperations().create(super.getTableNames(1)[0]); + + Scanner rootScanner = c.createScanner(RootTable.NAME, Authorizations.EMPTY); + rootScanner.setRange(MetadataSchema.TabletsSection.getRange()); + rootScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME); + + Set<String> files1 = new HashSet<String>(); + for (Entry<Key,Value> entry : rootScanner) + files1.add(entry.getKey().getColumnQualifier().toString()); + + c.tableOperations().flush(MetadataTable.NAME, null, null, true); + + Set<String> files2 = new HashSet<String>(); + for (Entry<Key,Value> entry : rootScanner) + files2.add(entry.getKey().getColumnQualifier().toString()); + + // flush of metadata table should change file set in root table + Assert.assertTrue(files2.size() > 0); + Assert.assertNotEquals(files1, files2); + + c.tableOperations().compact(MetadataTable.NAME, null, null, false, true); + + Set<String> files3 = new HashSet<String>(); + for (Entry<Key,Value> entry : rootScanner) + + files3.add(entry.getKey().getColumnQualifier().toString()); + // compaction of metadata table should change file set in root table + Assert.assertNotEquals(files2, files3); + } + + @Test(timeout = 60 * 1000) public void mergeMeta() throws Exception { Connector c = getConnector(); SortedSet<Text> splits = new TreeSet<Text>();
