Repository: accumulo Updated Branches: refs/heads/1.6 a95127d1d -> 46958bf7b refs/heads/1.7 6cc5ab56b -> a706b019a refs/heads/master b4cd908a3 -> 2c8088e99
ACCUMULO-4119 Use DataFileValue consistently * Ensure DataFileValue encodes/decodes with UTF-8 * Ensure usage of DataFileValue instead of manual parsing * Provide more encoding utility methods (String, Value) Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c462001a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c462001a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c462001a Branch: refs/heads/1.6 Commit: c462001a71cf4e4e5b9b0865cb6d0036bf4d84c7 Parents: a95127d Author: Christopher Tubbs <ctubb...@apache.org> Authored: Wed Jan 27 19:08:56 2016 -0500 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Tue Feb 2 18:24:28 2016 -0500 ---------------------------------------------------------------------- .../core/metadata/schema/DataFileValue.java | 18 ++++++++++++--- .../org/apache/accumulo/core/util/Merge.java | 8 ++----- .../apache/accumulo/server/util/CloneTest.java | 24 ++++++++++++++------ .../metadata/MetadataBatchScanTest.java | 3 ++- 4 files changed, 36 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java index cebe041..dfa1114 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java @@ -16,6 +16,10 @@ */ package org.apache.accumulo.core.metadata.schema; +import static com.google.common.base.Charsets.UTF_8; + +import org.apache.accumulo.core.data.Value; + public class DataFileValue { private long size; private long numEntries; @@ -34,7 +38,7 @@ public class DataFileValue { } public DataFileValue(byte[] encodedDFV) { - String[] ba = new String(encodedDFV).split(","); + String[] ba = new String(encodedDFV, UTF_8).split(","); size = Long.parseLong(ba[0]); numEntries = Long.parseLong(ba[1]); @@ -62,9 +66,17 @@ public class DataFileValue { } public byte[] encode() { + return encodeAsString().getBytes(UTF_8); + } + + public String encodeAsString() { if (time >= 0) - return ("" + size + "," + numEntries + "," + time).getBytes(); - return ("" + size + "," + numEntries).getBytes(); + return ("" + size + "," + numEntries + "," + time); + return ("" + size + "," + numEntries); + } + + public Value encodeAsValue() { + return new Value(encode()); } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/core/src/main/java/org/apache/accumulo/core/util/Merge.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/Merge.java b/core/src/main/java/org/apache/accumulo/core/util/Merge.java index c3bc678..c5e3e8e 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java +++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java @@ -16,8 +16,6 @@ */ package org.apache.accumulo.core.util; -import static com.google.common.base.Charsets.UTF_8; - import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -34,6 +32,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; @@ -238,10 +237,7 @@ public class Merge { Entry<Key,Value> entry = iterator.next(); Key key = entry.getKey(); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { - String[] sizeEntries = new String(entry.getValue().get(), UTF_8).split(","); - if (sizeEntries.length == 2) { - tabletSize += Long.parseLong(sizeEntries[0]); - } + tabletSize += new DataFileValue(entry.getValue().get()).getSize(); } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) { KeyExtent extent = new KeyExtent(key.getRow(), entry.getValue()); return new Size(extent, tabletSize); http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java index ebb83bf..74d5e64 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java @@ -16,11 +16,12 @@ */ package org.apache.accumulo.server.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.util.HashSet; import java.util.Map.Entry; -import junit.framework.TestCase; - import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; @@ -32,13 +33,16 @@ import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.io.Text; +import org.junit.Test; -public class CloneTest extends TestCase { +public class CloneTest { + @Test public void testNoFiles() throws Exception { MockInstance mi = new MockInstance(); Connector conn = mi.getConnector("", new PasswordToken("")); @@ -67,6 +71,7 @@ public class CloneTest extends TestCase { } + @Test public void testFilesChange() throws Exception { MockInstance mi = new MockInstance(); Connector conn = mi.getConnector("", new PasswordToken("")); @@ -76,7 +81,7 @@ public class CloneTest extends TestCase { TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes())); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value("/default_tablet".getBytes())); - mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", "1,200"); + mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", new DataFileValue(1, 200).encodeAsString()); BatchWriter bw1 = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); @@ -90,7 +95,7 @@ public class CloneTest extends TestCase { Mutation mut2 = new Mutation(ke.getMetadataEntry()); mut2.putDelete(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf"); - mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", "2,300"); + mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", new DataFileValue(2, 300).encodeAsString()); bw1.addMutation(mut2); bw1.flush(); @@ -119,6 +124,7 @@ public class CloneTest extends TestCase { } // test split where files of children are the same + @Test public void testSplit1() throws Exception { MockInstance mi = new MockInstance(); Connector conn = mi.getConnector("", new PasswordToken("")); @@ -161,6 +167,7 @@ public class CloneTest extends TestCase { } // test split where files of children differ... like majc and split occurred + @Test public void testSplit2() throws Exception { MockInstance mi = new MockInstance(); Connector conn = mi.getConnector("", new PasswordToken("")); @@ -226,12 +233,13 @@ public class CloneTest extends TestCase { TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes())); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes())); - mut.put(DataFileColumnFamily.NAME.toString(), file, "10,200"); + mut.put(DataFileColumnFamily.NAME.toString(), file, new DataFileValue(10, 200).encodeAsString()); return mut; } // test two tablets splitting into four + @Test public void testSplit3() throws Exception { MockInstance mi = new MockInstance(); Connector conn = mi.getConnector("", new PasswordToken("")); @@ -278,6 +286,7 @@ public class CloneTest extends TestCase { } // test cloned marker + @Test public void testClonedMarker() throws Exception { MockInstance mi = new MockInstance(); @@ -343,6 +352,7 @@ public class CloneTest extends TestCase { } // test two tablets splitting into four + @Test public void testMerge() throws Exception { MockInstance mi = new MockInstance(); Connector conn = mi.getConnector("", new PasswordToken("")); @@ -360,7 +370,7 @@ public class CloneTest extends TestCase { bw1.addMutation(deleteTablet("0", "m", null, "/d1", "/d1/file1")); Mutation mut = createTablet("0", null, null, "/d2", "/d2/file2"); - mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", "10,200"); + mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", new DataFileValue(10, 200).encodeAsString()); bw1.addMutation(mut); bw1.flush(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java index d764d35..07bfdc7 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; @@ -108,7 +109,7 @@ public class MetadataBatchScanTest { TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8))); for (int i = 0; i < 5; i++) { - mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new Value("10000,1000000".getBytes(UTF_8))); + mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"), new DataFileValue(10000, 1000000).encodeAsValue()); } bw.addMutation(mut);
