This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e12fcbefb6 Refactor Root Tablet serialization code (#2718)
e12fcbefb6 is described below
commit e12fcbefb6f2979c3aeabd5597f3799c107df1ba
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Fri May 27 11:26:59 2022 +0000
Refactor Root Tablet serialization code (#2718)
* Remove static state from RootTabletMetadata and RootGcCandidates
* Replaced static from() methods in RootTabletMetadata and RootGcCandidates
with constructors
* Added comment to AccumuloDataVersion javadoc
* Store entries natively in the simpler string-based multi-map, rather
than converting to key/value pairs and back again. Only convert to
key/value pairs when needed.
* Get rid of `new GsonBuilder().create()`. Even Gson's own javadoc
recommends `new Gson()` if all you want is the defaults.
Co-authored-by: Christopher Tubbs <ctubb...@apache.org>
---
.../core/clientImpl/bulk/BulkSerialize.java | 3 +-
.../schema/ExternalCompactionFinalState.java | 3 +-
.../schema/ExternalCompactionMetadata.java | 3 +-
.../core/metadata/schema/RootTabletMetadata.java | 224 +++++++++------------
.../core/metadata/schema/TabletMetadata.java | 2 +-
.../core/metadata/schema/TabletsMetadata.java | 9 +-
.../accumulo/server/AccumuloDataVersion.java | 2 +-
.../accumulo/server/init/ZooKeeperInitializer.java | 32 ++-
.../accumulo/server/metadata/RootGcCandidates.java | 106 +++++-----
.../server/metadata/RootTabletMutatorImpl.java | 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 17 +-
.../manager/state/RootTabletStateStoreTest.java | 11 +-
.../accumulo/test/CountNameNodeOpsBulkIT.java | 3 +-
13 files changed, 198 insertions(+), 219 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
index 82fb70bc1b..f20efaa9ef 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkSerialize.java
@@ -130,8 +130,7 @@ public class BulkSerialize {
final Path renamingFile = new Path(bulkDir, Constants.BULK_RENAME_FILE);
try (OutputStream fsOut = output.create(renamingFile);
BufferedWriter writer = new BufferedWriter(new
OutputStreamWriter(fsOut))) {
- Gson gson = new GsonBuilder().create();
- gson.toJson(oldToNewNameMap, writer);
+ new Gson().toJson(oldToNewNameMap, writer);
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
index 4a01d246a9..6e2aaa15ed 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
@@ -27,11 +27,10 @@ import org.apache.hadoop.io.Text;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
public class ExternalCompactionFinalState {
- private static final Gson GSON = new GsonBuilder().create();
+ private static final Gson GSON = new Gson();
public enum FinalState {
FINISHED, FAILED
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index b4a11c8084..fcd6bf2c3f 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -33,11 +33,10 @@ import
org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.hadoop.fs.Path;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
public class ExternalCompactionMetadata {
- private static final Gson GSON = new GsonBuilder().create();
+ private static final Gson GSON = new Gson();
private final Set<StoredTabletFile> jobFiles;
private final Set<StoredTabletFile> nextFiles;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
index c62b13145d..bbd85b6305 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
@@ -18,183 +18,153 @@
*/
package org.apache.accumulo.core.metadata.schema;
+import static com.google.common.base.Preconditions.checkArgument;
import static java.nio.charset.StandardCharsets.UTF_8;
-import java.util.Arrays;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetDecoder;
+import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.EnumSet;
-import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.TreeMap;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
-import org.apache.accumulo.core.client.admin.TimeType;
-import org.apache.accumulo.core.data.ArrayByteSequence;
-import org.apache.accumulo.core.data.ByteSequence;
-import org.apache.accumulo.core.data.ColumnUpdate;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.RootTable;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
-import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
-import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
-import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
/**
- * Serializes the root tablet metadata as Json using Accumulo's standard
metadata table schema.
+ * This class is used to serialize and deserialize root tablet metadata using
GSon. The only data
+ * stored about the Root Table is the COLUMN_FAMILY, COLUMN_QUALIFIER and
VALUE.
+ *
+ * @since 2.1.0
*/
public class RootTabletMetadata {
- private static final Gson GSON = new GsonBuilder().create();
-
- private static final ByteSequence CURR_LOC_FAM =
- new ArrayByteSequence(CurrentLocationColumnFamily.STR_NAME);
- private static final ByteSequence FUTURE_LOC_FAM =
- new ArrayByteSequence(FutureLocationColumnFamily.STR_NAME);
+ private static final Logger log =
LoggerFactory.getLogger(RootTabletMetadata.class);
+ private static final CharsetDecoder UTF8_error_detecting_decoder =
UTF_8.newDecoder();
+ private static final Predicate<Entry<String,TreeMap<String,String>>>
isLocationCF = e -> {
+ String fam = e.getKey();
+ return fam.equals(CurrentLocationColumnFamily.STR_NAME)
+ || fam.equals(FutureLocationColumnFamily.STR_NAME);
+ };
- private TreeMap<Key,Value> entries = new TreeMap<>();
+ // JSON Mapping Version 1. Released with Accumulo version 2.1.0
+ private static final int VERSION = 1;
// This class is used to serialize and deserialize root tablet metadata
using GSon. Any changes to
// this class must consider persisted data.
- private static class GSonData {
- int version = 1;
+ private static class Data {
+ private final int version;
+
+ /*
+ * The data is mapped using Strings as follows:
+ *
+ * TreeMap<column_family, TreeMap<column_qualifier, value>>
+ */
+ private final TreeMap<String,TreeMap<String,String>> columnValues;
+
+ public Data(int version, TreeMap<String,TreeMap<String,String>>
columnValues) {
+ this.version = version;
+ this.columnValues = columnValues;
+ }
+ }
- // Map<column_family, Map<column_qualifier, value>>
- Map<String,Map<String,String>> columnValues;
+ /**
+ * The expectation is that all data stored in the root tablet can be
converted to UTF8. This
+ * method checks to ensure the byte sequence can be converted from byte[] to
UTF8 to byte[] w/o
+ * data corruption. Not all byte arrays can be converted to UTF8.
+ */
+ private static String bytesToUtf8(byte[] byteSequence) {
+ try {
+ return
UTF8_error_detecting_decoder.decode(ByteBuffer.wrap(byteSequence)).toString();
+ } catch (CharacterCodingException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ private final Gson gson = new Gson();
+ private final Data data;
+
+ public RootTabletMetadata(String json) {
+ log.debug("Creating root tablet metadata from stored JSON: {}", json);
+ this.data = gson.fromJson(json, Data.class);
+ checkArgument(data.version == VERSION, "Invalid Root Table Metadata JSON
version %s",
+ data.version);
+ data.columnValues.forEach((fam, qualVals) -> {
+ checkArgument(!fam.isBlank(), "Blank column family in %s",
data.columnValues);
+ checkArgument(!qualVals.isEmpty(), "No columns in family %s", fam);
+ });
+ }
+
+ public RootTabletMetadata() {
+ this.data = new Data(VERSION, new TreeMap<>());
}
/**
- * Apply a metadata table mutation to update internal json.
+ * Apply a metadata table mutation to update internal entries.
*/
public void update(Mutation m) {
- Preconditions.checkArgument(new
Text(m.getRow()).equals(RootTable.EXTENT.toMetaRow()));
+ checkArgument(new Text(m.getRow()).equals(RootTable.EXTENT.toMetaRow()),
+ "Invalid Root Table Row " + new Text(m.getRow()));
m.getUpdates().forEach(cup -> {
- Preconditions.checkArgument(!cup.hasTimestamp());
- Preconditions.checkArgument(cup.getColumnVisibility().length == 0);
+ checkArgument(!cup.hasTimestamp(), "Root Table timestamp must be
empty.");
+ checkArgument(cup.getColumnVisibility().length == 0, "Root Table
visibility must be empty.");
});
- for (ColumnUpdate cup : m.getUpdates()) {
- Key newKey = new Key(m.getRow(), cup.getColumnFamily(),
cup.getColumnQualifier(),
- cup.getColumnVisibility(), 1, false, false);
-
+ m.getUpdates().forEach(cup -> {
+ String fam = bytesToUtf8(cup.getColumnFamily());
+ String qual = bytesToUtf8(cup.getColumnQualifier());
+ String val = bytesToUtf8(cup.getValue());
if (cup.isDeleted()) {
- entries.remove(newKey);
+ data.columnValues.computeIfPresent(fam, (key, qualVals) -> {
+ qualVals.remove(qual);
+ return qualVals.isEmpty() ? null : qualVals;
+ });
} else {
- entries.put(newKey, new Value(cup.getValue()));
+ data.columnValues.computeIfAbsent(fam, k -> new TreeMap<>()).put(qual,
val);
}
- }
+ });
// Ensure there is ever only one location
- long locsSeen = entries.keySet().stream().map(Key::getColumnFamilyData)
- .filter(fam -> fam.equals(CURR_LOC_FAM) ||
fam.equals(FUTURE_LOC_FAM)).count();
-
- if (locsSeen > 1) {
+ if
(data.columnValues.entrySet().stream().filter(isLocationCF).map(Entry::getValue)
+ .mapToInt(TreeMap::size).sum() > 1) {
throw new IllegalStateException(
- "After mutation, root tablet has multiple locations : " + m + " " +
entries);
- }
-
- }
-
- /**
- * Convert json to tablet metadata. *
- */
- public TabletMetadata convertToTabletMetadata() {
- return TabletMetadata.convertRow(entries.entrySet().iterator(),
EnumSet.allOf(ColumnType.class),
- false);
- }
-
- private static String bs2Str(byte[] bs) {
- String str = new String(bs, UTF_8);
-
- // The expectation is that all data stored in the root tablet can be
converted to UTF8. This is
- // a sanity check to ensure the byte sequence can be converted from byte[]
to UTF8 to byte[] w/o
- // data corruption. Not all byte arrays can be converted to UTF8.
- Preconditions.checkArgument(Arrays.equals(bs, str.getBytes(UTF_8)),
- "Unsuccessful conversion of %s to utf8", str);
-
- return str;
- }
-
- /**
- * @return a json representation of this object, use {@link
#fromJson(String)} to convert the json
- * back to an object.
- */
- public String toJson() {
- GSonData gd = new GSonData();
- gd.columnValues = new TreeMap<>();
-
- Set<Entry<Key,Value>> es = entries.entrySet();
- for (Entry<Key,Value> entry : es) {
- String fam = bs2Str(entry.getKey().getColumnFamilyData().toArray());
- String qual = bs2Str(entry.getKey().getColumnQualifierData().toArray());
- String val = bs2Str(entry.getValue().get());
-
- gd.columnValues.computeIfAbsent(fam, k -> new TreeMap<>()).put(qual,
val);
+ "After mutation, root tablet has multiple locations : " + m + " " +
data.columnValues);
}
-
- return GSON.toJson(gd);
}
/**
- * Converts created by calling {@link #toJson()} back to an object.
+ * Convert this class to a {@link TabletMetadata}
*/
- public static RootTabletMetadata fromJson(String json) {
- GSonData gd = GSON.fromJson(json, GSonData.class);
-
- Preconditions.checkArgument(gd.version == 1);
-
+ public TabletMetadata toTabletMetadata() {
String row = RootTable.EXTENT.toMetaRow().toString();
-
- TreeMap<Key,Value> entries = new TreeMap<>();
-
- gd.columnValues.forEach((fam, qualVals) -> {
- qualVals.forEach((qual, val) -> {
- Key k = new Key(row, fam, qual, 1);
- Value v = new Value(val);
-
- entries.put(k, v);
- });
- });
-
- RootTabletMetadata rtm = new RootTabletMetadata();
- rtm.entries = entries;
-
- return rtm;
+ // use a stream so we don't have to re-sort in a new TreeMap<Key,Value>
structure
+ Stream<SimpleImmutableEntry<Key,Value>> entries =
data.columnValues.entrySet().stream()
+ .flatMap(famToQualVal -> famToQualVal.getValue().entrySet().stream()
+ .map(qualVal -> new SimpleImmutableEntry<>(
+ new Key(row, famToQualVal.getKey(), qualVal.getKey(), 1),
+ new Value(qualVal.getValue()))));
+ return TabletMetadata.convertRow(entries.iterator(),
+ EnumSet.allOf(TabletMetadata.ColumnType.class), false);
}
/**
- * Converts created by calling {@link #toJson()} back to an object. Assumes
the json is UTF8
- * encoded.
+ * @return a JSON representation of the root tablet's data.
*/
- public static RootTabletMetadata fromJson(byte[] bs) {
- return fromJson(new String(bs, UTF_8));
+ public String toJson() {
+ return gson.toJson(data);
}
- /**
- * Generate initial json for the root tablet metadata.
- */
- public static byte[] getInitialJson(String dirName, String file) {
- ServerColumnFamily.validateDirCol(dirName);
- Mutation mutation =
TabletColumnFamily.createPrevRowMutation(RootTable.EXTENT);
- ServerColumnFamily.DIRECTORY_COLUMN.put(mutation, new Value(dirName));
-
- mutation.put(DataFileColumnFamily.STR_NAME, file, new DataFileValue(0,
0).encodeAsValue());
-
- ServerColumnFamily.TIME_COLUMN.put(mutation,
- new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
-
- RootTabletMetadata rtm = new RootTabletMetadata();
-
- rtm.update(mutation);
-
- return rtm.toJson().getBytes(UTF_8);
- }
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index fc98313a67..376fabf236 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -296,7 +296,7 @@ public class TabletMetadata {
}
@VisibleForTesting
- public static TabletMetadata convertRow(Iterator<Entry<Key,Value>> rowIter,
+ public static <E extends Entry<Key,Value>> TabletMetadata
convertRow(Iterator<E> rowIter,
EnumSet<ColumnType> fetchedColumns, boolean buildKeyValueMap) {
Objects.requireNonNull(rowIter);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 86ae72a075..ff32a55dc4 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.core.metadata.schema;
import static com.google.common.base.Preconditions.checkState;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
@@ -512,8 +513,8 @@ public class TabletsMetadata implements
Iterable<TabletMetadata>, AutoCloseable
case IMMEDIATE:
ZooReader zooReader = ctx.getZooReader();
try {
- return RootTabletMetadata.fromJson(zooReader.getData(zkRoot +
RootTable.ZROOT_TABLET))
- .convertToTabletMetadata();
+ byte[] bytes = zooReader.getData(zkRoot + RootTable.ZROOT_TABLET);
+ return new RootTabletMetadata(new String(bytes,
UTF_8)).toTabletMetadata();
} catch (InterruptedException | KeeperException e) {
throw new RuntimeException(e);
}
@@ -523,8 +524,8 @@ public class TabletsMetadata implements
Iterable<TabletMetadata>, AutoCloseable
}
public static TabletMetadata getRootMetadata(String zkRoot, ZooCache zc) {
- return RootTabletMetadata.fromJson(zc.get(zkRoot + RootTable.ZROOT_TABLET))
- .convertToTabletMetadata();
+ byte[] jsonBytes = zc.get(zkRoot + RootTable.ZROOT_TABLET);
+ return new RootTabletMetadata(new String(jsonBytes,
UTF_8)).toTabletMetadata();
}
private final AutoCloseable closeable;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
index ad617b8e25..035c59e57c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
@@ -38,7 +38,7 @@ public class AccumuloDataVersion {
/**
* version (10) reflects changes to how root tablet metadata is serialized
in zookeeper starting
- * with 2.1
+ * with 2.1. See {@link
org.apache.accumulo.core.metadata.schema.RootTabletMetadata}.
*/
public static final int ROOT_TABLET_META_CHANGES = 10;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
index 71d910b6a4..2dbd617c23 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
@@ -24,12 +24,18 @@ import static
org.apache.accumulo.server.init.Initialize.REPL_TABLE_ID;
import java.io.IOException;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.data.InstanceId;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.manager.thrift.ManagerGoalState;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataTime;
import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooUtil;
@@ -43,7 +49,7 @@ import org.apache.accumulo.server.tables.TableManager;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
-class ZooKeeperInitializer {
+public class ZooKeeperInitializer {
private final byte[] EMPTY_BYTE_ARRAY = new byte[0];
private final byte[] ZERO_CHAR_ARRAY = {'0'};
@@ -130,7 +136,7 @@ class ZooKeeperInitializer {
zoo.putPersistentData(zkInstanceRoot + Constants.ZPROBLEMS,
EMPTY_BYTE_ARRAY,
ZooUtil.NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + RootTable.ZROOT_TABLET,
- RootTabletMetadata.getInitialJson(rootTabletDirName,
rootTabletFileUri),
+ getInitialRootTabletJson(rootTabletDirName, rootTabletFileUri),
ZooUtil.NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot +
RootTable.ZROOT_TABLET_GC_CANDIDATES,
new RootGcCandidates().toJson().getBytes(UTF_8),
ZooUtil.NodeExistsPolicy.FAIL);
@@ -176,4 +182,26 @@ class ZooKeeperInitializer {
}
+ /**
+ * Generate initial json for the root tablet metadata. Return the JSON
converted to a byte[].
+ */
+ public static byte[] getInitialRootTabletJson(String dirName, String file) {
+ MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(dirName);
+ Mutation mutation =
+
MetadataSchema.TabletsSection.TabletColumnFamily.createPrevRowMutation(RootTable.EXTENT);
+
MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mutation,
+ new Value(dirName));
+
+ mutation.put(MetadataSchema.TabletsSection.DataFileColumnFamily.STR_NAME,
file,
+ new DataFileValue(0, 0).encodeAsValue());
+
+ MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mutation,
+ new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
+
+ RootTabletMetadata rootTabletJson = new RootTabletMetadata();
+ rootTabletJson.update(mutation);
+
+ return rootTabletJson.toJson().getBytes(UTF_8);
+ }
+
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
index 50a465fa53..5b24808f04 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
@@ -18,10 +18,8 @@
*/
package org.apache.accumulo.server.metadata;
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static com.google.common.base.Preconditions.checkArgument;
-import java.util.Collection;
-import java.util.Iterator;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
@@ -31,85 +29,73 @@ import java.util.stream.Stream;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.hadoop.fs.Path;
-import com.google.common.base.Preconditions;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
public class RootGcCandidates {
- private static final Gson GSON = new GsonBuilder().create();
+ // Version 1. Released with Accumulo version 2.1.0
+ private static final int VERSION = 1;
+
+ private final Gson gson = new Gson();
+ private final Data data;
// This class is used to serialize and deserialize root tablet metadata
using GSon. Any changes to
// this class must consider persisted data.
- private static class GSonData {
- int version = 1;
-
- // SortedMap<dir path, SortedSet<file name>>
- SortedMap<String,SortedSet<String>> candidates;
+ private static class Data {
+ private final int version;
+
+ /*
+ * The root tablet will only have a single dir on each volume. Therefore,
root file paths will
+ * have a small set of unique prefixes. The following map is structured to
avoid storing the
+ * same dir prefix over and over in JSon and java.
+ *
+ * SortedMap<dir path, SortedSet<file name>>
+ */
+ private final SortedMap<String,SortedSet<String>> candidates;
+
+ public Data(int version, SortedMap<String,SortedSet<String>> candidates) {
+ this.version = version;
+ this.candidates = candidates;
+ }
}
- /*
- * The root tablet will only have a single dir on each volume. Therefore
root file paths will have
- * a small set of unique prefixes. The following map is structured to avoid
storing the same dir
- * prefix over and over in JSon and java.
- *
- * SortedMap<dir path, SortedSet<file name>>
- */
- private SortedMap<String,SortedSet<String>> candidates;
-
public RootGcCandidates() {
- this.candidates = new TreeMap<>();
+ this.data = new Data(VERSION, new TreeMap<>());
}
- private RootGcCandidates(SortedMap<String,SortedSet<String>> candidates) {
- this.candidates = candidates;
+ public RootGcCandidates(String jsonString) {
+ this.data = gson.fromJson(jsonString, Data.class);
+ checkArgument(data.version == VERSION, "Invalid Root Table GC Candidates
JSON version %s",
+ data.version);
+ data.candidates.forEach((parent, files) -> {
+ checkArgument(!parent.isBlank(), "Blank parent dir in %s",
data.candidates);
+ checkArgument(!files.isEmpty(), "Empty files for dir %s", parent);
+ });
}
- public void add(Iterator<StoredTabletFile> refs) {
- refs.forEachRemaining(ref -> {
- String parent = ref.getPath().getParent().toString();
- candidates.computeIfAbsent(parent, k -> new
TreeSet<>()).add(ref.getFileName());
- });
+ public void add(Stream<StoredTabletFile> refs) {
+ refs.forEach(ref -> data.candidates
+ .computeIfAbsent(ref.getPath().getParent().toString(), k -> new
TreeSet<>())
+ .add(ref.getFileName()));
}
- public void remove(Collection<String> refs) {
- refs.forEach(ref -> {
- Path path = new Path(ref);
- String parent = path.getParent().toString();
- String name = path.getName();
-
- SortedSet<String> names = candidates.get(parent);
- if (names != null) {
- names.remove(name);
- if (names.isEmpty()) {
- candidates.remove(parent);
- }
- }
- });
+ public void remove(Stream<String> refs) {
+ refs.map(Path::new).forEach(
+ path -> data.candidates.computeIfPresent(path.getParent().toString(),
(key, values) -> {
+ values.remove(path.getName());
+ return values.isEmpty() ? null : values;
+ }));
}
- public Stream<String> stream() {
- return candidates.entrySet().stream().flatMap(entry -> {
+ public Stream<String> sortedStream() {
+ return data.candidates.entrySet().stream().flatMap(entry -> {
String parent = entry.getKey();
SortedSet<String> names = entry.getValue();
- return names.stream().map(name -> new Path(parent, name).toString());
- });
+ return names.stream().map(name -> new Path(parent, name));
+ }).map(Path::toString).sorted();
}
public String toJson() {
- GSonData gd = new GSonData();
- gd.candidates = candidates;
- return GSON.toJson(gd);
- }
-
- public static RootGcCandidates fromJson(String json) {
- GSonData gd = GSON.fromJson(json, GSonData.class);
-
- Preconditions.checkArgument(gd.version == 1);
-
- return new RootGcCandidates(gd.candidates);
+ return gson.toJson(data);
}
- public static RootGcCandidates fromJson(byte[] json) {
- return fromJson(new String(json, UTF_8));
- }
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootTabletMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootTabletMutatorImpl.java
index 7681b5988d..dc1d157d86 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootTabletMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootTabletMutatorImpl.java
@@ -103,7 +103,7 @@ public class RootTabletMutatorImpl extends
TabletMutatorBase implements Ample.Ta
// TODO examine implementation of getZooReaderWriter().mutate()
context.getZooReaderWriter().mutateOrCreate(zpath, new byte[0], currVal
-> {
String currJson = new String(currVal, UTF_8);
- var rtm = RootTabletMetadata.fromJson(currJson);
+ var rtm = new RootTabletMetadata(currJson);
rtm.update(mutation);
String newJson = rtm.toJson();
log.debug("mutation: from:[{}] to: [{}]", currJson, newJson);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index 6da3a28827..08e20434fb 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -84,7 +84,7 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
try {
context.getZooReaderWriter().mutateOrCreate(zpath, new byte[0], currVal
-> {
String currJson = new String(currVal, UTF_8);
- RootGcCandidates rgcc = RootGcCandidates.fromJson(currJson);
+ RootGcCandidates rgcc = new RootGcCandidates(currJson);
log.debug("Root GC candidates before change : {}", currJson);
mutator.accept(rgcc);
String newJson = rgcc.toJson();
@@ -106,7 +106,7 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
public void putGcCandidates(TableId tableId, Collection<StoredTabletFile>
candidates) {
if (RootTable.ID.equals(tableId)) {
- mutateRootGcCandidates(rgcc -> rgcc.add(candidates.iterator()));
+ mutateRootGcCandidates(rgcc -> rgcc.add(candidates.stream()));
return;
}
@@ -125,8 +125,7 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
if (RootTable.ID.equals(tableId)) {
// Directories are unexpected for the root tablet, so convert to stored
tablet file
- mutateRootGcCandidates(
- rgcc ->
rgcc.add(candidates.stream().map(StoredTabletFile::new).iterator()));
+ mutateRootGcCandidates(rgcc ->
rgcc.add(candidates.stream().map(StoredTabletFile::new)));
return;
}
@@ -143,7 +142,7 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
public void deleteGcCandidates(DataLevel level, Collection<String> paths) {
if (level == DataLevel.ROOT) {
- mutateRootGcCandidates(rgcc -> rgcc.remove(paths));
+ mutateRootGcCandidates(rgcc -> rgcc.remove(paths.stream()));
return;
}
@@ -162,14 +161,14 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
public Iterator<String> getGcCandidates(DataLevel level) {
if (level == DataLevel.ROOT) {
var zooReader = context.getZooReader();
- byte[] json;
+ byte[] jsonBytes;
try {
- json = zooReader.getData(context.getZooKeeperRoot() +
RootTable.ZROOT_TABLET_GC_CANDIDATES);
+ jsonBytes =
+ zooReader.getData(context.getZooKeeperRoot() +
RootTable.ZROOT_TABLET_GC_CANDIDATES);
} catch (KeeperException | InterruptedException e) {
throw new RuntimeException(e);
}
- Stream<String> candidates =
RootGcCandidates.fromJson(json).stream().sorted();
- return candidates.iterator();
+ return new RootGcCandidates(new String(jsonBytes,
UTF_8)).sortedStream().iterator();
} else if (level == DataLevel.METADATA || level == DataLevel.USER) {
Range range = DeletesSection.getRange();
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
index f6991550de..4564466bb1 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.server.manager.state;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.accumulo.server.init.ZooKeeperInitializer.getInitialRootTabletJson;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -49,14 +50,14 @@ public class RootTabletStateStoreTest {
private static class TestAmple implements Ample {
- private String json = new String(
- RootTabletMetadata.getInitialJson("dir",
"hdfs://nn/acc/tables/some/dir/0000.rf"), UTF_8);
+ private String json =
+ new String(getInitialRootTabletJson("dir",
"hdfs://nn/acc/tables/some/dir/0000.rf"), UTF_8);
@Override
public TabletMetadata readTablet(KeyExtent extent, ReadConsistency rc,
ColumnType... colsToFetch) {
Preconditions.checkArgument(extent.equals(RootTable.EXTENT));
- return RootTabletMetadata.fromJson(json).convertToTabletMetadata();
+ return new RootTabletMetadata(json).toTabletMetadata();
}
@Override
@@ -73,10 +74,8 @@ public class RootTabletStateStoreTest {
public void mutate() {
Mutation m = getMutation();
- RootTabletMetadata rtm = RootTabletMetadata.fromJson(json);
-
+ var rtm = new RootTabletMetadata(json);
rtm.update(m);
-
json = rtm.toJson();
}
};
diff --git
a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
index 36d1f04890..fcee1bf48e 100644
--- a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java
@@ -78,8 +78,7 @@ public class CountNameNodeOpsBulkIT extends
ConfigurableMacBase {
URL url = new URL(uri + "/jmx");
log.debug("Fetching web page " + url);
String jsonString = FunctionalTestUtils.readWebPage(url).body();
- Gson gson = new Gson();
- Map<?,?> jsonObject = gson.fromJson(jsonString, Map.class);
+ Map<?,?> jsonObject = new Gson().fromJson(jsonString, Map.class);
List<?> beans = (List<?>) jsonObject.get("beans");
for (Object bean : beans) {
Map<?,?> map = (Map<?,?>) bean;
