This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a8f566ef76772fd82143ca5c8054b4bd2d5d0780
Merge: 59a4937fc9 d811807bf1
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Mon Nov 13 11:07:51 2023 -0500
Merge branch 'main' into elasticity
.../java/org/apache/accumulo/core/conf/Property.java | 2 +-
.../accumulo/core/tabletserver/log/LogEntry.java | 17 +++++------------
.../core/metadata/schema/TabletMetadataTest.java | 18 ++++++++----------
.../accumulo/core/tabletserver/log/LogEntryTest.java | 20 +++++++-------------
.../manager/state/AbstractTabletStateStore.java | 2 +-
.../server/manager/state/TabletManagementTest.java | 8 ++++----
.../org/apache/accumulo/tserver/TabletServer.java | 4 +---
.../shell/commands/ListTabletsCommandTest.java | 4 ++--
.../java/org/apache/accumulo/test/MetaSplitIT.java | 3 ++-
.../test/MissingWalHeaderCompletesRecoveryIT.java | 4 ++--
.../test/functional/TabletManagementIteratorIT.java | 4 ++--
11 files changed, 35 insertions(+), 51 deletions(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index a2ebe442bf,a3a3d33a45..b890a54299
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -286,98 -268,4 +284,98 @@@ public class TabletMetadataTest
});
return rowMap;
}
+
+ @Test
+ public void testBuilder() {
+ TServerInstance ser1 = new
TServerInstance(HostAndPort.fromParts("server1", 8555), "s001");
+
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ StoredTabletFile sf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert();
+ DataFileValue dfv1 = new DataFileValue(89, 67);
+
+ StoredTabletFile sf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf2.rf")).insert();
+ DataFileValue dfv2 = new DataFileValue(890, 670);
+
+ ReferencedTabletFile rf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/imp1.rf"));
+ ReferencedTabletFile rf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/imp2.rf"));
+
+ StoredTabletFile sf3 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf3.rf")).insert();
+ StoredTabletFile sf4 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf4.rf")).insert();
+
+ TabletMetadata tm =
TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.NEVER)
+ .putLocation(Location.future(ser1)).putFile(sf1, dfv1).putFile(sf2,
dfv2)
+ .putCompactionId(23).putBulkFile(rf1, 25).putBulkFile(rf2,
35).putFlushId(27)
+
.putDirName("dir1").putScan(sf3).putScan(sf4).putCompacted(17).putCompacted(23)
+ .build(ECOMP, HOSTING_REQUESTED);
+
+ assertEquals(extent, tm.getExtent());
+ assertEquals(TabletHostingGoal.NEVER, tm.getHostingGoal());
+ assertEquals(Location.future(ser1), tm.getLocation());
+ assertEquals(23L, tm.getCompactId().orElse(-1));
+ assertEquals(27L, tm.getFlushId().orElse(-1));
+ assertEquals(Map.of(sf1, dfv1, sf2, dfv2), tm.getFilesMap());
+ assertEquals(Map.of(rf1.insert(), 25L, rf2.insert(), 35L),
tm.getLoaded());
+ assertEquals("dir1", tm.getDirName());
+ assertEquals(Set.of(sf3, sf4), Set.copyOf(tm.getScans()));
+ assertEquals(Set.of(), tm.getExternalCompactions().keySet());
+ assertEquals(Set.of(17L, 23L), tm.getCompacted());
+ assertFalse(tm.getHostingRequested());
+ assertThrows(IllegalStateException.class, tm::getOperationId);
+ assertThrows(IllegalStateException.class, tm::getSuspend);
+ assertThrows(IllegalStateException.class, tm::getTime);
+
+ TabletOperationId opid1 =
TabletOperationId.from(TabletOperationType.SPLITTING, 55);
+ TabletMetadata tm2 =
TabletMetadata.builder(extent).putOperation(opid1).build(LOCATION);
+
+ assertEquals(extent, tm2.getExtent());
+ assertEquals(opid1, tm2.getOperationId());
+ assertNull(tm2.getLocation());
+ assertThrows(IllegalStateException.class, tm2::getFiles);
+ assertThrows(IllegalStateException.class, tm2::getHostingGoal);
+ assertThrows(IllegalStateException.class, tm2::getCompactId);
+ assertThrows(IllegalStateException.class, tm2::getFlushId);
+ assertThrows(IllegalStateException.class, tm2::getFiles);
+ assertThrows(IllegalStateException.class, tm2::getLogs);
+ assertThrows(IllegalStateException.class, tm2::getLoaded);
+ assertThrows(IllegalStateException.class, tm2::getDirName);
+ assertThrows(IllegalStateException.class, tm2::getScans);
+ assertThrows(IllegalStateException.class, tm2::getExternalCompactions);
+ assertThrows(IllegalStateException.class, tm2::getHostingRequested);
+ assertThrows(IllegalStateException.class, tm2::getSelectedFiles);
+ assertThrows(IllegalStateException.class, tm2::getCompacted);
+
+ var ecid1 = ExternalCompactionId.generate(UUID.randomUUID());
+ ExternalCompactionMetadata ecm = new
ExternalCompactionMetadata(Set.of(sf1, sf2), rf1, "cid1",
+ CompactionKind.USER, (short) 3,
CompactionExecutorIdImpl.externalId("Q1"), true, 99L);
+
- LogEntry le1 = new LogEntry(55, "localhost:8020/" + UUID.randomUUID());
- LogEntry le2 = new LogEntry(57, "localhost:8020/" + UUID.randomUUID());
++ LogEntry le1 = new LogEntry("localhost:8020/" + UUID.randomUUID());
++ LogEntry le2 = new LogEntry("localhost:8020/" + UUID.randomUUID());
+
+ SelectedFiles selFiles = new SelectedFiles(Set.of(sf1, sf4), false, 159L);
+
+ TabletMetadata tm3 =
TabletMetadata.builder(extent).putExternalCompaction(ecid1, ecm)
+ .putSuspension(ser1, 45L).putTime(new MetadataTime(479,
TimeType.LOGICAL)).putWal(le1)
+ .putWal(le2).setHostingRequested().putSelectedFiles(selFiles).build();
+
+ assertEquals(Set.of(ecid1), tm3.getExternalCompactions().keySet());
+ assertEquals(Set.of(sf1, sf2),
tm3.getExternalCompactions().get(ecid1).getJobFiles());
+ assertEquals(ser1.getHostAndPort(), tm3.getSuspend().server);
+ assertEquals(45L, tm3.getSuspend().suspensionTime);
+ assertEquals(new MetadataTime(479, TimeType.LOGICAL), tm3.getTime());
+ assertTrue(tm3.getHostingRequested());
+ assertEquals(Stream.of(le1, le2).map(LogEntry::toString).collect(toSet()),
+ tm3.getLogs().stream().map(LogEntry::toString).collect(toSet()));
+ assertEquals(Set.of(sf1, sf4), tm3.getSelectedFiles().getFiles());
+ assertEquals(159L, tm3.getSelectedFiles().getFateTxId());
+ assertFalse(tm3.getSelectedFiles().initiallySelectedAll());
+ assertEquals(selFiles.getMetadataValue(),
tm3.getSelectedFiles().getMetadataValue());
+ }
+
}
diff --cc
server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
index d58830a371,0000000000..1f7a9b057b
mode 100644,000000..100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java
@@@ -1,169 -1,0 +1,169 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.manager.state;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import
org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult.Status;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.server.util.ManagerMetadataUtil;
+import org.apache.hadoop.fs.Path;
+
+import com.google.common.base.Preconditions;
+
+public abstract class AbstractTabletStateStore implements TabletStateStore {
+
+ private final ClientContext context;
+ private final Ample ample;
+
+ protected AbstractTabletStateStore(ClientContext context) {
+ this.context = context;
+ this.ample = context.getAmple();
+ }
+
+ @Override
+ public void setLocations(Collection<Assignment> assignments) throws
DistributedStoreException {
+ try (var tabletsMutator = ample.conditionallyMutateTablets()) {
+ for (Assignment assignment : assignments) {
+ var conditionalMutator =
tabletsMutator.mutateTablet(assignment.tablet)
+
.requireLocation(TabletMetadata.Location.future(assignment.server))
+ .putLocation(TabletMetadata.Location.current(assignment.server))
+
.deleteLocation(TabletMetadata.Location.future(assignment.server)).deleteSuspension();
+
+ ManagerMetadataUtil.updateLastForAssignmentMode(context,
conditionalMutator,
+ assignment.server, assignment.lastLocation);
+
+ conditionalMutator.submit(tabletMetadata -> {
+
Preconditions.checkArgument(tabletMetadata.getExtent().equals(assignment.tablet));
+ return tabletMetadata.getLocation() != null &&
tabletMetadata.getLocation()
+ .equals(TabletMetadata.Location.current(assignment.server));
+ });
+ }
+
+ if (tabletsMutator.process().values().stream()
+ .anyMatch(result -> result.getStatus() != Status.ACCEPTED)) {
+ throw new DistributedStoreException(
+ "failed to set tablet location, conditional mutation failed");
+ }
+ } catch (RuntimeException ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public void setFutureLocations(Collection<Assignment> assignments)
+ throws DistributedStoreException {
+ try (var tabletsMutator = ample.conditionallyMutateTablets()) {
+ for (Assignment assignment : assignments) {
+
tabletsMutator.mutateTablet(assignment.tablet).requireAbsentOperation()
+ .requireAbsentLocation().deleteSuspension()
+ .putLocation(TabletMetadata.Location.future(assignment.server))
+ .submit(tabletMetadata -> {
+
Preconditions.checkArgument(tabletMetadata.getExtent().equals(assignment.tablet));
+ return tabletMetadata.getLocation() != null &&
tabletMetadata.getLocation()
+ .equals(TabletMetadata.Location.future(assignment.server));
+ });
+ }
+
+ var results = tabletsMutator.process();
+
+ if (results.values().stream().anyMatch(result -> result.getStatus() !=
Status.ACCEPTED)) {
+ throw new DistributedStoreException(
+ "failed to set tablet location, conditional mutation failed. ");
+ }
+
+ } catch (RuntimeException ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+
+ @Override
+ public void unassign(Collection<TabletMetadata> tablets,
+ Map<TServerInstance,List<Path>> logsForDeadServers) throws
DistributedStoreException {
+ unassign(tablets, logsForDeadServers, -1);
+ }
+
+ @Override
+ public void suspend(Collection<TabletMetadata> tablets,
+ Map<TServerInstance,List<Path>> logsForDeadServers, long
suspensionTimestamp)
+ throws DistributedStoreException {
+ unassign(tablets, logsForDeadServers, suspensionTimestamp);
+ }
+
+ protected abstract void processSuspension(Ample.ConditionalTabletMutator
tabletMutator,
+ TabletMetadata tm, long suspensionTimestamp);
+
+ private void unassign(Collection<TabletMetadata> tablets,
+ Map<TServerInstance,List<Path>> logsForDeadServers, long
suspensionTimestamp)
+ throws DistributedStoreException {
+ try (var tabletsMutator = ample.conditionallyMutateTablets()) {
+ for (TabletMetadata tm : tablets) {
+ if (tm.getLocation() == null) {
+ continue;
+ }
+
+ var tabletMutator =
+
tabletsMutator.mutateTablet(tm.getExtent()).requireLocation(tm.getLocation());
+
+ if (tm.hasCurrent()) {
+
+ ManagerMetadataUtil.updateLastForAssignmentMode(context,
tabletMutator,
+ tm.getLocation().getServerInstance(), tm.getLast());
+ tabletMutator.deleteLocation(tm.getLocation());
+ if (logsForDeadServers != null) {
+ List<Path> logs =
logsForDeadServers.get(tm.getLocation().getServerInstance());
+ if (logs != null) {
+ for (Path log : logs) {
- LogEntry entry = new LogEntry(0, log.toString());
++ LogEntry entry = new LogEntry(log.toString());
+ tabletMutator.putWal(entry);
+ }
+ }
+ }
+ }
+
+ if (tm.getLocation() != null && tm.getLocation().getType() != null
+ && tm.getLocation().getType().equals(LocationType.FUTURE)) {
+ tabletMutator.deleteLocation(tm.getLocation());
+ }
+
+ processSuspension(tabletMutator, tm, suspensionTimestamp);
+
+ tabletMutator.submit(tabletMetadata -> tabletMetadata.getLocation()
== null);
+ }
+
+ Map<KeyExtent,Ample.ConditionalResult> results =
tabletsMutator.process();
+
+ if (results.values().stream()
+ .anyMatch(conditionalResult -> conditionalResult.getStatus() !=
Status.ACCEPTED)) {
+ throw new DistributedStoreException("Some unassignments did not
satisfy conditions.");
+ }
+
+ } catch (RuntimeException ex) {
+ throw new DistributedStoreException(ex);
+ }
+ }
+}
diff --cc
server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
index a71308dc3e,0000000000..2fd6df2978
mode 100644,000000..100644
---
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
@@@ -1,193 -1,0 +1,193 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.manager.state;
+
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateTxId;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.manager.state.TabletManagement;
+import
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.Test;
+
+public class TabletManagementTest {
+
+ private SortedMap<Key,Value> toRowMap(Mutation mutation) {
+ SortedMap<Key,Value> rowMap = new TreeMap<>();
+ mutation.getUpdates().forEach(cu -> {
+ Key k = new Key(mutation.getRow(), cu.getColumnFamily(),
cu.getColumnQualifier(),
+ cu.getTimestamp());
+ Value v = new Value(cu.getValue());
+ rowMap.put(k, v);
+ });
+ return rowMap;
+ }
+
+ private SortedMap<Key,Value> createMetadataEntryKV(KeyExtent extent) {
+
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+
+ COMPACT_COLUMN.put(mutation, new Value("5"));
+ DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
+ FLUSH_COLUMN.put(mutation, new Value("6"));
+ TIME_COLUMN.put(mutation, new Value("M123456789"));
+
+ StoredTabletFile bf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/bf1")).insert();
+ StoredTabletFile bf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/bf2")).insert();
+
mutation.at().family(BulkFileColumnFamily.NAME).qualifier(bf1.getMetadata())
+ .put(FateTxId.formatTid(56));
+
mutation.at().family(BulkFileColumnFamily.NAME).qualifier(bf2.getMetadata())
+ .put(FateTxId.formatTid(59));
+
+ mutation.at().family(ClonedColumnFamily.NAME).qualifier("").put("OK");
+
+ DataFileValue dfv1 = new DataFileValue(555, 23);
+ StoredTabletFile tf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/df1.rf")).insert();
+ StoredTabletFile tf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/df2.rf")).insert();
+
mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf1.getMetadata()).put(dfv1.encode());
+ DataFileValue dfv2 = new DataFileValue(234, 13);
+
mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetadata()).put(dfv2.encode());
+
+
mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555");
+
+
mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555");
+
- LogEntry le1 = new LogEntry(55, "localhost:8020/" + UUID.randomUUID());
++ LogEntry le1 = new LogEntry("localhost:8020/" + UUID.randomUUID());
+
mutation.at().family(LogColumnFamily.NAME).qualifier(le1.getColumnQualifier())
- .timestamp(le1.getTimestamp()).put(le1.getValue());
- LogEntry le2 = new LogEntry(57, "localhost:8020/" + UUID.randomUUID());
++ .put(le1.getValue());
++ LogEntry le2 = new LogEntry("localhost:8020/" + UUID.randomUUID());
+
mutation.at().family(LogColumnFamily.NAME).qualifier(le2.getColumnQualifier())
- .timestamp(le2.getTimestamp()).put(le2.getValue());
++ .put(le2.getValue());
+
+ StoredTabletFile sf1 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert();
+ StoredTabletFile sf2 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf2.rf")).insert();
+
mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf1.getMetadata()).put("");
+
mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf2.getMetadata()).put("");
+
+ return toRowMap(mutation);
+
+ }
+
+ @Test
+ public void testEncodeDecodeWithReasons() throws Exception {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ final Set<ManagementAction> actions =
+ Set.of(ManagementAction.NEEDS_LOCATION_UPDATE,
ManagementAction.NEEDS_SPLITTING);
+
+ final SortedMap<Key,Value> entries = createMetadataEntryKV(extent);
+
+ TabletManagement.addActions(entries, actions);
+ Key key = entries.firstKey();
+ Value val = WholeRowIterator.encodeRow(new ArrayList<>(entries.keySet()),
+ new ArrayList<>(entries.values()));
+
+ // Remove the REASONS column from the entries map for the comparison check
+ // below
+ entries.remove(new Key(key.getRow().toString(), "REASONS", ""));
+
+ TabletManagement tmi = new TabletManagement(key, val, true);
+ assertEquals(entries, tmi.getTabletMetadata().getKeyValues());
+ assertEquals(actions, tmi.getActions());
+ }
+
+ @Test
+ public void testEncodeDecodeWithErrors() throws Exception {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ final SortedMap<Key,Value> entries = createMetadataEntryKV(extent);
+
+ TabletManagement.addError(entries, new UnsupportedOperationException("Not
supported."));
+ Key key = entries.firstKey();
+ Value val = WholeRowIterator.encodeRow(new ArrayList<>(entries.keySet()),
+ new ArrayList<>(entries.values()));
+
+ // Remove the ERROR column from the entries map for the comparison check
+ // below
+ entries.remove(new Key(key.getRow().toString(), "ERROR", ""));
+
+ TabletManagement tmi = new TabletManagement(key, val, true);
+ assertEquals(entries, tmi.getTabletMetadata().getKeyValues());
+ assertEquals("Not supported.", tmi.getErrorMessage());
+ }
+
+ @Test
+ public void testBinary() throws Exception {
+ // test end row with non ascii data
+ Text endRow = new Text(new byte[] {'m', (byte) 0xff});
+ KeyExtent extent = new KeyExtent(TableId.of("5"), endRow, new Text("da"));
+
+ final Set<ManagementAction> actions =
+ Set.of(ManagementAction.NEEDS_LOCATION_UPDATE,
ManagementAction.NEEDS_SPLITTING);
+
+ final SortedMap<Key,Value> entries = createMetadataEntryKV(extent);
+
+ TabletManagement.addActions(entries, actions);
+ Key key = entries.firstKey();
+ Value val = WholeRowIterator.encodeRow(new ArrayList<>(entries.keySet()),
+ new ArrayList<>(entries.values()));
+
+ assertTrue(entries.keySet().stream().allMatch(k ->
k.getRow().equals(extent.toMetaRow())));
+
+ // Remove the REASONS column from the entries map for the comparison check
+ // below
+ entries.remove(new Key(key.getRow(), new Text("REASONS"), new Text("")));
+
+ TabletManagement tmi = new TabletManagement(key, val, true);
+ assertEquals(entries, tmi.getTabletMetadata().getKeyValues());
+ assertEquals(actions, tmi.getActions());
+
+ }
+}
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index eb158c1d8e,4b610d72ca..f8208a3a04
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -935,12 -1098,10 +935,10 @@@ public class TabletServer extends Abstr
logger.minorCompactionStarted(tablet, lastUpdateSequence,
newDataFileLocation, durability);
}
- public void recover(VolumeManager fs, KeyExtent extent, List<LogEntry>
logEntries,
+ public void recover(VolumeManager fs, KeyExtent extent,
Collection<LogEntry> logEntries,
Set<String> tabletFiles, MutationReceiver mutationReceiver) throws
IOException {
List<Path> recoveryDirs = new ArrayList<>();
- List<LogEntry> sorted = new ArrayList<>(logEntries);
- sorted.sort((e1, e2) -> (int) (e1.getTimestamp() - e2.getTimestamp()));
- for (LogEntry entry : sorted) {
+ for (LogEntry entry : logEntries) {
Path recovery = null;
Path finished = RecoveryPath.getRecoveryPath(new
Path(entry.getFilePath()));
finished = SortedLogState.getFinishedMarkerPath(finished);
diff --cc
shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java
index eb4d7e8410,583819b96b..14856bb9e7
---
a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java
@@@ -128,50 -113,6 +128,50 @@@ public class ListTabletsCommandTest
public void mockTest() throws Exception {
ListTabletsCommand cmd = new TestListTabletsCommand();
+ TableId tableId = TableId.of("123");
+
+ TServerInstance ser1 = new
TServerInstance(HostAndPort.fromParts("server1", 8555), "s001");
+ TServerInstance ser2 = new
TServerInstance(HostAndPort.fromParts("server2", 2354), "s002");
+
+ StoredTabletFile sf11 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-dir1/sf11.rf")).insert();
+ DataFileValue dfv11 = new DataFileValue(5643, 89);
+
+ StoredTabletFile sf12 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-dir1/sf12.rf")).insert();
+ DataFileValue dfv12 = new DataFileValue(379963, 1027);
+
+ StoredTabletFile sf21 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-dir2/sf21.rf")).insert();
+ DataFileValue dfv21 = new DataFileValue(5323, 142);
+
+ StoredTabletFile sf31 =
+ new ReferencedTabletFile(new
Path("hdfs://nn1/acc/tables/1/t-dir3/sf31.rf")).insert();
+ DataFileValue dfv31 = new DataFileValue(95832L, 231);
+
+ KeyExtent extent = new KeyExtent(tableId, new Text("d"), null);
+
- LogEntry le1 = new LogEntry(55, "localhost:8020/" + UUID.randomUUID());
- LogEntry le2 = new LogEntry(57, "localhost:8020/" + UUID.randomUUID());
++ LogEntry le1 = new LogEntry("localhost:8020/" + UUID.randomUUID());
++ LogEntry le2 = new LogEntry("localhost:8020/" + UUID.randomUUID());
+
+ TabletMetadata tm1 =
TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.ONDEMAND)
+ .putLocation(TabletMetadata.Location.current(ser1)).putFile(sf11,
dfv11)
+ .putFile(sf12, dfv12).putWal(le1).putDirName("t-dir1").build();
+
+ extent = new KeyExtent(tableId, new Text("k"), new Text("e"));
+ TabletMetadata tm2 =
TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.ALWAYS)
+ .putLocation(TabletMetadata.Location.current(ser2)).putFile(sf21,
dfv21)
+ .putDirName("t-dir2").build(LOGS);
+
+ extent = new KeyExtent(tableId, null, new Text("l"));
+ TabletMetadata tm3 =
TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.NEVER)
+ .putFile(sf31,
dfv31).putWal(le1).putWal(le2).putDirName("t-dir3").build(LOCATION);
+
+ TabletInformationImpl[] tabletInformation = new TabletInformationImpl[3];
+ tabletInformation[0] = new TabletInformationImpl(tm1, "HOSTED");
+ tabletInformation[1] = new TabletInformationImpl(tm2, "HOSTED");
+ tabletInformation[2] = new TabletInformationImpl(tm3, "UNASSIGNED");
+
AccumuloClient client = EasyMock.createMock(AccumuloClient.class);
ClientContext context = EasyMock.createMock(ClientContext.class);
TableOperations tableOps = EasyMock.createMock(TableOperations.class);
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
index d6b18a7a8c,25c013fa53..f47d46b45c
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
@@@ -396,46 -289,66 +396,46 @@@ public class TabletManagementIteratorI
}
}
- private static class State implements CurrentState {
-
- final ClientContext context;
-
- State(AccumuloClient client) {
- this.context = (ClientContext) client;
+ // Creates a log entry on the "some split" extent, this could be modified
easily to support
+ // other extents
+ private void createLogEntry(AccumuloClient client, String table, String
tableNameToModify)
+ throws MutationsRejectedException, TableNotFoundException {
+ TableId tableIdToModify =
+
TableId.of(client.tableOperations().tableIdMap().get(tableNameToModify));
+ KeyExtent extent = new KeyExtent(tableIdToModify, new Text("some split"),
null);
+ Mutation m = new Mutation(extent.toMetaRow());
+ LogEntry logEntry =
- new LogEntry(55, Path.of(validHost.toString(),
UUID.randomUUID().toString()).toString());
++ new LogEntry(Path.of(validHost.toString(),
UUID.randomUUID().toString()).toString());
+
m.at().family(LogColumnFamily.NAME).qualifier(logEntry.getColumnQualifier())
- .timestamp(logEntry.getTimestamp()).put(logEntry.getValue());
++ .put(logEntry.getValue());
+ try (BatchWriter bw = client.createBatchWriter(table)) {
+ bw.addMutation(m);
}
+ }
- private Set<TServerInstance> tservers;
- private Set<TableId> onlineTables;
-
- @Override
- public Set<TServerInstance> onlineTabletServers() {
- HashSet<TServerInstance> tservers = new HashSet<>();
- for (String tserver : context.instanceOperations().getTabletServers()) {
- try {
- var zPath =
ServiceLock.path(ZooUtil.getRoot(context.instanceOperations().getInstanceId())
- + Constants.ZTSERVERS + "/" + tserver);
- long sessionId = ServiceLock.getSessionId(context.getZooCache(),
zPath);
- tservers.add(new TServerInstance(tserver, sessionId));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
+ private static TabletManagementParameters createParameters(AccumuloClient
client) {
+ var context = (ClientContext) client;
+ Set<TableId> onlineTables =
Sets.filter(context.getTableIdToNameMap().keySet(),
+ tableId -> context.getTableState(tableId) == TableState.ONLINE);
+
+ HashSet<TServerInstance> tservers = new HashSet<>();
+ for (String tserver : context.instanceOperations().getTabletServers()) {
+ try {
+ var zPath =
ServiceLock.path(ZooUtil.getRoot(context.instanceOperations().getInstanceId())
+ + Constants.ZTSERVERS + "/" + tserver);
+ long sessionId = ServiceLock.getSessionId(context.getZooCache(),
zPath);
+ tservers.add(new TServerInstance(tserver, sessionId));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
- this.tservers = Collections.unmodifiableSet(tservers);
- return tservers;
- }
-
- @Override
- public Set<TableId> onlineTables() {
- Set<TableId> onlineTables = context.getTableIdToNameMap().keySet();
- this.onlineTables =
- Sets.filter(onlineTables, tableId -> context.getTableState(tableId)
== TableState.ONLINE);
- return this.onlineTables;
- }
-
- @Override
- public Collection<MergeInfo> merges() {
- return Collections.emptySet();
}
- @Override
- public Set<KeyExtent> migrationsSnapshot() {
- return Collections.emptySet();
- }
-
- @Override
- public Set<TServerInstance> shutdownServers() {
- return Collections.emptySet();
- }
-
- @Override
- public ManagerState getManagerState() {
- return ManagerState.NORMAL;
- }
-
- @Override
- public String toString() {
- return "tservers: " + tservers + " onlineTables: " + onlineTables;
- }
+ return new TabletManagementParameters(ManagerState.NORMAL,
+ Map.of(
+ Ample.DataLevel.ROOT, true, Ample.DataLevel.USER, true,
Ample.DataLevel.METADATA, true),
+ onlineTables,
+ new LiveTServerSet.LiveTServersSnapshot(tservers,
+ Map.of(Constants.DEFAULT_RESOURCE_GROUP_NAME, tservers)),
+ Set.of(), Map.of(), Ample.DataLevel.USER, Map.of(), true);
}
-
}
