This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit dc2800691e3b4e269d8f86d6b9f99861c4d9257d
Merge: 3792630c86 319ec30b7f
Author: Daniel Roberts <ddani...@gmail.com>
AuthorDate: Thu Jan 4 01:53:51 2024 +0000
Merge branch '2.1'
.../apache/accumulo/test/functional/MergeIT.java | 60 ++++++++++++++++++++++
1 file changed, 60 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index 4d265a1120,43590e8030..af02e42f51
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@@ -18,52 -18,46 +18,63 @@@
*/
package org.apache.accumulo.test.functional;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.accumulo.test.util.FileMetadataUtil.printAndVerifyFileMetadata;
+import static
org.apache.accumulo.test.util.FileMetadataUtil.verifyMergedMarkerCleared;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+ import java.util.UUID;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.dataImpl.KeyExtent;
++import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
import org.apache.accumulo.core.security.Authorizations;
+ import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
+ import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.core.util.Merge;
+ import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.TestIngest.IngestParams;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.accumulo.test.VerifyIngest.VerifyParams;
+ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MergeIT extends AccumuloClusterHarness {
@@@ -568,4 -230,52 +579,53 @@@
}
}
}
+
+ // Test that merge handles metadata from compactions
+ @Test
+ public void testCompactionMetadata() throws Exception {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ var split = new Text("m");
+ c.tableOperations().addSplits(tableName, new TreeSet<>(List.of(split)));
+
+ TableId tableId = getServerContext().getTableId(tableName);
+
+ // add metadata from compactions to tablets prior to merge
+ try (var tabletsMutator =
getServerContext().getAmple().mutateTablets()) {
+ for (var extent : List.of(new KeyExtent(tableId, split, null),
+ new KeyExtent(tableId, null, split))) {
+ var tablet = tabletsMutator.mutateTablet(extent);
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+
- TabletFile tmpFile = new TabletFile(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
++ ReferencedTabletFile tmpFile =
++ ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
+ CompactionExecutorId ceid =
CompactionExecutorIdImpl.externalId("G1");
+ Set<StoredTabletFile> jobFiles =
+ Set.of(new
StoredTabletFile("file:///accumulo/tables/t-0/b-0/b2.rf"));
+ ExternalCompactionMetadata ecMeta = new
ExternalCompactionMetadata(jobFiles, jobFiles,
+ tmpFile, "localhost:4444", CompactionKind.SYSTEM, (short) 2,
ceid, false, false, 44L);
+ tablet.putExternalCompaction(ecid, ecMeta);
+ tablet.mutate();
+ }
+ }
+
+ // ensure data is in metadata table as expected
+ try (var tablets =
getServerContext().getAmple().readTablets().forTable(tableId).build()) {
+ for (var tablet : tablets) {
+ assertFalse(tablet.getExternalCompactions().isEmpty());
+ }
+ }
+
+ c.tableOperations().merge(tableName, null, null);
+
+ // ensure merge operation remove compaction entries
+ try (var tablets =
getServerContext().getAmple().readTablets().forTable(tableId).build()) {
+ for (var tablet : tablets) {
+ assertTrue(tablet.getExternalCompactions().isEmpty());
+ }
+ }
+ }
+ }
}
