This is an automated email from the ASF dual-hosted git repository.
DomGarguilo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 94363afcf4 Convert some ITs to use SharedMiniClusterBase (#6492)
94363afcf4 is described below
commit 94363afcf443b54ba72b04618878d6b2c72355ab
Author: Dom G. <[email protected]>
AuthorDate: Mon Aug 3 19:21:04 2026 -0400
Convert some ITs to use SharedMiniClusterBase (#6492)
---
.../apache/accumulo/test/functional/DeleteIT.java | 2 +-
.../accumulo/test/functional/DurabilityIT.java | 80 +++++++++++---------
.../accumulo/test/functional/ScanIteratorIT.java | 16 +++-
.../apache/accumulo/test/functional/ScannerIT.java | 36 ++++++---
.../apache/accumulo/test/functional/SplitIT.java | 85 +++++++---------------
5 files changed, 113 insertions(+), 106 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java
index 552fd5b91f..719391a5f6 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/DeleteIT.java
@@ -50,7 +50,7 @@ public class DeleteIT extends AccumuloClusterHarness {
public static void deleteTest(AccumuloClient c, AccumuloCluster cluster,
String tableName)
throws Exception {
- VerifyParams params = new VerifyParams(getClientProps(), tableName, 1000);
+ VerifyParams params = new VerifyParams(c.properties(), tableName, 1000);
params.cols = 1;
params.random = 56;
TestIngest.ingest(c, params);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
index ba87fe2cf8..133bc59b61 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@ -18,7 +18,6 @@
*/
package org.apache.accumulo.test.functional;
-import static
org.apache.accumulo.test.harness.AccumuloITBase.MINI_CLUSTER_ONLY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -35,28 +34,34 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.metadata.SystemTables;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.miniclusterImpl.ProcessReference;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.accumulo.test.harness.SharedMiniClusterBase;
import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import com.google.common.collect.Iterators;
-@Tag(MINI_CLUSTER_ONLY)
-public class DurabilityIT extends ConfigurableMacBase {
+public class DurabilityIT extends SharedMiniClusterBase {
- @Override
- protected Duration defaultTimeout() {
- return Duration.ofMinutes(4);
+ @BeforeAll
+ public static void start() throws Exception {
+ SharedMiniClusterBase.startMiniClusterWithConfig((cfg, coreSite) -> {
+ coreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1);
+ });
+ }
+
+ @AfterAll
+ public static void stop() {
+ SharedMiniClusterBase.stopMiniCluster();
}
@Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
- hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
- cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1);
+ protected Duration defaultTimeout() {
+ return Duration.ofMinutes(4);
}
static final long N = 100000;
@@ -87,7 +92,7 @@ public class DurabilityIT extends ConfigurableMacBase {
@Test
public void testSync() throws Exception {
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
String[] tableNames = init(client);
// sync table should lose nothing
writeSome(client, tableNames[0], N);
@@ -99,7 +104,7 @@ public class DurabilityIT extends ConfigurableMacBase {
@Test
public void testFlush() throws Exception {
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
String[] tableNames = init(client);
// flush table won't lose anything since we're not losing power/dfs
writeSome(client, tableNames[1], N);
@@ -111,7 +116,7 @@ public class DurabilityIT extends ConfigurableMacBase {
@Test
public void testLog() throws Exception {
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
String[] tableNames = init(client);
// we're probably going to lose something the the log setting
writeSome(client, tableNames[2], N);
@@ -124,7 +129,7 @@ public class DurabilityIT extends ConfigurableMacBase {
@Test
public void testNone() throws Exception {
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
String[] tableNames = init(client);
// probably won't get any data back without logging
writeSome(client, tableNames[3], N);
@@ -137,7 +142,7 @@ public class DurabilityIT extends ConfigurableMacBase {
@Test
public void testIncreaseDurability() throws Exception {
- try (AccumuloClient c =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
c.tableOperations().setProperty(tableName,
Property.TABLE_DURABILITY.getKey(), "none");
@@ -154,20 +159,27 @@ public class DurabilityIT extends ConfigurableMacBase {
@Test
public void testMetaDurability() throws Exception {
- try (AccumuloClient c =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
String tableName = getUniqueNames(1)[0];
- c.namespaceOperations().setProperty(Namespace.ACCUMULO.name(),
- Property.TABLE_DURABILITY.getKey(), "none");
- c.namespaceOperations().setProperty(Namespace.DEFAULT.name(),
- Property.TABLE_DURABILITY.getKey(), "none");
- Map<String,String> props =
-
c.tableOperations().getConfiguration(SystemTables.METADATA.tableName());
- assertEquals("sync", props.get(Property.TABLE_DURABILITY.getKey()));
- c.tableOperations().create(tableName);
- props = c.tableOperations().getConfiguration(tableName);
- assertEquals("none", props.get(Property.TABLE_DURABILITY.getKey()));
- restartTServer();
- assertTrue(c.tableOperations().exists(tableName));
+ try {
+ c.namespaceOperations().setProperty(Namespace.ACCUMULO.name(),
+ Property.TABLE_DURABILITY.getKey(), "none");
+ c.namespaceOperations().setProperty(Namespace.DEFAULT.name(),
+ Property.TABLE_DURABILITY.getKey(), "none");
+ Map<String,String> props =
+
c.tableOperations().getConfiguration(SystemTables.METADATA.tableName());
+ assertEquals("sync", props.get(Property.TABLE_DURABILITY.getKey()));
+ c.tableOperations().create(tableName);
+ props = c.tableOperations().getConfiguration(tableName);
+ assertEquals("none", props.get(Property.TABLE_DURABILITY.getKey()));
+ restartTServer();
+ assertTrue(c.tableOperations().exists(tableName));
+ } finally {
+ c.namespaceOperations().removeProperty(Namespace.ACCUMULO.name(),
+ Property.TABLE_DURABILITY.getKey());
+ c.namespaceOperations().removeProperty(Namespace.DEFAULT.name(),
+ Property.TABLE_DURABILITY.getKey());
+ }
}
}
@@ -176,10 +188,10 @@ public class DurabilityIT extends ConfigurableMacBase {
}
private void restartTServer() throws Exception {
- for (ProcessReference proc :
cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
- cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ for (ProcessReference proc :
getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
+ getCluster().killProcess(ServerType.TABLET_SERVER, proc);
}
- cluster.start();
+ getCluster().start();
}
private void writeSome(AccumuloClient c, String table, long count) throws
Exception {
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
index c1caae73cc..8a7e959649 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ScanIteratorIT.java
@@ -54,16 +54,18 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.WrappingIterator;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.test.harness.AccumuloClusterHarness;
+import org.apache.accumulo.test.harness.SharedMiniClusterBase;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class ScanIteratorIT extends AccumuloClusterHarness {
+public class ScanIteratorIT extends SharedMiniClusterBase {
private static final Logger log =
LoggerFactory.getLogger(ScanIteratorIT.class);
private AccumuloClient accumuloClient;
@@ -71,6 +73,16 @@ public class ScanIteratorIT extends AccumuloClusterHarness {
private String user;
private boolean saslEnabled;
+ @BeforeAll
+ public static void start() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ }
+
+ @AfterAll
+ public static void stop() {
+ SharedMiniClusterBase.stopMiniCluster();
+ }
+
@Override
protected Duration defaultTimeout() {
return Duration.ofMinutes(1);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ScannerIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ScannerIT.java
index 4b34643122..d5fc1d1945 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ScannerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ScannerIT.java
@@ -59,33 +59,44 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Timer;
import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.CloseScannerIT;
+import org.apache.accumulo.test.harness.SharedMiniClusterBase;
import org.apache.accumulo.test.util.Wait;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.MoreCollectors;
-public class ScannerIT extends ConfigurableMacBase {
+public class ScannerIT extends SharedMiniClusterBase {
- @Override
- protected Duration defaultTimeout() {
- return Duration.ofMinutes(1);
+ private static final Logger log = LoggerFactory.getLogger(ScannerIT.class);
+
+ @BeforeAll
+ public static void start() throws Exception {
+ SharedMiniClusterBase.startMiniClusterWithConfig(
+ (cfg, coreSite) ->
cfg.getClusterServerConfiguration().setNumDefaultScanServers(1));
+ }
+
+ @AfterAll
+ public static void stop() {
+ SharedMiniClusterBase.stopMiniCluster();
}
@Override
- protected void configure(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
- cfg.getClusterServerConfiguration().setNumDefaultScanServers(1);
+ protected Duration defaultTimeout() {
+ return Duration.ofMinutes(1);
}
@Test
public void testScannerReadaheadConfiguration() throws Exception {
final String table = getUniqueNames(1)[0];
- try (AccumuloClient c =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
c.tableOperations().create(table);
try (BatchWriter bw = c.createBatchWriter(table)) {
@@ -166,7 +177,7 @@ public class ScannerIT extends ConfigurableMacBase {
public void testSessionCleanup(ConsistencyLevel consistency) throws
Exception {
final String tableName = getUniqueNames(1)[0] + "_" + consistency;
final ServerType serverType = consistency == IMMEDIATE ? TABLET_SERVER :
SCAN_SERVER;
- try (AccumuloClient accumuloClient =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient accumuloClient =
Accumulo.newClient().from(getClientProps()).build()) {
accumuloClient.tableOperations().create(tableName);
@@ -260,7 +271,7 @@ public class ScannerIT extends ConfigurableMacBase {
getCluster().getClusterControl().startAllServers(SCAN_SERVER);
var random = new SecureRandom();
- Properties props = getClientProperties();
+ Properties props = getClientProps();
// configure scan server not to fallback to tablet servers
String profiles = "[{'isDefault':true,'maxBusyTimeout':'1s',
'busyTimeoutMultiplier':8,"
+ "'timeToWaitForScanServers':10h, "
@@ -349,7 +360,7 @@ public class ScannerIT extends ConfigurableMacBase {
getCluster().getClusterControl().startAllServers(SCAN_SERVER);
final String table = getUniqueNames(1)[0];
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
client.tableOperations().create(table);
try (var writer = client.createBatchWriter(table)) {
@@ -393,6 +404,7 @@ public class ScannerIT extends ConfigurableMacBase {
} finally {
executor.shutdownNow();
}
+ client.tableOperations().delete(table);
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
index aba4391e2a..ac6065ff70 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
@@ -28,7 +28,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
import java.time.Duration;
import java.util.ArrayList;
@@ -55,7 +54,6 @@ import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.admin.CompactionConfig;
-import org.apache.accumulo.core.client.admin.InstanceOperations;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TabletMergeability;
import org.apache.accumulo.core.client.rfile.RFile;
@@ -74,18 +72,16 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.ServerType;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.server.util.checkCommand.MetadataCheckRunner;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.accumulo.test.VerifyIngest.VerifyParams;
-import org.apache.accumulo.test.harness.AccumuloClusterHarness;
+import org.apache.accumulo.test.harness.SharedMiniClusterBase;
import org.apache.accumulo.test.util.Wait;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -93,53 +89,28 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.common.collect.MoreCollectors;
-public class SplitIT extends AccumuloClusterHarness {
+public class SplitIT extends SharedMiniClusterBase {
private static final Logger log = LoggerFactory.getLogger(SplitIT.class);
- @Override
- protected Duration defaultTimeout() {
- return Duration.ofMinutes(4);
+ @BeforeAll
+ public static void start() throws Exception {
+ SharedMiniClusterBase.startMiniClusterWithConfig((cfg, coreSite) -> {
+ cfg.setProperty(Property.TSERV_MAXMEM, "5K");
+ cfg.setMemory(ServerType.TABLET_SERVER, 384, MemoryUnit.MEGABYTE);
+ // Splitting a tablet w/ a compaction can result in a dead compaction.
Run the detector more
+ // frequently to clean them up as they could cause tests to hang.
+
cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL,
"5s");
+ });
}
- @Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
- cfg.setProperty(Property.TSERV_MAXMEM, "5K");
- cfg.setMemory(ServerType.TABLET_SERVER, 384, MemoryUnit.MEGABYTE);
- // Splitting a tablet w/ a compaction can result in a dead compaction. Run
the detector more
- // frequently to clean them up as they could cause tests to hang.
-
cfg.setProperty(Property.COMPACTION_COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL,
"5s");
+ @AfterAll
+ public static void stop() {
+ SharedMiniClusterBase.stopMiniCluster();
}
- private String tservMaxMem;
-
- @BeforeEach
- public void alterConfig() throws Exception {
- assumeTrue(getClusterType() == ClusterType.MINI);
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- InstanceOperations iops = client.instanceOperations();
- Map<String,String> config = iops.getSystemConfiguration();
- tservMaxMem = config.get(Property.TSERV_MAXMEM.getKey());
-
- // Property.TSERV_MAXMEM can't be altered on a running server
- if (!tservMaxMem.equals("5K")) {
- iops.setProperty(Property.TSERV_MAXMEM.getKey(), "5K");
-
getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
-
getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
- }
- }
- }
-
- @AfterEach
- public void resetConfig() throws Exception {
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- if (tservMaxMem != null) {
- log.info("Resetting {}={}", Property.TSERV_MAXMEM.getKey(),
tservMaxMem);
-
client.instanceOperations().setProperty(Property.TSERV_MAXMEM.getKey(),
tservMaxMem);
- tservMaxMem = null;
-
getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
-
getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
- }
- }
+ @Override
+ protected Duration defaultTimeout() {
+ return Duration.ofMinutes(4);
}
// Test that checks the estimated file sizes created by a split are
reasonable
@@ -170,9 +141,9 @@ public class SplitIT extends AccumuloClusterHarness {
}
}
- var tableId = getServerContext().getTableId(table);
- var files = getServerContext().getAmple().readTablet(new
KeyExtent(tableId, null, null))
- .getFilesMap();
+ var tableId = getCluster().getServerContext().getTableId(table);
+ var files = getCluster().getServerContext().getAmple()
+ .readTablet(new KeyExtent(tableId, null, null)).getFilesMap();
// map of file name and the estimates for that file from the original
tablet
Map<String,DataFileValue> filesSizes1 = new HashMap<>();
@@ -187,8 +158,8 @@ public class SplitIT extends AccumuloClusterHarness {
// map of file name and the estimates for that file from all splits
Map<String,List<DataFileValue>> filesSizes2 = new HashMap<>();
- try (var tablets =
-
getServerContext().getAmple().readTablets().forTable(tableId).fetch(FILES).build())
{
+ try (var tablets =
getCluster().getServerContext().getAmple().readTablets().forTable(tableId)
+ .fetch(FILES).build()) {
for (var tablet : tablets) {
tablet.getFilesMap().forEach((file, dfv) -> filesSizes2
.computeIfAbsent(file.getFileName(), k -> new
ArrayList<>()).add(dfv));
@@ -517,8 +488,8 @@ public class SplitIT extends AccumuloClusterHarness {
// create a file with a single row and lots of columns. The files size
will exceed the split
// threshold configured above.
- try (
- RFileWriter writer =
RFile.newWriter().to(file).withFileSystem(getFileSystem()).build()) {
+ try (RFileWriter writer =
+
RFile.newWriter().to(file).withFileSystem(getCluster().getFileSystem()).build())
{
writer.startDefaultLocalityGroup();
for (int i = 0; i < 1000; i++) {
random.nextBytes(val);
@@ -531,7 +502,7 @@ public class SplitIT extends AccumuloClusterHarness {
c.tableOperations().importDirectory(dir).to(tableName).load();
// wait for the tablet to be marked unsplittable
- var ctx = getServerContext();
+ var ctx = getCluster().getServerContext();
Wait.waitFor(() -> {
var tableId = ctx.getTableId(tableName);
try (var tabletsMeta =
ctx.getAmple().readTablets().forTable(tableId).build()) {
@@ -568,7 +539,7 @@ public class SplitIT extends AccumuloClusterHarness {
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
- var ctx = getServerContext();
+ var ctx = getCluster().getServerContext();
var tableId = ctx.getTableId(tableName);
var extent = new KeyExtent(tableId, null, null);
