[accumulo] branch main updated (65c451a -> 8718d02)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 65c451a Minor warning cleanup from IDE add 8718d02 Improve tablet details in Monitor. Closes #1940 (#1943) No new revisions were added by this update. Summary of changes: .../src/main/resources/org/apache/accumulo/monitor/templates/server.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] branch main updated (8718d02 -> a17397d)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 8718d02 Improve tablet details in Monitor. Closes #1940 (#1943) add a17397d Increase difference between test values. Fixes #1789 (#1948) No new revisions were added by this update. Summary of changes: .../tserver/memory/LargestFirstMemoryManager.java | 2 +- .../memory/LargestFirstMemoryManagerTest.java | 55 -- 2 files changed, 30 insertions(+), 27 deletions(-)
[accumulo] branch main updated: Closes #1929. Add property verification for Sampler Options (#1944)
This is an automated email from the ASF dual-hosted git repository. jmanno pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new a8db761 Closes #1929. Add property verification for Sampler Options (#1944) a8db761 is described below commit a8db76146740770fe86a84ff5428b15f5c8e7e68 Author: Jeffrey Manno AuthorDate: Thu Feb 25 13:12:40 2021 -0500 Closes #1929. Add property verification for Sampler Options (#1944) Co-authored-by: Christopher Tubbs --- .../core/client/sample/AbstractHashSampler.java| 29 ++ .../core/client/sample/RowColumnSampler.java | 10 ++-- .../accumulo/core/client/sample/Sampler.java | 8 ++ .../accumulo/core/sample/impl/SamplerFactory.java | 13 +++--- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java b/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java index df22797..8f5f29c 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java +++ b/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java @@ -23,6 +23,7 @@ import static java.util.Objects.requireNonNull; import java.io.DataOutput; import java.io.IOException; +import java.util.Map; import java.util.Set; import org.apache.accumulo.core.data.Key; @@ -57,12 +58,34 @@ public abstract class AbstractHashSampler implements Sampler { private int modulus; private static final Set VALID_OPTIONS = Set.of("hasher", "modulus"); + private static final Set VALID_VALUES_HASHER = Set.of("murmur3_32", "md5", "sha1"); + + /** + * Subclasses with options should override this method to validate subclass options while also + * calling {@code super.validateOptions(config)} to validate base class options. + */ + @Override + public void validateOptions(Map config) { +for (Map.Entry entry : config.entrySet()) { + checkArgument(VALID_OPTIONS.contains(entry.getKey()), "Unknown option: %s", entry.getKey()); + + if (entry.getKey().equals("hasher")) +checkArgument(VALID_VALUES_HASHER.contains(entry.getValue()), +"Unknown value for hasher: %s", entry.getValue()); + + if (entry.getKey().equals("modulus")) +checkArgument(Integer.parseInt(entry.getValue()) > 0, +"Improper Integer value for modulus: %s", entry.getValue()); +} + } /** * Subclasses with options should override this method and return true if the option is valid for * the subclass or if {@code super.isValidOption(opt)} returns true. + * + * @deprecated since 2.1.0, replaced by {@link #validateOptions(Map)} */ - + @Deprecated(since = "2.1.0") protected boolean isValidOption(String option) { return VALID_OPTIONS.contains(option); } @@ -80,10 +103,6 @@ public abstract class AbstractHashSampler implements Sampler { requireNonNull(hasherOpt, "Hasher not specified"); requireNonNull(modulusOpt, "Modulus not specified"); -for (String option : config.getOptions().keySet()) { - checkArgument(isValidOption(option), "Unknown option : %s", option); -} - switch (hasherOpt) { case "murmur3_32": hashFunction = Hashing.murmur3_32(); diff --git a/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java b/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java index a2785e3..1d71214 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java +++ b/core/src/main/java/org/apache/accumulo/core/client/sample/RowColumnSampler.java @@ -18,8 +18,11 @@ */ package org.apache.accumulo.core.client.sample; +import static com.google.common.base.Preconditions.checkArgument; + import java.io.DataOutput; import java.io.IOException; +import java.util.Map; import java.util.Set; import org.apache.accumulo.core.data.ByteSequence; @@ -80,8 +83,11 @@ public class RowColumnSampler extends AbstractHashSampler { } @Override - protected boolean isValidOption(String option) { -return super.isValidOption(option) || VALID_OPTIONS.contains(option); + public void validateOptions(Map config) { +super.validateOptions(config); +for (String option : config.keySet()) { + checkArgument(VALID_OPTIONS.contains(option), "Unknown option : %s", option); +} } @Override diff --git a/core/src/main/java/org/apache/accumulo/core/client/sample/Sampler.java b/core/src/main/java/org/apache/accumulo/core/client/sample/Sampler.java index d7e07c8..810b8ae 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/sample/Sampler.java +++ b/core/src/main/java/org/apache/accumulo/core/client/sample/Sampler.java @@ -18,6 +18,8 @@ */ package org.apache.accumulo.core.client.sample;
[accumulo] branch main updated: Fix #1791 - Fix SuspendedTabletsIT (#1888)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 05abf9f Fix #1791 - Fix SuspendedTabletsIT (#1888) 05abf9f is described below commit 05abf9fefbb9de46f7d27b6f93d1a6a839dc6963 Author: Dom G <47725857+domgargu...@users.noreply.github.com> AuthorDate: Thu Feb 25 13:35:09 2021 -0500 Fix #1791 - Fix SuspendedTabletsIT (#1888) * ensure metadata table tablets are on a single server * removed server hosting metadata tablets from list to shutdown * pre-split tables at creation time instead of immediately after --- .../accumulo/test/manager/SuspendedTabletsIT.java | 71 +- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java index 92f6401..770648e 100644 --- a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java @@ -19,10 +19,8 @@ package org.apache.accumulo.test.manager; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.security.SecureRandom; @@ -45,8 +43,10 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.ManagerClient; +import org.apache.accumulo.core.clientImpl.TabletLocator; import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Range; @@ -54,6 +54,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; +import org.apache.accumulo.core.spi.balancer.HostRegexTableLoadBalancer; import org.apache.accumulo.core.util.HostAndPort; import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; @@ -77,7 +78,7 @@ public class SuspendedTabletsIT extends ConfigurableMacBase { private static ExecutorService THREAD_POOL; public static final int TSERVERS = 3; - public static final long SUSPEND_DURATION = 20; + public static final long SUSPEND_DURATION = 80; public static final int TABLETS = 30; @Override @@ -91,6 +92,11 @@ public class SuspendedTabletsIT extends ConfigurableMacBase { cfg.setClientProperty(ClientProperty.INSTANCE_ZOOKEEPERS_TIMEOUT, "5s"); cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s"); cfg.setNumTservers(TSERVERS); +// config custom balancer to keep all metadata on one server +cfg.setProperty(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + MetadataTable.NAME, "*"); +cfg.setProperty(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "7s"); +cfg.setProperty(Property.TABLE_LOAD_BALANCER.getKey(), +HostRegexTableLoadBalancer.class.getName()); } @Test @@ -115,13 +121,30 @@ public class SuspendedTabletsIT extends ConfigurableMacBase { // Run the test body. When we get to the point where we need tservers to go away, stop them via // a clean shutdown. suspensionTestBody((ctx, locs, count) -> { - Set tserversSet = new HashSet<>(); + Set tserverSet = new HashSet<>(); + Set metadataServerSet = new HashSet<>(); + + TabletLocator tl = TabletLocator.getLocator(ctx, MetadataTable.ID); for (TabletLocationState tls : locs.locationStates.values()) { if (tls.current != null) { - tserversSet.add(tls.current); + // add to set of all servers + tserverSet.add(tls.current); + + // get server that the current tablets metadata is on + TabletLocator.TabletLocation tab = + tl.locateTablet(ctx, tls.extent.toMetaRow(), false, false); + // add it to the set of servers with metadata + metadataServerSet + .add(new TServerInstance(tab.tablet_location, Long.valueOf(tab.tablet_session, 16))); } } - List tserversList = new ArrayList<>(tserversSet); + + // remove servers with metadata on them from the list of servers to be shutdown + tserverSet.removeAll(metadataServerSet); + + assertEquals("Expecting two tServers in shutd
[accumulo] branch main updated: Fix typo in RecoveryManager
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 7e3c67d Fix typo in RecoveryManager 7e3c67d is described below commit 7e3c67d0c0b831d7d29c69e24d07dd700f24c5e0 Author: Mike Miller AuthorDate: Thu Feb 25 15:37:53 2021 -0500 Fix typo in RecoveryManager --- .../main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index 4406f5a..c849b8d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -112,7 +112,7 @@ public class RecoveryManager { initiateSort(sortId, source, destination); } } catch (FileNotFoundException e) { -log.debug("Unable to initate log sort for " + source + ": " + e); +log.debug("Unable to initiate log sort for " + source + ": " + e); } catch (Exception e) { log.warn("Failed to initiate log sort " + source, e); } finally {