(accumulo) branch elasticity updated (b4db128dcd -> 0e3ed1d31f)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a change to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from b4db128dcd Remove coordinator from service status command (#4613)
add 04f5aaf7c2 Improves log message for missing iterator class (#4591)
add 38d9da710b Use micrometer noop classes for default meter
initialization (#4542)
add 510cfda7b6 Merge remote-tracking branch 'upstream/2.1'
add 0e3ed1d31f Merge remote-tracking branch 'upstream/main' into elasticity
No new revisions were added by this update.
Summary of changes:
.../server/metrics/NoOpDistributionSummary.java| 83 --
.../accumulo/server/metrics/NoopMetrics.java | 79
.../accumulo/server/metrics/ThriftMetrics.java | 4 +-
...butionSummaryTest.java => NoopMetricsTest.java} | 20 +-
.../accumulo/server/metrics/ThriftMetricsTest.java | 3 +-
.../apache/accumulo/tserver/ScanServerMetrics.java | 5 +-
.../tserver/metrics/TabletServerMinCMetrics.java | 5 +-
.../tserver/metrics/TabletServerScanMetrics.java | 9 +--
.../tserver/metrics/TabletServerUpdateMetrics.java | 10 +--
.../apache/accumulo/test/metrics/MetricsIT.java| 21 +++---
10 files changed, 130 insertions(+), 109 deletions(-)
delete mode 100644
server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java
create mode 100644
server/base/src/main/java/org/apache/accumulo/server/metrics/NoopMetrics.java
rename
server/base/src/test/java/org/apache/accumulo/server/metrics/{NoOpDistributionSummaryTest.java
=> NoopMetricsTest.java} (67%)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 510cfda7b69efc4491087d94e53db8cc645fd4e9
Merge: 7211081310 38d9da710b
Author: Ed Coleman
AuthorDate: Tue May 28 23:30:24 2024 +
Merge remote-tracking branch 'upstream/2.1'
.../server/metrics/NoOpDistributionSummary.java| 83 --
.../accumulo/server/metrics/NoopMetrics.java | 79
.../accumulo/server/metrics/ThriftMetrics.java | 4 +-
...butionSummaryTest.java => NoopMetricsTest.java} | 20 +-
.../accumulo/server/metrics/ThriftMetricsTest.java | 3 +-
.../apache/accumulo/tserver/ScanServerMetrics.java | 5 +-
.../tserver/metrics/TabletServerMinCMetrics.java | 5 +-
.../tserver/metrics/TabletServerScanMetrics.java | 9 +--
.../tserver/metrics/TabletServerUpdateMetrics.java | 10 +--
.../apache/accumulo/test/metrics/MetricsIT.java| 16 +++--
10 files changed, 128 insertions(+), 106 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
index 45cbdc6810,b56bc20b1a..46a3793861
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
@@@ -35,9 -37,9 +36,9 @@@ import io.micrometer.core.instrument.bi
public class ScanServerMetrics implements MetricsProducer {
- private Timer totalReservationTimer;
- private Timer writeOutReservationTimer;
+ private Timer totalReservationTimer = NoopMetrics.useNoopTimer();
+ private Timer writeOutReservationTimer = NoopMetrics.useNoopTimer();
- private Counter busyTimeoutCount = NoopMetrics.useNoopCounter();
+ private final AtomicLong busyTimeoutCount = new AtomicLong(0);
private final AtomicLong reservationConflictCount = new AtomicLong(0);
private final LoadingCache tabletMetadataCache;
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java
index 5ca15c5c9b,c2c8dd0d32..ea246c1c72
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerScanMetrics.java
@@@ -24,10 -23,10 +24,10 @@@ import java.util.concurrent.atomic.Atom
import java.util.concurrent.atomic.LongAdder;
import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.server.metrics.NoOpDistributionSummary;
+ import org.apache.accumulo.server.metrics.NoopMetrics;
-import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
+import io.micrometer.core.instrument.FunctionCounter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
@@@ -35,15 -34,13 +35,16 @@@
public class TabletServerScanMetrics implements MetricsProducer {
private final AtomicInteger openFiles = new AtomicInteger(0);
- private Timer scans;
- private DistributionSummary resultsPerScan = new NoOpDistributionSummary();
- private DistributionSummary yields = new NoOpDistributionSummary();
++
+ private Timer scans = NoopMetrics.useNoopTimer();
+ private DistributionSummary resultsPerScan =
NoopMetrics.useNoopDistributionSummary();
+ private DistributionSummary yields =
NoopMetrics.useNoopDistributionSummary();
- private Counter startScanCalls = NoopMetrics.useNoopCounter();
- private Counter continueScanCalls = NoopMetrics.useNoopCounter();;
- private Counter closeScanCalls = NoopMetrics.useNoopCounter();;
- private Counter busyTimeoutCount = NoopMetrics.useNoopCounter();;
+ private final AtomicLong startScanCalls = new AtomicLong(0);
+ private final AtomicLong continueScanCalls = new AtomicLong(0);
+ private final AtomicLong closeScanCalls = new AtomicLong(0);
+ private final AtomicLong busyTimeoutCount = new AtomicLong(0);
+ private final AtomicLong pausedForMemory = new AtomicLong(0);
+ private final AtomicLong earlyReturnForMemory = new AtomicLong(0);
private final LongAdder lookupCount = new LongAdder();
private final LongAdder queryResultCount = new LongAdder();
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java
index 6dbce396b1,a369458b6c..1c30517e32
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerUpdateMetrics.java
@@@ -19,28 -19,27 +19,28 @@@
package org.apache.accumulo.tserver.metrics;
import java.time.Duration;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.server.metrics.NoOpDistributionSumm
(accumulo) branch main updated (7211081310 -> 510cfda7b6)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 7211081310 Merge branch '2.1'
add 04f5aaf7c2 Improves log message for missing iterator class (#4591)
add 38d9da710b Use micrometer noop classes for default meter
initialization (#4542)
new 510cfda7b6 Merge remote-tracking branch 'upstream/2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../server/metrics/NoOpDistributionSummary.java| 83 --
.../accumulo/server/metrics/NoopMetrics.java | 79
.../accumulo/server/metrics/ThriftMetrics.java | 4 +-
...butionSummaryTest.java => NoopMetricsTest.java} | 20 +-
.../accumulo/server/metrics/ThriftMetricsTest.java | 3 +-
.../apache/accumulo/tserver/ScanServerMetrics.java | 5 +-
.../tserver/metrics/TabletServerMinCMetrics.java | 5 +-
.../tserver/metrics/TabletServerScanMetrics.java | 9 +--
.../tserver/metrics/TabletServerUpdateMetrics.java | 10 +--
.../apache/accumulo/test/metrics/MetricsIT.java| 16 +++--
10 files changed, 128 insertions(+), 106 deletions(-)
delete mode 100644
server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java
create mode 100644
server/base/src/main/java/org/apache/accumulo/server/metrics/NoopMetrics.java
rename
server/base/src/test/java/org/apache/accumulo/server/metrics/{NoOpDistributionSummaryTest.java
=> NoopMetricsTest.java} (67%)
(accumulo) branch 2.1 updated: Use micrometer noop classes for default meter initialization (#4542)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 38d9da710b Use micrometer noop classes for default meter
initialization (#4542)
38d9da710b is described below
commit 38d9da710b13ab7e81770ca015055da9a8b3d755
Author: EdColeman
AuthorDate: Tue May 28 18:50:05 2024 -0400
Use micrometer noop classes for default meter initialization (#4542)
* Use micrometer noop classes for default meter initialization
- removes custom NoOpDistributionSummary
- adds default initializations for other meter types
-organize MetricsIT excludes to ease future merging.
---
.../server/metrics/NoOpDistributionSummary.java| 83 --
.../accumulo/server/metrics/NoopMetrics.java | 79
.../accumulo/server/metrics/ThriftMetrics.java | 4 +-
...butionSummaryTest.java => NoopMetricsTest.java} | 20 +-
.../accumulo/server/metrics/ThriftMetricsTest.java | 3 +-
.../manager/metrics/ReplicationMetrics.java| 3 +-
.../apache/accumulo/tserver/ScanServerMetrics.java | 7 +-
.../tserver/metrics/TabletServerMinCMetrics.java | 5 +-
.../tserver/metrics/TabletServerScanMetrics.java | 16 ++---
.../tserver/metrics/TabletServerUpdateMetrics.java | 16 ++---
.../apache/accumulo/test/metrics/MetricsIT.java| 20 --
11 files changed, 140 insertions(+), 116 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java
b/server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java
deleted file mode 100644
index 262dc0802a..00
---
a/server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.metrics;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.micrometer.core.instrument.DistributionSummary;
-import io.micrometer.core.instrument.Tag;
-import io.micrometer.core.instrument.Tags;
-import io.micrometer.core.instrument.distribution.HistogramSnapshot;
-
-/**
- * Provides a default DistributionSummary that does not do anything. This can
be used to prevent NPE
- * if metrics have not been initialized when a DistributionSummary is expected.
- *
- * Normally DistributionSummaries are created using a builder that takes a
registry.
- *
- *
- * DistributionSummary distSum;
- * ...
- * public void registerMetrics(MeterRegistry registry) {
- * ...
- * distSum = DistributionSummary.builder("metric
name").description("...").register(registry);
- * ...
- * }
- *
- *
- * Until the registration is called, the instance variable is null. If code
then tries to update the
- * metric, a NPE is thrown. Using this class to provide a default value
prevents this from occurring
- * and on registration, it is replaced with a valid instance.
- */
-public class NoOpDistributionSummary implements DistributionSummary {
-
- private static final Logger LOG =
LoggerFactory.getLogger(NoOpDistributionSummary.class);
-
- @Override
- public void record(double v) {
-LOG.debug("record ignored - distribution summary will not be available.");
- }
-
- @Override
- public long count() {
-return 0;
- }
-
- @Override
- public double totalAmount() {
-return 0;
- }
-
- @Override
- public double max() {
-return 0;
- }
-
- @Override
- public HistogramSnapshot takeSnapshot() {
-return new HistogramSnapshot(0L, 0.0, 0.0, null, null, null);
- }
-
- @Override
- public Id getId() {
-return new Id("thrift.metrics.uninitialized", Tags.of(Tag.of("none",
"uninitialized")), null,
-"placeholder for uninitialized thrift metrics", Type.OTHER);
- }
-}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metrics/NoopMetrics.java
b/server/base/src/main/java/org/apache/accumulo/server/metrics/NoopMetric
(accumulo) branch elasticity updated: Remove coordinator from service status command (#4613)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new b4db128dcd Remove coordinator from service status command (#4613)
b4db128dcd is described below
commit b4db128dcda12bf1435a195188d71cd8e71b94e5
Author: EdColeman
AuthorDate: Tue May 28 18:11:13 2024 -0400
Remove coordinator from service status command (#4613)
- The coordinator moved to the manager and is no longer a stand-alone
service.
---
.../accumulo/server/util/ServiceStatusCmd.java | 11
.../accumulo/server/util/ServiceStatusCmdTest.java | 59 --
2 files changed, 70 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
index 9b65fed0af..25928b0505 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
@@ -71,8 +71,6 @@ public class ServiceStatusCmd {
services.put(ServiceStatusReport.ReportKey.MONITOR,
getMonitorStatus(zooReader, zooRoot));
services.put(ServiceStatusReport.ReportKey.T_SERVER,
getTServerStatus(zooReader, zooRoot));
services.put(ServiceStatusReport.ReportKey.S_SERVER,
getScanServerStatus(zooReader, zooRoot));
-services.put(ServiceStatusReport.ReportKey.COORDINATOR,
-getCoordinatorStatus(zooReader, zooRoot));
services.put(ServiceStatusReport.ReportKey.COMPACTOR,
getCompactorStatus(zooReader, zooRoot));
services.put(ServiceStatusReport.ReportKey.GC, getGcStatus(zooReader,
zooRoot));
@@ -178,15 +176,6 @@ public class ServiceStatusCmd {
return getStatusSummary(ServiceStatusReport.ReportKey.GC, zooReader,
lockPath);
}
- /**
- * The coordinator paths in ZooKeeper are: {@code
/accumulo/[IID]/coordinators/lock/zlock#[NUM]}
- * with the lock data providing host:port
- */
- @VisibleForTesting
- StatusSummary getCoordinatorStatus(final ZooReader zooReader, String
zRootPath) {
-throw new UnsupportedOperationException();
- }
-
/**
* The compactor paths in ZooKeeper are:
* {@code /accumulo/[IID]/compactors/[QUEUE_NAME]/host:port/zlock#[NUM]}
with the host:port pulled
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java
b/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java
index 93551fcbe8..6f82e589cc 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java
@@ -324,63 +324,6 @@ public class ServiceStatusCmdTest {
}
- @Test
- void testCoordinatorHosts() throws Exception {
-String lock1Name = "zlock#" + UUID.randomUUID() + "#01";
-String lock2Name = "zlock#" + UUID.randomUUID() + "#02";
-String lock3Name = "zlock#" + UUID.randomUUID() + "#03";
-
-String host1 = "hostA:8080";
-String host2 = "hostB:9090";
-String host3 = "host1:9091";
-
-String lockData1 =
-
"{\"descriptors\":[{\"uuid\":\"1d55f7a5-090d-48fc-a3ea-f1a66e984a21\",\"service\":\"COORDINATOR\",\"address\":\""
-+ host1 + "\",\"group\":\"default\"}]}\n";
-String lockData2 =
-
"{\"descriptors\":[{\"uuid\":\"1d55f7a5-090d-48fc-a3ea-f1a66e984a21\",\"service\":\"COORDINATOR\",\"address\":\""
-+ host2 + "\",\"group\":\"coord1\"}]}\n";
-String lockData3 =
-
"{\"descriptors\":[{\"uuid\":\"1d55f7a5-090d-48fc-a3ea-f1a66e984a21\",\"service\":\"COORDINATOR\",\"address\":\""
-+ host3 + "\",\"group\":\"coord2\"}]}\n";
-
-String lockPath = null;
-// String lockPath = zRoot + Constants.ZCOORDINATOR_LOCK;
-expect(zooReader.getChildren(eq(lockPath))).andReturn(List.of(lock1Name,
lock2Name, lock3Name))
-.anyTimes();
-expect(zooReader.getData(eq(lockPath + "/" +
lock1Name))).andReturn(lockData1.getBytes(UTF_8))
-.anyTimes();
-expect(zooReader.getData(eq(lockPath + "/" +
lock2Name))).andReturn(lockData2.getBytes(UTF_8))
-.anyTimes();
-expect(zooReader.getData(eq(lockPath + "/" +
lock3Name))).andReturn(lockData3.getBytes(UTF_8))
-.anyTimes();
-
-replay(zooReader);
-
-Ser
(accumulo) branch main updated: Admin service status server lock changes (#4605)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 814ac68a67 Admin service status server lock changes (#4605)
814ac68a67 is described below
commit 814ac68a6732601b7bbcbe61f6785741deee7ca5
Author: EdColeman
AuthorDate: Tue May 28 09:42:29 2024 -0400
Admin service status server lock changes (#4605)
* update admin serviceStatus to read host info from ServiceLockData
---
.../accumulo/server/util/ServiceStatusCmd.java | 63 +++---
.../util/serviceStatus/ServiceStatusReport.java| 29 ++-
.../accumulo/server/util/ServiceStatusCmdTest.java | 214 +++--
.../serviceStatus/ServiceStatusReportTest.java | 22 +--
4 files changed, 226 insertions(+), 102 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
index 3f5dbafaf5..b042fcf838 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
@@ -19,6 +19,9 @@
package org.apache.accumulo.server.util;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.accumulo.core.lock.ServiceLockData.ThriftService.TABLET_SCAN;
+import static
org.apache.accumulo.core.lock.ServiceLockData.ThriftService.TSERV;
+import static org.apache.accumulo.core.util.LazySingletons.GSON;
import java.util.Collection;
import java.util.Map;
@@ -29,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.fate.zookeeper.ZooReader;
+import org.apache.accumulo.core.lock.ServiceLockData;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.util.serviceStatus.ServiceStatusReport;
@@ -85,7 +89,7 @@ public class ServiceStatusCmd {
/**
* The manager paths in ZooKeeper are: {@code
/accumulo/[IID]/managers/lock/zlock#[NUM]} with the
- * lock data providing host:port.
+ * lock data providing a service descriptor with host and port.
*/
@VisibleForTesting
StatusSummary getManagerStatus(final ZooReader zooReader, String zRootPath) {
@@ -95,7 +99,7 @@ public class ServiceStatusCmd {
/**
* The monitor paths in ZooKeeper are: {@code
/accumulo/[IID]/monitor/lock/zlock#[NUM]} with the
- * lock data providing host:port.
+ * lock data providing a service descriptor with host and port.
*/
@VisibleForTesting
StatusSummary getMonitorStatus(final ZooReader zooReader, String zRootPath) {
@@ -110,7 +114,7 @@ public class ServiceStatusCmd {
@VisibleForTesting
StatusSummary getTServerStatus(final ZooReader zooReader, String zRootPath) {
String lockPath = zRootPath + Constants.ZTSERVERS;
-return getServerHostStatus(zooReader, lockPath,
ServiceStatusReport.ReportKey.T_SERVER);
+return getServerHostStatus(zooReader, lockPath,
ServiceStatusReport.ReportKey.T_SERVER, TSERV);
}
/**
@@ -120,7 +124,8 @@ public class ServiceStatusCmd {
@VisibleForTesting
StatusSummary getScanServerStatus(final ZooReader zooReader, String
zRootPath) {
String lockPath = zRootPath + Constants.ZSSERVERS;
-return getServerHostStatus(zooReader, lockPath,
ServiceStatusReport.ReportKey.S_SERVER);
+return getServerHostStatus(zooReader, lockPath,
ServiceStatusReport.ReportKey.S_SERVER,
+TABLET_SCAN);
}
/**
@@ -128,7 +133,7 @@ public class ServiceStatusCmd {
* {@code /accumulo/IID/[tservers | sservers]/HOST:PORT/[LOCK]}
*/
private StatusSummary getServerHostStatus(final ZooReader zooReader, String
basePath,
- ServiceStatusReport.ReportKey displayNames) {
+ ServiceStatusReport.ReportKey displayNames,
ServiceLockData.ThriftService serviceType) {
AtomicInteger errorSum = new AtomicInteger(0);
// Set hostNames = new TreeSet<>();
@@ -137,25 +142,26 @@ public class ServiceStatusCmd {
var nodeNames = readNodeNames(zooReader, basePath);
-nodeNames.getHosts().forEach(host -> {
+nodeNames.getData().forEach(host -> {
var lock = readNodeNames(zooReader, basePath + "/" + host);
- lock.getHosts().forEach(l -> {
+ lock.getData().forEach(l -> {
var nodeData = readNodeData(zooReader, basePath + "/" + host + "/" +
l);
int err = nodeData.getErrorCount();
if (err > 0) {
errorSum.addAndGet(nodeData.getErrorCount());
} else {
- // process resource groups
- String[] tokens = nodeData.getHosts().split(",");
- if (tokens.length == 2) {
-String
(accumulo) branch main updated (f2005ca011 -> 8e5d967f24)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from f2005ca011 Use Duration instead of long+TimeUnit in ReadOnlyTStore.unreserve (#4371) add e3c540d246 Add service status as command option to admin command (#4567) add 4426b90418 adds trace logging to CompactionDriver (#4578) new 8e5d967f24 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/server/util/Admin.java | 42 ++- .../accumulo/server/util/ServiceStatusCmd.java | 342 + .../util/serviceStatus/ServiceStatusReport.java| 183 ++ .../server/util/serviceStatus/StatusSummary.java | 92 + .../accumulo/server/util/ServiceStatusCmdTest.java | 406 + .../serviceStatus/ServiceStatusReportTest.java | 157 .../manager/tableOps/compact/CompactionDriver.java | 23 ++ 7 files changed, 1232 insertions(+), 13 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/util/serviceStatus/ServiceStatusReport.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/util/serviceStatus/StatusSummary.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/util/ServiceStatusCmdTest.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/util/serviceStatus/ServiceStatusReportTest.java
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 8e5d967f24870caa66659f54a3379749c84c8a9a
Merge: f2005ca011 4426b90418
Author: Ed Coleman
AuthorDate: Fri May 24 21:14:23 2024 +
Merge remote-tracking branch 'upstream/2.1'
- the merge of #4567 needs additional changes because of ServiceLock
data in ZooKeeper. This will be done as a separate PR.
.../org/apache/accumulo/server/util/Admin.java | 42 ++-
.../accumulo/server/util/ServiceStatusCmd.java | 342 +
.../util/serviceStatus/ServiceStatusReport.java| 183 ++
.../server/util/serviceStatus/StatusSummary.java | 92 +
.../accumulo/server/util/ServiceStatusCmdTest.java | 406 +
.../serviceStatus/ServiceStatusReportTest.java | 157
.../manager/tableOps/compact/CompactionDriver.java | 23 ++
7 files changed, 1232 insertions(+), 13 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
index 00,204e536c93..3f5dbafaf5
mode 00,100644..100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ServiceStatusCmd.java
@@@ -1,0 -1,368 +1,342 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.accumulo.server.util;
+
+ import static java.nio.charset.StandardCharsets.UTF_8;
+
+ import java.util.Collection;
+ import java.util.Map;
+ import java.util.Set;
+ import java.util.TreeMap;
+ import java.util.TreeSet;
+ import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
+
+ import org.apache.accumulo.core.Constants;
+ import org.apache.accumulo.core.fate.zookeeper.ZooReader;
+ import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.ServerServices;
+ import org.apache.accumulo.server.ServerContext;
+ import org.apache.accumulo.server.util.serviceStatus.ServiceStatusReport;
+ import org.apache.accumulo.server.util.serviceStatus.StatusSummary;
+ import org.apache.zookeeper.KeeperException;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import com.beust.jcommander.Parameter;
+ import com.beust.jcommander.Parameters;
+ import com.google.common.annotations.VisibleForTesting;
+
+ public class ServiceStatusCmd {
+
+ // used when grouping by resource group when there is no group.
+ public static final String NO_GROUP_TAG = "NO_GROUP";
+
+ private static final Logger LOG =
LoggerFactory.getLogger(ServiceStatusCmd.class);
+
+ public ServiceStatusCmd() {}
+
+ /**
+* Read the service statuses from ZooKeeper, build the status report and
then output the report to
+* stdout.
+*/
+ public void execute(final ServerContext context, final Opts opts) {
+
+ ZooReader zooReader = context.getZooReader();
+
+ final String zooRoot = context.getZooKeeperRoot();
+ LOG.trace("zooRoot: {}", zooRoot);
+
+ final Map services = new
TreeMap<>();
+
+ services.put(ServiceStatusReport.ReportKey.MANAGER,
getManagerStatus(zooReader, zooRoot));
+ services.put(ServiceStatusReport.ReportKey.MONITOR,
getMonitorStatus(zooReader, zooRoot));
+ services.put(ServiceStatusReport.ReportKey.T_SERVER,
getTServerStatus(zooReader, zooRoot));
+ services.put(ServiceStatusReport.ReportKey.S_SERVER,
getScanServerStatus(zooReader, zooRoot));
+ services.put(ServiceStatusReport.ReportKey.COORDINATOR,
+ getCoordinatorStatus(zooReader, zooRoot));
+ services.put(ServiceStatusReport.ReportKey.COMPACTOR,
getCompactorStatus(zooReader, zooRoot));
+ services.put(ServiceStatusReport.ReportKey.GC, getGcStatus(zooReader,
zooRoot));
+
+ ServiceStatusReport report = new ServiceStatusReport(services,
opts.noHosts);
+
+ if (opts.json) {
+ System.out.println(report.toJson());
+ } else {
+ StringBuilder sb = new StringBuilder(8192);
+ report.report(sb);
+ System.out.println(sb);
+ }
+
(accumulo) branch 2.1 updated: Add service status as command option to admin command (#4567)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e3c540d246 Add service status as command option to admin command
(#4567)
e3c540d246 is described below
commit e3c540d246f00330f7e5a18aa9f07b2ff961527f
Author: EdColeman
AuthorDate: Fri May 24 13:18:01 2024 -0400
Add service status as command option to admin command (#4567)
Add service status as command option to admin command
* Organizes service status by resource group.
---
.../org/apache/accumulo/server/util/Admin.java | 42 ++-
.../accumulo/server/util/ServiceStatusCmd.java | 368 +++
.../util/serviceStatus/ServiceStatusReport.java| 183 ++
.../server/util/serviceStatus/StatusSummary.java | 92 +
.../accumulo/server/util/ServiceStatusCmdTest.java | 406 +
.../serviceStatus/ServiceStatusReportTest.java | 157
6 files changed, 1235 insertions(+), 13 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
index ff9c4d9592..b8351ee43f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
@@ -283,6 +283,9 @@ public class Admin implements KeywordExecutable {
JCommander cl = new JCommander(opts);
cl.setProgramName("accumulo admin");
+ServiceStatusCmd.Opts serviceStatusCommandOpts = new
ServiceStatusCmd.Opts();
+cl.addCommand("serviceStatus", serviceStatusCommandOpts);
+
ChangeSecretCommand changeSecretCommand = new ChangeSecretCommand();
cl.addCommand("changeSecret", changeSecretCommand);
@@ -398,6 +401,8 @@ public class Admin implements KeywordExecutable {
tServerLocksOpts.delete);
} else if (cl.getParsedCommand().equals("fate")) {
executeFateOpsCommand(context, fateOpsCommand);
+ } else if (cl.getParsedCommand().equals("serviceStatus")) {
+printServiceStatus(context, serviceStatusCommandOpts);
} else {
everything = cl.getParsedCommand().equals("stopAll");
@@ -425,6 +430,11 @@ public class Admin implements KeywordExecutable {
}
}
+ private static void printServiceStatus(ServerContext context,
ServiceStatusCmd.Opts opts) {
+ServiceStatusCmd ssc = new ServiceStatusCmd();
+ssc.execute(context, opts);
+ }
+
private static int ping(ClientContext context, List args) {
InstanceOperations io = context.instanceOperations();
@@ -590,21 +600,11 @@ public class Admin implements KeywordExecutable {
private Map siteConfig, systemConfig;
private List localUsers;
- @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
- justification = "code runs in same security context as user who provided
input")
public void printConfig(ClientContext context, DumpConfigCommand opts)
throws Exception {
-File outputDirectory = null;
-if (opts.directory != null) {
- outputDirectory = new File(opts.directory);
- if (!outputDirectory.isDirectory()) {
-throw new IllegalArgumentException(
-opts.directory + " does not exist on the local filesystem.");
- }
- if (!outputDirectory.canWrite()) {
-throw new IllegalArgumentException(opts.directory + " is not
writable");
- }
-}
+@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
+justification = "app is run in same security context as user providing
the filename")
+File outputDirectory = getOutputDirectory(opts.directory);
defaultConfig = DefaultConfiguration.getInstance();
siteConfig = context.instanceOperations().getSiteConfiguration();
systemConfig = context.instanceOperations().getSystemConfiguration();
@@ -651,6 +651,22 @@ public class Admin implements KeywordExecutable {
}
}
+ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
+ justification = "app is run in same security context as user providing
the filename")
+ private static File getOutputDirectory(final String directory) {
+File outputDirectory = null;
+if (directory != null) {
+ outputDirectory = new File(directory);
+ if (!outputDirectory.isDirectory()) {
+throw new IllegalArgumentException(directory + " does not exist on the
local filesystem.");
+ }
+ if (!outputDirectory.canWrite()) {
+throw new IllegalArgumentException(directory + " is not writable");
+ }
+}
+return outputDirectory;
+ }
+
private String getDefaultConfigValue(String key) {
if (key == null) {
return null;
diff --git
a/server/base/
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 6ab37857263c6baab3806b9dc5921c8bbeff7780 Merge: 4a5d8da7a5 5e48ff92fb Author: Ed Coleman AuthorDate: Fri May 24 16:00:29 2024 + Merge remote-tracking branch 'upstream/2.1' test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --cc test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java index 0b76970d6d,5d9c833be5..7667f84207 --- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java @@@ -99,9 -99,12 +99,11 @@@ public class MetricsIT extends Configur doWorkToGenerateMetrics(); cluster.stop(); -Set unexpectedMetrics = Set.of(METRICS_SCAN_YIELDS, METRICS_UPDATE_ERRORS, -METRICS_REPLICATION_QUEUE, METRICS_COMPACTOR_MAJC_STUCK, METRICS_SCAN_BUSY_TIMEOUT_COUNTER); +Set unexpectedMetrics = Set.of(METRICS_COMPACTOR_MAJC_STUCK, METRICS_SCAN_YIELDS); + // add sserver as flaky until scan server included in mini tests. -Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS, -METRICS_SCAN_BUSY_TIMEOUT_COUNTER, METRICS_SCAN_RESERVATION_TOTAL_TIMER, -METRICS_SCAN_RESERVATION_WRITEOUT_TIMER, METRICS_SCAN_RESERVATION_CONFLICT_COUNTER, +Set flakyMetrics = Set.of(METRICS_FATE_TYPE_IN_PROGRESS, METRICS_GC_WAL_ERRORS, - METRICS_SCAN_RESERVATION_TOTAL_TIMER, METRICS_SCAN_BUSY_TIMEOUT_COUNTER, ++METRICS_SCAN_BUSY_TIMEOUT_COUNTER, METRICS_SCAN_RESERVATION_CONFLICT_COUNTER, ++METRICS_SCAN_RESERVATION_TOTAL_TIMER, METRICS_SCAN_RESERVATION_WRITEOUT_TIMER, METRICS_SCAN_TABLET_METADATA_CACHE); Map expectedMetricNames = this.getMetricFields();
(accumulo) branch main updated (4a5d8da7a5 -> 6ab3785726)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4a5d8da7a5 Use NanoTime to measure reservation time for scan server metrics add 5e48ff92fb Add new scan metrics to flaky list in MetricsIT (#4601) new 6ab3785726 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
(accumulo) branch 2.1 updated (5f571506dd -> 5e48ff92fb)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5f571506dd Fixed NPE in ScanServerMetrics (#4598) add 5e48ff92fb Add new scan metrics to flaky list in MetricsIT (#4601) No new revisions were added by this update. Summary of changes: test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java | 1 + 1 file changed, 1 insertion(+)
(accumulo) branch elasticity updated (14efea2e75 -> f9dabfc250)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 14efea2e75 ignores failures to set future location (#4579) add b21837a39e Add metrics for scan server reservation write out time and collisions (#4577) add 5f571506dd Fixed NPE in ScanServerMetrics (#4598) add cc7e56c4d9 Merge remote-tracking branch 'upstream/2.1' add 4a5d8da7a5 Use NanoTime to measure reservation time for scan server metrics new f9dabfc250 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metrics/MetricsProducer.java | 8 +++-- .../org/apache/accumulo/tserver/ScanServer.java| 15 + .../apache/accumulo/tserver/ScanServerMetrics.java | 38 ++ .../apache/accumulo/test/metrics/MetricsIT.java| 3 +- 4 files changed, 47 insertions(+), 17 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f9dabfc2501b893695bb04abf7d387f696851b98
Merge: 14efea2e75 4a5d8da7a5
Author: Ed Coleman
AuthorDate: Fri May 24 15:23:38 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../accumulo/core/metrics/MetricsProducer.java | 8 +++--
.../org/apache/accumulo/tserver/ScanServer.java| 15 +
.../apache/accumulo/tserver/ScanServerMetrics.java | 38 ++
.../apache/accumulo/test/metrics/MetricsIT.java| 3 +-
4 files changed, 47 insertions(+), 17 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index f9aed7e2cf,616800168e..323117c0c3
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@@ -90,8 -89,8 +90,9 @@@ import org.apache.accumulo.core.tablets
import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.util.Halt;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.cache.Caches.CacheName;
import org.apache.accumulo.core.util.threads.ThreadPools;
+ import org.apache.accumulo.core.util.time.NanoTime;
import org.apache.accumulo.server.AbstractServer;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.client.ClientServiceHandler;
diff --cc test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
index 53f981,0b76970d6d..402ed1ff50
--- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
@@@ -103,13 -96,13 +103,14 @@@ public class MetricsIT extends Configur
@Test
public void confirmMetricsPublished() throws Exception {
-doWorkToGenerateMetrics();
-cluster.stop();
-
Set unexpectedMetrics = Set.of(METRICS_COMPACTOR_MAJC_STUCK,
METRICS_SCAN_YIELDS);
-Set flakyMetrics = Set.of(METRICS_FATE_TYPE_IN_PROGRESS,
METRICS_GC_WAL_ERRORS,
-METRICS_SCAN_RESERVATION_TOTAL_TIMER,
METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
-METRICS_SCAN_TABLET_METADATA_CACHE);
+Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS,
METRICS_FATE_TYPE_IN_PROGRESS,
+METRICS_TSERVER_TABLETS_ONLINE_ONDEMAND,
METRICS_TSERVER_TABLETS_ONDEMAND_UNLOADED_FOR_MEM,
+METRICS_COMPACTOR_MAJC_STUCK, METRICS_MANAGER_ROOT_TGW_ERRORS,
+METRICS_MANAGER_META_TGW_ERRORS, METRICS_MANAGER_USER_TGW_ERRORS,
+METRICS_SCAN_TABLET_METADATA_CACHE, METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
- METRICS_SCAN_RESERVATION_TIMER, METRICS_SERVER_IDLE);
++METRICS_SCAN_RESERVATION_TOTAL_TIMER,
METRICS_SCAN_RESERVATION_WRITEOUT_TIMER,
++METRICS_SERVER_IDLE);
Map expectedMetricNames = this.getMetricFields();
flakyMetrics.forEach(expectedMetricNames::remove); // might not see these
(accumulo) branch elasticity updated (6afdc88f8e -> 6c0f610ed2)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 6afdc88f8e Modified ScanServer to not allow scans on Fate table as well add 5cd9cdfc70 Update MetricsIT (#4576) add c53bebc3e9 Added Scan Server Group Configuration IT (#4506) add 52be928da0 Merge branch '2.1' add 6c0f610ed2 Merge remote-tracking branch 'upstream/main' into elasticity No new revisions were added by this update. Summary of changes: .../test/ScanServerGroupConfigurationIT.java | 198 + .../org/apache/accumulo/test/ScanServerIT.java | 4 +- .../apache/accumulo/test/metrics/MetricsIT.java| 6 +- 3 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 test/src/main/java/org/apache/accumulo/test/ScanServerGroupConfigurationIT.java
(accumulo) branch main updated: Update MetricsIT (#4576)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5cd9cdfc70 Update MetricsIT (#4576)
5cd9cdfc70 is described below
commit 5cd9cdfc70e8da152d31d23f547eb9f005beba82
Author: EdColeman
AuthorDate: Mon May 20 09:32:52 2024 -0400
Update MetricsIT (#4576)
Changes to counters are now publishing metrics that may have not seen
before:
- METRICS_MAJC_PAUSED
- METRICS_MINC_PAUSED
- METRICS_SCAN_PAUSED_FOR_MEM
- METRICS_SCAN_RETURN_FOR_MEM
- METRICS_UPDATE_ERRORS
Also sorted metrics excludes list to make things easier to find.
---
.../main/java/org/apache/accumulo/test/metrics/MetricsIT.java | 11 ---
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
index f2be6e71a5..35efb186ef 100644
--- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
@@ -99,13 +99,10 @@ public class MetricsIT extends ConfigurableMacBase
implements MetricsProducer {
doWorkToGenerateMetrics();
cluster.stop();
-Set unexpectedMetrics =
-Set.of(METRICS_SCAN_YIELDS, METRICS_UPDATE_ERRORS,
METRICS_COMPACTOR_MAJC_STUCK,
-METRICS_SCAN_BUSY_TIMEOUT_COUNTER, METRICS_SCAN_PAUSED_FOR_MEM,
-METRICS_SCAN_RETURN_FOR_MEM, METRICS_MINC_PAUSED,
METRICS_MAJC_PAUSED);
-Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS,
METRICS_FATE_TYPE_IN_PROGRESS,
-METRICS_SCAN_BUSY_TIMEOUT_COUNTER, METRICS_SCAN_RESERVATION_TIMER,
-METRICS_SCAN_TABLET_METADATA_CACHE);
+Set unexpectedMetrics = Set.of(METRICS_COMPACTOR_MAJC_STUCK,
METRICS_SCAN_YIELDS);
+Set flakyMetrics =
+Set.of(METRICS_FATE_TYPE_IN_PROGRESS, METRICS_GC_WAL_ERRORS,
METRICS_SCAN_RESERVATION_TIMER,
+METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
METRICS_SCAN_TABLET_METADATA_CACHE);
Map expectedMetricNames = this.getMetricFields();
flakyMetrics.forEach(expectedMetricNames::remove); // might not see these
(accumulo) branch main updated (220cdc54da -> a42c431506)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 220cdc54da Merge branch '2.1' add 4b5234bd87 Added ZK cleanup thread to Manager for Scan Server nodes (#4562) new a42c431506 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/manager/Manager.java | 51 +- 1 file changed, 49 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a42c4315061548e8edc03d45e8bfd4d6c141fd11
Merge: 220cdc54da 4b5234bd87
Author: Ed Coleman
AuthorDate: Thu May 16 20:52:34 2024 +
Merge remote-tracking branch 'upstream/2.1'
.../java/org/apache/accumulo/manager/Manager.java | 51 +-
1 file changed, 49 insertions(+), 2 deletions(-)
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index ded6d62f83,84e8e68519..a778744c0f
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -39,6 -38,6 +39,7 @@@ import java.util.Iterator
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
++import java.util.Optional;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@@ -71,6 -69,10 +72,7 @@@ import org.apache.accumulo.core.data.Va
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.fate.AgeOffStore;
import org.apache.accumulo.core.fate.Fate;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock.LockLossReason;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock.ServiceLockPath;
+ import org.apache.accumulo.core.fate.zookeeper.ZooCache.ZcStat;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy;
@@@ -737,6 -741,48 +739,49 @@@ public class Manager extends AbstractSe
}
}
+ private class ScanServerZKCleaner implements Runnable {
+
+ @Override
+ public void run() {
+
+ final ZooReaderWriter zrw = getContext().getZooReaderWriter();
+ final String sserverZNodePath = getContext().getZooKeeperRoot() +
Constants.ZSSERVERS;
+
+ while (stillManager()) {
+ try {
+ for (String sserverClientAddress :
zrw.getChildren(sserverZNodePath)) {
+
+ final String sServerZPath = sserverZNodePath + "/" +
sserverClientAddress;
+ final var zLockPath = ServiceLock.path(sServerZPath);
+ ZcStat stat = new ZcStat();
-byte[] lockData =
ServiceLock.getLockData(getContext().getZooCache(), zLockPath, stat);
++Optional lockData =
++ServiceLock.getLockData(getContext().getZooCache(),
zLockPath, stat);
+
-if (lockData == null) {
++if (lockData.isEmpty()) {
+ try {
+ log.debug("Deleting empty ScanServer ZK node {}",
sServerZPath);
+ zrw.delete(sServerZPath);
+ } catch (KeeperException.NotEmptyException e) {
+ log.debug(
+ "Failed to delete ScanServer ZK node {} its not empty,
likely an expected race condition.",
+ sServerZPath);
+ }
+ }
+ }
+ } catch (KeeperException e) {
+ log.error("Exception trying to delete empty scan server ZNodes,
will retry", e);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ log.error("Interrupted trying to delete empty scan server ZNodes,
will retry", e);
+ } finally {
+ // sleep for 5 mins
+ sleepUninterruptibly(CLEANUP_INTERVAL_MINUTES, MINUTES);
+ }
+ }
+ }
+
+ }
+
private class StatusThread implements Runnable {
private boolean goodStats() {
(accumulo) branch elasticity updated (5e4ec67ac0 -> ef34a4890a)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5e4ec67ac0 Merge branch 'main' into elasticity add 3cac724f00 Renames a fate metric (#4543) add 8c485e362d Merge remote-tracking branch 'upstream/2.1' new ef34a4890a Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 7 --- .../java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit ef34a4890a76b592a805947d7c8f7d99a0363952
Merge: 5e4ec67ac0 8c485e362d
Author: Ed Coleman
AuthorDate: Fri May 10 21:34:39 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 7 ---
.../java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 6f82281257,abfdfc9b5a..bdc695e787
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -59,53 -59,12 +59,54 @@@ import io.micrometer.core.instrument.Me
*
*
*
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUES}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED}
+ * Gauge
+ *
+ *
+ *
* currentFateOps
* Gauge
- * {@value #METRICS_FATE_TOTAL_IN_PROGRESS}
+ * {@value #METRICS_FATE_OPS}
* Gauge
- *
+ * Was previously named "accumulo.fate.ops.in.progress". Changed to
better reflect what the
+ * gauge is actually tracking which is all the current fate ops in any
state.
*
*
* FateTxOpType_{name}
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
index 1664dd8d93,2f66aef256..e4e5150671
---
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
@@@ -126,29 -128,31 +126,29 @@@ public class FateMetrics implements Met
@Override
public void registerMetrics(final MeterRegistry registry) {
- registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS, totalCurrentOpsGauge);
-totalCurrentOpsGauge = registry.gauge(METRICS_FATE_OPS, new
AtomicLong(0));
-totalOpsGauge = registry.gauge(METRICS_FATE_OPS_ACTIVITY, new
AtomicLong(0));
-fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS,
List.of(Tag.of("type", "zk.connection")),
-new AtomicLong(0));
-newTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.NEW.name().toLowerCase())),
-new AtomicLong(0));
-submittedTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.SUBMITTED.name().toLowerCase())),
-new AtomicLong(0));
-inProgressTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.IN_PROGRESS.name().toLowerCase())),
-new AtomicLong(0));
-failedInProgressTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())),
-new AtomicLong(0));
-failedTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.FAILED.name().toLowerCase())),
-new AtomicLong(0));
-successfulTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.SUCCESSFUL.name().toLowerCase())),
-new AtomicLong(0));
-unknownTxGauge = registry.gauge(METRICS_FATE_TX,
-List.of(Tag.of("state",
ReadOnlyTStore.TStatus.UNKNOWN.name().toLowerCase())),
-new AtomicLong(0));
++registry.gauge(METRICS_FATE_OPS, totalCurrentOpsGauge);
+registry.gauge(METRICS_FATE_OPS_ACTIVITY, totalOpsGauge);
+registry.gauge(METRICS_FATE_ERRORS, List.of(Tag.of("type",
"zk.connection")), fateErrorsGauge);
+registry.gauge(METRICS_FATE_TX,
+List.of(Tag.of("state",
ReadOnlyFateStore.TStatus.NEW.name().toLowerCase())), newTxGauge);
+registry.gauge(METRICS_FATE_TX,
+List.of(Tag.of("state",
ReadOnlyFateStore.TStatus.SUBMITTED.name().toLowerCase())),
+submittedTxGauge);
+registry.gauge(METRICS_FATE_TX,
+List.of(Tag.of("state",
ReadOnlyFateStore.TStatus.IN_PROGRESS.name().toLowerCase())),
+inProgressTxGauge);
+registry.gauge(METRICS_FATE_TX,
+List.of(Tag.of("state"
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8c485e362d3eac62d466ad4c56d3ebf606bdb441 Merge: 5aca4f67bc 3cac724f00 Author: Ed Coleman AuthorDate: Fri May 10 21:19:47 2024 + Merge remote-tracking branch 'upstream/2.1' .../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 7 --- .../java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-)
(accumulo) branch main updated (5aca4f67bc -> 8c485e362d)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5aca4f67bc simplifies handling of time in unload handler (#4548) add 3cac724f00 Renames a fate metric (#4543) new 8c485e362d Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 7 --- .../java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-)
(accumulo) branch 2.1 updated: Renames a fate metric (#4543)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 3cac724f00 Renames a fate metric (#4543)
3cac724f00 is described below
commit 3cac724f0030a600e9195fc87400321e2298b36c
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Fri May 10 17:18:17 2024 -0400
Renames a fate metric (#4543)
Renamed accumulo.fate.ops.in.progress to accumulo.fate.ops and updated
MetricsProducer with name change and description.
---
.../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 7 ---
.../java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 2da6303b5d..ddc9278b43 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@ -54,9 +54,10 @@ import io.micrometer.core.instrument.MeterRegistry;
*
* currentFateOps
* Gauge
- * {@value #METRICS_FATE_TOTAL_IN_PROGRESS}
+ * {@value #METRICS_FATE_OPS}
* Gauge
- *
+ * Was previously named "accumulo.fate.ops.in.progress". Changed to better
reflect what the
+ * gauge is actually tracking which is all the current fate ops in any
state.
*
*
* FateTxOpType_{name}
@@ -585,7 +586,7 @@ public interface MetricsProducer {
String METRICS_FATE_PREFIX = "accumulo.fate.";
String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX +
"ops.in.progress.by.type";
- String METRICS_FATE_TOTAL_IN_PROGRESS = METRICS_FATE_PREFIX +
"ops.in.progress";
+ String METRICS_FATE_OPS = METRICS_FATE_PREFIX + "ops";
String METRICS_FATE_OPS_ACTIVITY = METRICS_FATE_PREFIX + "ops.activity";
String METRICS_FATE_ERRORS = METRICS_FATE_PREFIX + "errors";
String METRICS_FATE_TX = METRICS_FATE_PREFIX + "tx";
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
index bed2e53411..2f66aef256 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
@@ -128,7 +128,7 @@ public class FateMetrics implements MetricsProducer {
@Override
public void registerMetrics(final MeterRegistry registry) {
-totalCurrentOpsGauge = registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS, new
AtomicLong(0));
+totalCurrentOpsGauge = registry.gauge(METRICS_FATE_OPS, new AtomicLong(0));
totalOpsGauge = registry.gauge(METRICS_FATE_OPS_ACTIVITY, new
AtomicLong(0));
fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS,
List.of(Tag.of("type", "zk.connection")),
new AtomicLong(0));
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4542108269fb8da4a523f1e5513974bc45ee13bd
Merge: b32a32cb58 f4c1f24658
Author: Ed Coleman
AuthorDate: Wed May 8 18:01:13 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../accumulo/core/metrics/MetricsProducer.java | 9
.../apache/accumulo/tserver/BlockCacheMetrics.java | 61 ++
.../org/apache/accumulo/tserver/ScanServer.java| 18 +++
.../org/apache/accumulo/tserver/TabletServer.java | 5 +-
4 files changed, 82 insertions(+), 11 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 7f325c39fa,0590fb8aa2..8aa8f65f4b
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -166,7 -181,9 +166,8 @@@ public class TabletServer extends Abstr
TabletServerUpdateMetrics updateMetrics;
TabletServerScanMetrics scanMetrics;
TabletServerMinCMetrics mincMetrics;
- CompactionExecutorsMetrics ceMetrics;
PausedCompactionMetrics pausedMetrics;
+ BlockCacheMetrics blockCacheMetrics;
@Override
public TabletServerScanMetrics getScanMetrics() {
@@@ -582,14 -712,21 +583,16 @@@
updateMetrics = new TabletServerUpdateMetrics();
scanMetrics = new TabletServerScanMetrics();
mincMetrics = new TabletServerMinCMetrics();
-ceMetrics = new CompactionExecutorsMetrics();
pausedMetrics = new PausedCompactionMetrics();
+ blockCacheMetrics = new
BlockCacheMetrics(this.resourceManager.getIndexCache(),
+ this.resourceManager.getDataCache(),
this.resourceManager.getSummaryCache());
metricsInfo.addMetricsProducers(this, metrics, updateMetrics,
scanMetrics, mincMetrics,
- pausedMetrics);
-ceMetrics, pausedMetrics, blockCacheMetrics);
++pausedMetrics, blockCacheMetrics);
metricsInfo.init();
-this.compactionManager = new CompactionManager(() -> Iterators
-.transform(onlineTablets.snapshot().values().iterator(),
Tablet::asCompactable),
-getContext(), ceMetrics);
-compactionManager.start();
-
announceExistence();
+getContext().setServiceLock(tabletServerLock);
try {
walMarker.initWalMarker(getTabletSession());
(accumulo) branch elasticity updated (b32a32cb58 -> 4542108269)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from b32a32cb58 Changed default value of TSERV_PORTSEARCH from false to true (#4514) add 7f9d3f3efe Emit block cache metrics (#4518) add 33e2d055af Merge remote-tracking branch 'upstream/2.1' add 86c81e42d2 Removed duplicate call to Ample.deleteScanServerFileReferences in ScanServer (#4512) add f4c1f24658 Merge branch '2.1' new 4542108269 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metrics/MetricsProducer.java | 9 .../apache/accumulo/tserver/BlockCacheMetrics.java | 61 ++ .../org/apache/accumulo/tserver/ScanServer.java| 18 +++ .../org/apache/accumulo/tserver/TabletServer.java | 5 +- 4 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 server/tserver/src/main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java
(accumulo) branch elasticity updated (0d40533307 -> c6f7c91d82)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 0d40533307 Merge remote-tracking branch 'upstream/main' into elasticity add 5d5339559d Fix ZombieTServer metrics in for LateLastContactIT failure (#4527) add d46ef7cdc6 Merge remote-tracking branch 'upstream/2.1' new c6f7c91d82 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../server/metrics/NoOpDistributionSummary.java| 83 ++ .../accumulo/server/metrics/ThriftMetrics.java | 4 +- .../metrics/NoOpDistributionSummaryTest.java | 19 ++--- .../accumulo/server/metrics/ThriftMetricsTest.java | 62 .../tserver/metrics/TabletServerScanMetrics.java | 5 +- .../tserver/metrics/TabletServerUpdateMetrics.java | 3 +- .../accumulo/test/functional/ZombieTServer.java| 10 +-- 7 files changed, 167 insertions(+), 19 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java copy core/src/test/java/org/apache/accumulo/core/client/lexicoder/DateLexicoderTest.java => server/base/src/test/java/org/apache/accumulo/server/metrics/NoOpDistributionSummaryTest.java (64%) create mode 100644 server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c6f7c91d829c6b30ac28f3f18f705913f611e25c
Merge: 0d40533307 d46ef7cdc6
Author: Ed Coleman
AuthorDate: Mon May 6 21:40:51 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../server/metrics/NoOpDistributionSummary.java| 83 ++
.../accumulo/server/metrics/ThriftMetrics.java | 4 +-
.../metrics/NoOpDistributionSummaryTest.java} | 38 --
.../accumulo/server/metrics/ThriftMetricsTest.java | 62
.../tserver/metrics/TabletServerScanMetrics.java | 5 +-
.../tserver/metrics/TabletServerUpdateMetrics.java | 3 +-
.../accumulo/test/functional/ZombieTServer.java| 10 +--
7 files changed, 170 insertions(+), 35 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index bc10568f7c,dce3d5dda1..38e93abb4c
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -143,6 -140,10 +138,11 @@@ public class ZombieTServer
ServiceLock zlock = new ServiceLock(zoo.getZooKeeper(), zLockPath,
UUID.randomUUID());
+ MetricsInfo metricsInfo = context.getMetricsInfo();
-metricsInfo.addServiceTags("zombie.server", serverPort.address);
++metricsInfo.addServiceTags("zombie.server", serverPort.address,
++Constants.DEFAULT_RESOURCE_GROUP_NAME);
+ metricsInfo.init();
+
LockWatcher lw = new LockWatcher() {
@SuppressFBWarnings(value = "DM_EXIT",
(accumulo) branch main updated (a41366bbd2 -> d46ef7cdc6)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from a41366bbd2 Merge remote-tracking branch 'upstream/2.1' add 5d5339559d Fix ZombieTServer metrics in for LateLastContactIT failure (#4527) new d46ef7cdc6 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../server/metrics/NoOpDistributionSummary.java| 83 ++ .../accumulo/server/metrics/ThriftMetrics.java | 4 +- .../metrics/NoOpDistributionSummaryTest.java | 19 ++--- .../accumulo/server/metrics/ThriftMetricsTest.java | 62 .../tserver/metrics/TabletServerScanMetrics.java | 5 +- .../tserver/metrics/TabletServerUpdateMetrics.java | 3 +- .../accumulo/test/functional/ZombieTServer.java| 8 +-- 7 files changed, 166 insertions(+), 18 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java copy core/src/test/java/org/apache/accumulo/core/client/lexicoder/DateLexicoderTest.java => server/base/src/test/java/org/apache/accumulo/server/metrics/NoOpDistributionSummaryTest.java (64%) create mode 100644 server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit d46ef7cdc65165eab0fe4a264e572a3e1ffc62e0
Merge: a41366bbd2 5d5339559d
Author: Ed Coleman
AuthorDate: Mon May 6 21:08:46 2024 +
Merge remote-tracking branch 'upstream/2.1'
.../server/metrics/NoOpDistributionSummary.java| 83 ++
.../accumulo/server/metrics/ThriftMetrics.java | 4 +-
.../metrics/NoOpDistributionSummaryTest.java} | 38 --
.../accumulo/server/metrics/ThriftMetricsTest.java | 62
.../tserver/metrics/TabletServerScanMetrics.java | 5 +-
.../tserver/metrics/TabletServerUpdateMetrics.java | 3 +-
.../accumulo/test/functional/ZombieTServer.java| 8 +--
7 files changed, 169 insertions(+), 34 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index d7562f78c1,30d8331c93..dce3d5dda1
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -132,11 -129,6 +132,7 @@@ public class ZombieTServer
HostAndPort.fromParts("0.0.0.0", port));
String addressString = serverPort.address.toString();
+
- MetricsInfo metricsInfo = context.getMetricsInfo();
- metricsInfo.addServiceTags("zombie.server", serverPort.address);
- metricsInfo.init();
-
var zLockPath =
ServiceLock.path(context.getZooKeeperRoot() + Constants.ZTSERVERS +
"/" + addressString);
ZooReaderWriter zoo = context.getZooReaderWriter();
(accumulo) branch 2.1 updated (42e218e903 -> 5d5339559d)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 42e218e903 logs hints used to select a scan server (#4520) add 5d5339559d Fix ZombieTServer metrics in for LateLastContactIT failure (#4527) No new revisions were added by this update. Summary of changes: .../server/metrics/NoOpDistributionSummary.java| 83 ++ .../accumulo/server/metrics/ThriftMetrics.java | 4 +- .../metrics/NoOpDistributionSummaryTest.java | 19 ++--- .../accumulo/server/metrics/ThriftMetricsTest.java | 62 .../tserver/metrics/TabletServerScanMetrics.java | 5 +- .../tserver/metrics/TabletServerUpdateMetrics.java | 3 +- .../accumulo/test/functional/ZombieTServer.java| 5 ++ 7 files changed, 167 insertions(+), 14 deletions(-) create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metrics/NoOpDistributionSummary.java copy core/src/test/java/org/apache/accumulo/core/client/lexicoder/DateLexicoderTest.java => server/base/src/test/java/org/apache/accumulo/server/metrics/NoOpDistributionSummaryTest.java (64%) create mode 100644 server/base/src/test/java/org/apache/accumulo/server/metrics/ThriftMetricsTest.java
(accumulo) branch elasticity updated (82b80d0d62 -> 0d40533307)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 82b80d0d62 Merge remote-tracking branch 'upstream/main' into elasticity add 42e218e903 logs hints used to select a scan server (#4520) add a41366bbd2 Merge remote-tracking branch 'upstream/2.1' new 0d40533307 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/clientImpl/TabletServerBatchReaderIterator.java | 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 0d40533307ab421960fbcc950892a72e72437031
Merge: 82b80d0d62 a41366bbd2
Author: Ed Coleman
AuthorDate: Mon May 6 20:16:17 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../core/clientImpl/TabletServerBatchReaderIterator.java | 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
index b95f256786,040a765b1f..3828b2c799
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
@@@ -716,40 -650,26 +716,41 @@@ public class TabletServerBatchReaderIte
if (serverToUse == null) {
// no scan server was given so use the tablet server
serverToUse = extentToTserverMap.get(extent);
-log.trace("For tablet {} using hints {} scan server selector chose
tablet_server", tabletId,
-options.executionHints);
+if (serverToUse != null) {
- log.trace("For tablet {} scan server selector chose tablet_server
{}", tabletId,
- serverToUse);
++ log.trace("For tablet {} using hints {} scan server selector chose
tablet_server",
++ tabletId, options.executionHints);
+} else {
+ log.trace(
- "For tablet {} scan server selector chose tablet_server, but
tablet is not hosted",
- tabletId);
++ "For tablet {} using hints {} scan server selector chose
tablet_server, but tablet is not hosted",
++ tabletId, options.executionHints);
+}
} else {
- log.trace("For tablet {} scan server selector chose scan_server:{}",
tabletId, serverToUse);
+ log.trace("For tablet {} using hints {} scan server selector chose
scan_server:{}",
+ tabletId, options.executionHints, serverToUse);
}
- var rangeMap = binnedRanges2.computeIfAbsent(serverToUse, k -> new
HashMap<>());
- List ranges = extentToRangesMap.get(extent);
- rangeMap.put(extent, ranges);
+ if (serverToUse != null) {
+var rangeMap = binnedRanges.computeIfAbsent(serverToUse, k -> new
HashMap<>());
+List extentRanges = extentToRangesMap.get(extent);
+rangeMap.put(extent, extentRanges);
+
+var server = serverToUse;
+reporters.computeIfAbsent(serverToUse, k ->
scanAttempts.createReporter(server, tabletId));
+ } else {
+failures.addAll(extentToRangesMap.get(extent));
+ }
+}
- var server = serverToUse;
- reporters.computeIfAbsent(serverToUse, k ->
scanAttempts.createReporter(server, tabletId));
+if (!failures.isEmpty()) {
+ // if there are failures at this point its because tablets are not
hosted, so lets attempt to
+ // get them hosted
+ clientTabletCache.findTablets(context, ranges, (cachedTablet, range) ->
{},
+ LocationNeed.REQUIRED);
+ return new ScanServerData(failures);
}
-ScanServerData ssd = new ScanServerData();
+ScanServerData ssd = new ScanServerData(actions, reporters);
-ssd.binnedRanges = binnedRanges2;
-ssd.actions = actions;
-ssd.reporters = reporters;
log.trace("Scan server selector chose delay:{} busyTimeout:{}",
actions.getDelay(),
actions.getBusyTimeout());
return ssd;
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a41366bbd2690c05314b37f360ace26b16bb1aa6
Merge: 7f58fc03b2 42e218e903
Author: Ed Coleman
AuthorDate: Mon May 6 19:53:36 2024 +
Merge remote-tracking branch 'upstream/2.1'
.../accumulo/core/clientImpl/TabletServerBatchReaderIterator.java | 6 --
.../java/org/apache/accumulo/core/clientImpl/ThriftScanner.java| 7 ---
2 files changed, 8 insertions(+), 5 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
index 87726fd639,604aece29d..f36944daaf
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
@@@ -567,13 -567,15 +567,14 @@@ public class ThriftScanner
delay = actions.getDelay();
scanState.busyTimeout = actions.getBusyTimeout();
log.trace(
- "For tablet {} scan server selector chose scan_server:{}
delay:{} busyTimeout:{}",
- loc.getExtent(), scanServer, delay, scanState.busyTimeout);
+ "For tablet {} using hints {} scan server selector chose
scan_server:{} delay:{} busyTimeout:{}",
- loc.tablet_extent, scanState.executionHints, scanServer, delay,
- scanState.busyTimeout);
++ loc.getExtent(), scanState.executionHints, scanServer, delay,
scanState.busyTimeout);
} else {
newLoc = loc;
delay = actions.getDelay();
scanState.busyTimeout = Duration.ZERO;
- log.trace("For tablet {} scan server selector chose tablet_server",
loc.getExtent());
+ log.trace("For tablet {} using hints {} scan server selector chose
tablet_server",
- loc.tablet_extent, scanState.executionHints);
++ loc.getExtent(), scanState.executionHints);
}
if (!delay.isZero()) {
(accumulo) branch main updated (7f58fc03b2 -> a41366bbd2)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7f58fc03b2 Merge remote-tracking branch 'upstream/2.1' add 42e218e903 logs hints used to select a scan server (#4520) new a41366bbd2 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/clientImpl/TabletServerBatchReaderIterator.java | 6 -- .../java/org/apache/accumulo/core/clientImpl/ThriftScanner.java| 7 --- 2 files changed, 8 insertions(+), 5 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 82b80d0d62bd4290ac85a857b566d6cc0f864367
Merge: a198c1fbc4 7f58fc03b2
Author: Ed Coleman
AuthorDate: Mon May 6 17:29:28 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
includes multiple PRs:
- 4526 bouncy castle version bump
- 4522 update scan server metrics for file reservations
- 4520 log hints to select a scan server
.../accumulo/core/metrics/MetricsProducer.java | 108 +++--
pom.xml| 2 +-
.../accumulo/manager/TabletGroupWatcher.java | 18 ++--
.../java/org/apache/accumulo/monitor/Monitor.java | 6 +-
.../org/apache/accumulo/tserver/ScanServer.java| 53 --
.../apache/accumulo/tserver/ScanServerMetrics.java | 59 +++
.../accumulo/tserver/ThriftScanClientHandler.java | 4 +-
.../tserver/metrics/TabletServerScanMetrics.java | 20 ++--
.../apache/accumulo/tserver/ScanServerTest.java| 9 ++
.../apache/accumulo/test/metrics/MetricsIT.java| 8 +-
10 files changed, 204 insertions(+), 83 deletions(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 18682789c5,c3a77f2297..060411fcbf
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -972,43 -1449,20 +972,49 @@@ abstract class TabletGroupWatcher exten
}
tLists.assignments.addAll(tLists.assigned);
for (Assignment a : tLists.assignments) {
- TServerConnection client = manager.tserverSet.getConnection(a.server);
- if (client != null) {
- client.assignTablet(manager.managerLock, a.tablet);
- } else {
- Manager.log.warn("Could not connect to server {}", a.server);
+ try {
+ TServerConnection client = manager.tserverSet.getConnection(a.server);
+ if (client != null) {
+ client.assignTablet(manager.managerLock, a.tablet);
+ manager.assignedTablet(a.tablet);
+ } else {
+ Manager.log.warn("Could not connect to server {} for assignment of
{}", a.server,
+ a.tablet);
+ }
+ } catch (TException tException) {
+ Manager.log.warn("Could not connect to server {} for assignment of
{}", a.server, a.tablet,
+ tException);
}
- manager.assignedTablet(a.tablet);
}
+
+replaceVolumes(tLists.volumeReplacements);
+ }
+
+ private void replaceVolumes(List
volumeReplacementsList) {
+try (var tabletsMutator =
manager.getContext().getAmple().conditionallyMutateTablets()) {
+ for (VolumeUtil.VolumeReplacements vr : volumeReplacementsList) {
+var tabletMutator =
+
tabletsMutator.mutateTablet(vr.tabletMeta.getExtent()).requireAbsentOperation()
+.requireAbsentLocation().requireSame(vr.tabletMeta, FILES,
LOGS);
+vr.logsToRemove.forEach(tabletMutator::deleteWal);
+vr.logsToAdd.forEach(tabletMutator::putWal);
+
+vr.filesToRemove.forEach(tabletMutator::deleteFile);
+vr.filesToAdd.forEach(tabletMutator::putFile);
+
+tabletMutator.submit(
+tm -> tm.getLogs().containsAll(vr.logsToAdd) &&
tm.getFiles().containsAll(vr.filesToAdd
+
.keySet().stream().map(ReferencedTabletFile::insert).collect(Collectors.toSet(;
+ }
+
+ tabletsMutator.process().forEach((extent, result) -> {
+if (result.getStatus() == Ample.ConditionalResult.Status.REJECTED) {
+ // log that failure happened, should try again later
+ LOG.debug("Failed to update volumes for tablet {}", extent);
+}
+ });
+}
+
}
private static void markDeadServerLogsAsClosed(WalStateManager mgr,
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 51be4e13d8,10ccf19c6c..1fb3a66108
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -486,11 -489,11 +486,11 @@@ public class Monitor extends AbstractSe
log.error("Unable to get hostname", e);
}
}
- log.debug("Using {} to advertise monitor location in ZooKeeper",
advertiseHost);
+ HostAndPort monitorHostAndPort = HostAndPort.fromParts(advertiseHost,
livePort);
+ log.debug("Using {} to advertise monitor location in ZooKeeper",
monitorHostAndPort);
MetricsInfo metricsInfo = getContext().getMetricsInfo();
- metricsInfo.addServiceTags(getApplicationName(),
HostAndPort.fromParts(advertiseHost, livePort),
-
(accumulo) branch elasticity updated (a198c1fbc4 -> 82b80d0d62)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from a198c1fbc4 Merge remote-tracking branch 'upstream/main' into elasticity add 2ce89e0ec4 uses specific logger for compaction coordinator status update (#4489) add acaa6dc46c lowers logging level for first seen dead compaction (#4490) add 61ca7d8070 Handles RPC errors when requesting tablet unload (#4497) add 8da73ce467 Merge branch '2.1' add c488f788ad scan server metrics from #4458 based on #4459 (#4461) add 96b86a5f62 Update scan server metrics for file reservations (#4522) add 46ed92329c Merge remote-tracking branch 'upstream/2.1' add 1584ea83a1 Add the host:port information to the monitor service lock (#4517) add 279cd598bc Merge remote-tracking branch 'upstream/2.1' add fbcd746470 bump bouncy castle version from 1.77 to 1.78.1 (#4526) add 7f58fc03b2 Merge remote-tracking branch 'upstream/2.1' new 82b80d0d62 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metrics/MetricsProducer.java | 108 +++-- pom.xml| 2 +- .../accumulo/manager/TabletGroupWatcher.java | 18 ++-- .../java/org/apache/accumulo/monitor/Monitor.java | 6 +- .../org/apache/accumulo/tserver/ScanServer.java| 53 -- .../apache/accumulo/tserver/ScanServerMetrics.java | 59 +++ .../accumulo/tserver/ThriftScanClientHandler.java | 4 +- .../tserver/metrics/TabletServerScanMetrics.java | 20 ++-- .../apache/accumulo/tserver/ScanServerTest.java| 9 ++ .../apache/accumulo/test/metrics/MetricsIT.java| 8 +- 10 files changed, 204 insertions(+), 83 deletions(-) create mode 100644 server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 7f58fc03b2195d23b9fa155f843d49f1e1e4648c Merge: 279cd598bc fbcd746470 Author: Ed Coleman AuthorDate: Fri May 3 22:59:42 2024 + Merge remote-tracking branch 'upstream/2.1' pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch 2.1 updated: bump bouncy castle version from 1.77 to 1.78.1 (#4526)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/2.1 by this push: new fbcd746470 bump bouncy castle version from 1.77 to 1.78.1 (#4526) fbcd746470 is described below commit fbcd746470c37739eab7b200f61d12dd26fad2d7 Author: EdColeman AuthorDate: Fri May 3 18:58:35 2024 -0400 bump bouncy castle version from 1.77 to 1.78.1 (#4526) triggered by dependabot alert Fixes PR #4525 (starting at 2.1) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbffd60171..463fd2944f 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ -Xmx1G 1.1.1 -1.77 +1.78.1 5.5.0 2.24.1 3.3.6
(accumulo) branch main updated (279cd598bc -> 7f58fc03b2)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 279cd598bc Merge remote-tracking branch 'upstream/2.1' add fbcd746470 bump bouncy castle version from 1.77 to 1.78.1 (#4526) new 7f58fc03b2 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch main updated (46ed92329c -> 279cd598bc)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 46ed92329c Merge remote-tracking branch 'upstream/2.1' add 1584ea83a1 Add the host:port information to the monitor service lock (#4517) new 279cd598bc Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 279cd598bc61b4796793cc5a44f40f13a9412e24
Merge: 46ed92329c 1584ea83a1
Author: Ed Coleman
AuthorDate: Fri May 3 21:44:01 2024 +
Merge remote-tracking branch 'upstream/2.1'
This is basically a no-op imerge for main. The service lock data
is written and contains the hostname. Because of initialization
order, the port is not included, but it need it can be done with
a follow-on.
.../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 6c07e9bb32,39fe11d91b..10ccf19c6c
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -489,12 -484,16 +489,12 @@@ public class Monitor extends AbstractSe
log.error("Unable to get hostname", e);
}
}
- log.debug("Using {} to advertise monitor location in ZooKeeper",
advertiseHost);
+ HostAndPort monitorHostAndPort = HostAndPort.fromParts(advertiseHost,
livePort);
+ log.debug("Using {} to advertise monitor location in ZooKeeper",
monitorHostAndPort);
-try {
-
monitorLock.replaceLockData(monitorHostAndPort.toString().getBytes(UTF_8));
-} catch (KeeperException | InterruptedException e) {
- throw new IllegalStateException("Exception updating monitor lock with
host and port", e);
-}
MetricsInfo metricsInfo = getContext().getMetricsInfo();
- metricsInfo.addServiceTags(getApplicationName(),
- HostAndPort.fromParts(advertiseHost, livePort));
+ metricsInfo.addServiceTags(getApplicationName(), monitorHostAndPort);
+metricsInfo.addMetricsProducers(this);
metricsInfo.init();
try {
(accumulo) branch 2.1 updated (96b86a5f62 -> 1584ea83a1)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 96b86a5f62 Update scan server metrics for file reservations (#4522) add 1584ea83a1 Add the host:port information to the monitor service lock (#4517) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/monitor/Monitor.java| 11 --- 1 file changed, 8 insertions(+), 3 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 46ed92329cf83911c5b43f348658941e0979f3f9 Merge: 8da73ce467 96b86a5f62 Author: Ed Coleman AuthorDate: Fri May 3 16:57:08 2024 + Merge remote-tracking branch 'upstream/2.1' includes: - 96b86a5f62 - update for additional scan server metrics - c488f788ad - adds scan server metrics .../accumulo/core/metrics/MetricsProducer.java | 108 +++-- .../org/apache/accumulo/tserver/ScanServer.java| 55 +-- .../apache/accumulo/tserver/ScanServerMetrics.java | 59 +++ .../accumulo/tserver/ThriftScanClientHandler.java | 4 +- .../tserver/metrics/TabletServerScanMetrics.java | 24 +++-- .../apache/accumulo/tserver/ScanServerTest.java| 9 ++ .../apache/accumulo/test/metrics/MetricsIT.java| 11 ++- 7 files changed, 192 insertions(+), 78 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index 5e43b2a938,1bb2a1c10e..2fdf9172ab --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@@ -599,9 -613,14 +600,9 @@@ public interface MetricsProducer String METRICS_MINC_PREFIX = "accumulo.tserver.compactions.minc."; String METRICS_MINC_QUEUED = METRICS_MINC_PREFIX + "queued"; String METRICS_MINC_RUNNING = METRICS_MINC_PREFIX + "running"; - - String METRICS_REPLICATION_PREFIX = "accumulo.replication."; - String METRICS_REPLICATION_QUEUE = METRICS_REPLICATION_PREFIX + "queue"; - String METRICS_REPLICATION_PENDING_FILES = METRICS_REPLICATION_PREFIX + "files.pending"; - String METRICS_REPLICATION_PEERS = METRICS_REPLICATION_PREFIX + "peers"; - String METRICS_REPLICATION_THREADS = METRICS_REPLICATION_PREFIX + "threads"; + String METRICS_MINC_PAUSED = METRICS_MINC_PREFIX + "paused"; - String METRICS_SCAN_PREFIX = "accumulo.tserver.scans."; + String METRICS_SCAN_PREFIX = "accumulo.scan."; String METRICS_SCAN_TIMES = METRICS_SCAN_PREFIX + "times"; String METRICS_SCAN_OPEN_FILES = METRICS_SCAN_PREFIX + "files.open"; String METRICS_SCAN_RESULTS = METRICS_SCAN_PREFIX + "result"; @@@ -609,10 -628,15 +610,17 @@@ String METRICS_SCAN_START = METRICS_SCAN_PREFIX + "start"; String METRICS_SCAN_CONTINUE = METRICS_SCAN_PREFIX + "continue"; String METRICS_SCAN_CLOSE = METRICS_SCAN_PREFIX + "close"; - String METRICS_SCAN_BUSY_TIMEOUT = METRICS_SCAN_PREFIX + "busy.timeout"; + String METRICS_SCAN_BUSY_TIMEOUT_COUNTER = METRICS_SCAN_PREFIX + "busy.timeout.count"; + String METRICS_SCAN_RESERVATION_TIMER = METRICS_SCAN_PREFIX + "reservation.timer"; + String METRICS_SCAN_QUERIES = METRICS_SCAN_PREFIX + "queries"; + String METRICS_SCAN_QUERY_SCAN_RESULTS = METRICS_SCAN_PREFIX + "query.results"; + String METRICS_SCAN_QUERY_SCAN_RESULTS_BYTES = METRICS_SCAN_PREFIX + "query.results.bytes"; + String METRICS_SCAN_SCANNED_ENTRIES = METRICS_SCAN_PREFIX + "query.scanned.entries"; + String METRICS_SCAN_PAUSED_FOR_MEM = METRICS_SCAN_PREFIX + "paused.for.memory"; + String METRICS_SCAN_RETURN_FOR_MEM = METRICS_SCAN_PREFIX + "return.early.for.memory"; + String METRICS_SCAN_TABLET_METADATA_CACHE = METRICS_SCAN_PREFIX + "tablet.metadata.cache"; + String METRICS_TSERVER_PREFIX = "accumulo.tserver."; String METRICS_TSERVER_ENTRIES = METRICS_TSERVER_PREFIX + "entries"; String METRICS_TSERVER_MEM_ENTRIES = METRICS_TSERVER_PREFIX + "entries.mem"; diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java index ca4b28d06e,1e237e80e8..44ab680a98 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java @@@ -76,17 -70,20 +76,18 @@@ import org.apache.accumulo.core.metadat import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.securityImpl.thrift.TCredentials; -import org.apache.accumulo.core.spi.scan.ScanServerSelector; -import org.apache.accumulo.core.tabletserver.thrift.ActiveScan; +import org.apache.accumulo.core.tabletscan.thrift.ActiveScan; ++import org.apache.accumulo.core.tabletscan.thrift.ScanServerBusyException; +imp
(accumulo) branch main updated (8da73ce467 -> 46ed92329c)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8da73ce467 Merge branch '2.1' add c488f788ad scan server metrics from #4458 based on #4459 (#4461) add 96b86a5f62 Update scan server metrics for file reservations (#4522) new 46ed92329c Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metrics/MetricsProducer.java | 108 +++-- .../org/apache/accumulo/tserver/ScanServer.java| 55 +-- .../apache/accumulo/tserver/ScanServerMetrics.java | 59 +++ .../accumulo/tserver/ThriftScanClientHandler.java | 4 +- .../tserver/metrics/TabletServerScanMetrics.java | 24 +++-- .../apache/accumulo/tserver/ScanServerTest.java| 9 ++ .../apache/accumulo/test/metrics/MetricsIT.java| 11 ++- 7 files changed, 192 insertions(+), 78 deletions(-) create mode 100644 server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServerMetrics.java
(accumulo) branch 2.1 updated (c488f788ad -> 96b86a5f62)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from c488f788ad scan server metrics from #4458 based on #4459 (#4461) add 96b86a5f62 Update scan server metrics for file reservations (#4522) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/tserver/ScanServer.java | 7 --- .../src/test/java/org/apache/accumulo/tserver/ScanServerTest.java | 5 + 2 files changed, 9 insertions(+), 3 deletions(-)
(accumulo) branch 2.1 updated: scan server metrics from #4458 based on #4459 (#4461)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new c488f788ad scan server metrics from #4458 based on #4459 (#4461)
c488f788ad is described below
commit c488f788ad3add4992c30556d7b5bc941d6f0a39
Author: EdColeman
AuthorDate: Thu May 2 21:52:23 2024 -0400
scan server metrics from #4458 based on #4459 (#4461)
* Add scan server metrics
* scan metrics renaming to remove specific service in favor of tags
(removes tserver from scan metric names)
* enable stats on scan sever tablet metadata cache
* add resource.group to sserver common tags instead of on individual metrics
-
Co-authored-by: Keith Turner
---
.../accumulo/core/metrics/MetricsProducer.java | 102 -
.../org/apache/accumulo/tserver/ScanServer.java| 51 ++-
.../apache/accumulo/tserver/ScanServerMetrics.java | 59
.../accumulo/tserver/ThriftScanClientHandler.java | 4 +-
.../tserver/metrics/TabletServerScanMetrics.java | 20 ++--
.../apache/accumulo/tserver/ScanServerTest.java| 4 +
.../apache/accumulo/test/metrics/MetricsIT.java| 7 +-
7 files changed, 187 insertions(+), 60 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 568b295f58..1bb2a1c10e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@ -314,37 +314,6 @@ import io.micrometer.core.instrument.MeterRegistry;
*
*
*
- * queries
- * Gauge
- * {@value #METRICS_TSERVER_QUERIES}
- * Gauge
- *
- *
- *
- * scannedRate
- * Gauge
- * {@value #METRICS_TSERVER_SCANNED_ENTRIES}
- * Gauge
- * Prior to 2.1.0 this metric was reported as a rate, it is now the count
and the rate can be
- * derived
- *
- *
- * queryRate
- * Gauge
- * {@value #METRICS_TSERVER_SCAN_RESULTS}
- * Gauge
- * Prior to 2.1.0 this metric was reported as a rate, it is now the count
and the rate can be
- * derived
- *
- *
- * queryByteRate
- * Gauge
- * {@value #METRICS_TSERVER_SCAN_RESULTS_BYTES}
- * Gauge
- * Prior to 2.1.0 this metric was reported as a rate, it is now the count
and the rate can be
- * derived
- *
- *
* ingestRate
* Gauge
* {@value #METRICS_TSERVER_INGEST_MUTATIONS}
@@ -367,6 +336,28 @@ import io.micrometer.core.instrument.MeterRegistry;
* Gauge
*
*
+ *
+ *
+ * N/A
+ * N/A
+ * {@value #METRICS_SCAN_RESERVATION_TIMER}
+ * Timer
+ * Time to reserve a tablets files for scan
+ *
+ *
+ * N/A
+ * N/A
+ * {@value #METRICS_SCAN_BUSY_TIMEOUT_COUNTER}
+ * Counter
+ * Count of the scans where a busy timeout happened
+ *
+ *
+ * N/A
+ * N/A
+ * {@value #METRICS_SCAN_TABLET_METADATA_CACHE}
+ * Cache
+ * scan server tablet cache metrics
+ *
*
*
* scan
@@ -418,12 +409,36 @@ import io.micrometer.core.instrument.MeterRegistry;
*
*
*
- * N/A
- * N/A
- * {@value #METRICS_SCAN_BUSY_TIMEOUT}
- * Counter
+ * queries
+ * Gauge
+ * {@value #METRICS_SCAN_QUERIES}
+ * Gauge
*
*
+ *
+ * scannedRate
+ * Gauge
+ * {@value #METRICS_SCAN_SCANNED_ENTRIES}
+ * Gauge
+ * Prior to 2.1.0 this metric was reported as a rate, it is now the count
and the rate can be
+ * derived
+ *
+ *
+ * queryRate
+ * Gauge
+ * {@value #METRICS_SCAN_QUERY_SCAN_RESULTS}
+ * Gauge
+ * Prior to 2.1.0 this metric was reported as a rate, it is now the count
and the rate can be
+ * derived
+ *
+ *
+ * queryByteRate
+ * Gauge
+ * {@value #METRICS_SCAN_QUERY_SCAN_RESULTS_BYTES}
+ * Gauge
+ * Prior to 2.1.0 this metric was reported as a rate, it is now the count
and the rate can be
+ * derived
+ *
*
*
* {i|e}_{compactionServiceName}_{executor_name}_queued
@@ -605,7 +620,7 @@ public interface MetricsProducer {
String METRICS_REPLICATION_PEERS = METRICS_REPLICATION_PREFIX + "peers";
String METRICS_REPLICATION_THREADS = METRICS_REPLICATION_PREFIX + "threads";
- String METRICS_SCAN_PREFIX = "accumulo.tserver.scans.";
+ String METRICS_SCAN_PREFIX = "accumulo.scan.";
String METRICS_SCAN_TIMES = METRICS_SCAN_PREFIX + "times";
String METRICS_SCAN_OPEN_FILES = METRICS_SCAN_PREFIX + "files.open";
String METRICS_SCAN_RESULTS = METRICS_SCAN_PREFIX + "result";
@@ -613,7 +628,14 @@ public interface MetricsProducer {
String METRICS_SCAN_START = METRICS_SCAN_PREFIX + "start";
String METRICS_SCAN_CONTINUE = METRICS_SCAN_PREFIX + "continue";
String METRICS_SCAN_CLOSE = METRICS_SCAN_PREFIX + "close";
- String METRICS_SCAN_BUSY_TIMEOUT = METRICS_SCAN_PREFIX + "busy.timeout";
+ String METRICS_SCAN_BUSY_TIMEOUT_COUNTER
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a198c1fbc45dbe6203bc662cf667010eb8f8d50d Merge: 76fffdedf3 09adb11491 Author: Ed Coleman AuthorDate: Mon Apr 29 19:32:29 2024 + Merge remote-tracking branch 'upstream/main' into elasticity .../accumulo/test/compaction/ExternalCompactionMetricsIT.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) branch elasticity updated (76fffdedf3 -> a198c1fbc4)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 76fffdedf3 Modified Tablet Mutator implementations to always update srv:lock (#4421) add 09adb11491 enable metrics log to file for ExternalCompactionMetricsIT (#4502) new a198c1fbc4 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/test/compaction/ExternalCompactionMetricsIT.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) branch main updated: enable metrics log to file for ExternalCompactionMetricsIT (#4502)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 09adb11491 enable metrics log to file for ExternalCompactionMetricsIT
(#4502)
09adb11491 is described below
commit 09adb11491afc2e4343efabb0911f047de8846cb
Author: EdColeman
AuthorDate: Mon Apr 29 15:32:12 2024 -0400
enable metrics log to file for ExternalCompactionMetricsIT (#4502)
---
.../accumulo/test/compaction/ExternalCompactionMetricsIT.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
index 459fc8b2d4..ff54253869 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java
@@ -41,6 +41,7 @@ import
org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.metrics.MetricsProducer;
+import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
@@ -70,8 +71,11 @@ public class ExternalCompactionMetricsIT extends
SharedMiniClusterBase {
// Tell the server processes to use a StatsDMeterRegistry that will be
configured
// to push all metrics to the sink we started.
cfg.setProperty(Property.GENERAL_MICROMETER_ENABLED, "true");
- cfg.setProperty(Property.GENERAL_MICROMETER_FACTORY,
- TestStatsDRegistryFactory.class.getName());
+ cfg.setProperty("general.custom.metrics.opts.logging.step", "5s");
+ String clazzList = LoggingMeterRegistryFactory.class.getName() + ","
+ + TestStatsDRegistryFactory.class.getName();
+ cfg.setProperty(Property.GENERAL_MICROMETER_FACTORY, clazzList);
+
Map sysProps =
Map.of(TestStatsDRegistryFactory.SERVER_HOST, "127.0.0.1",
TestStatsDRegistryFactory.SERVER_PORT,
Integer.toString(sink.getPort()));
cfg.setSystemProperties(sysProps);
(accumulo) branch elasticity updated (03d3a6d911 -> 13bb871ac5)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 03d3a6d911 Merge remote-tracking branch 'upstream/main' into elasticity add 6c8e175eab address obsolete hadoop 2 metrics references. Removes deprecated properties and a comment (#4503) add 252b46bab3 Merge remote-tracking branch 'upstream/2.1' new 13bb871ac5 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java| 2 -- 1 file changed, 2 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 13bb871ac52613d5ca42127b6c36b3e36e23ccb0 Merge: 03d3a6d911 252b46bab3 Author: Ed Coleman AuthorDate: Mon Apr 29 16:54:34 2024 + Merge remote-tracking branch 'upstream/main' into elasticity .../org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java| 2 -- 1 file changed, 2 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 252b46bab311676ea567df65f7c8577c5921dde3 Merge: e682a2f229 6c8e175eab Author: Ed Coleman AuthorDate: Mon Apr 29 16:37:01 2024 + Merge remote-tracking branch 'upstream/2.1' .../org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java| 2 -- 1 file changed, 2 deletions(-)
(accumulo) branch main updated (e682a2f229 -> 252b46bab3)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from e682a2f229 Fix test failures after metrics merge (#4500) add 6c8e175eab address obsolete hadoop 2 metrics references. Removes deprecated properties and a comment (#4503) new 252b46bab3 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java| 2 -- 1 file changed, 2 deletions(-)
(accumulo) branch 2.1 updated: address obsolete hadoop 2 metrics references. Removes deprecated properties and a comment (#4503)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 6c8e175eab address obsolete hadoop 2 metrics references. Removes
deprecated properties and a comment (#4503)
6c8e175eab is described below
commit 6c8e175eab871428e0340d65bdcedbef0d332c42
Author: EdColeman
AuthorDate: Mon Apr 29 12:24:41 2024 -0400
address obsolete hadoop 2 metrics references. Removes deprecated properties
and a comment (#4503)
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 6 --
.../apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java| 2 --
2 files changed, 8 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 80a9b21b33..fc5a52f239 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -382,9 +382,6 @@ public enum Property {
MANAGER_WALOG_CLOSER_IMPLEMETATION("manager.walog.closer.implementation",
"org.apache.accumulo.server.manager.recovery.HadoopLogCloser",
PropertyType.CLASSNAME,
"A class that implements a mechanism to steal write access to a
write-ahead log.", "1.5.0"),
- @Deprecated
- MANAGER_FATE_METRICS_ENABLED("manager.fate.metrics.enabled", "true",
PropertyType.BOOLEAN,
- "Enable reporting of FATE metrics in JMX (and logging with Hadoop
Metrics2).", "1.9.3"),
MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL("manager.fate.metrics.min.update.interval",
"60s",
PropertyType.TIMEDURATION, "Limit calls from metric sinks to zookeeper
to update interval.",
"1.9.3"),
@@ -924,9 +921,6 @@ public enum Property {
+ " and possibly compacted. Legal values are: compact - which both
flushes and compacts the"
+ " metadata; flush - which flushes only (compactions may be
triggered if required); or none.",
"1.10.0"),
- @Deprecated
- GC_METRICS_ENABLED("gc.metrics.enabled", "true", PropertyType.BOOLEAN,
- "Enable detailed gc metrics reporting with hadoop metrics.", "1.10.0"),
// properties that are specific to the monitor server behavior
MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX,
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
index f508a6871e..599065d7c7 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java
@@ -24,8 +24,6 @@ import org.apache.accumulo.tserver.tablet.Tablet;
/**
* Wrapper around extracting metrics from a TabletServer instance
- *
- * Necessary to support both old custom JMX metrics and Hadoop Metrics2
*/
public class TabletServerMetricsUtil {
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 03d3a6d911f5711e95d5ebc990821bd2a69a85e9
Merge: 07bf1bc6fd e682a2f229
Author: Ed
AuthorDate: Sat Apr 27 16:40:04 2024 -0400
Merge remote-tracking branch 'upstream/main' into elasticity
.../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 4 ++--
.../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 1 +
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java| 2 +-
.../org/apache/accumulo/test/functional/MemoryStarvedScanIT.java | 7 +--
.../java/org/apache/accumulo/test/functional/ZombieTServer.java| 7 +++
5 files changed, 16 insertions(+), 5 deletions(-)
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 0d99ea3f0f,6c07e9bb32..74ec7b89a5
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -488,8 -492,9 +488,9 @@@ public class Monitor extends AbstractSe
log.debug("Using {} to advertise monitor location in ZooKeeper",
advertiseHost);
MetricsInfo metricsInfo = getContext().getMetricsInfo();
-metricsInfo.addServiceTags(getApplicationName(),
-HostAndPort.fromParts(advertiseHost, livePort));
+metricsInfo.addServiceTags(getApplicationName(),
HostAndPort.fromParts(advertiseHost, livePort),
+getResourceGroup());
+ metricsInfo.addMetricsProducers(this);
metricsInfo.init();
try {
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 9947b5e48b,409d1ecada..93f3e58def
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -582,12 -711,18 +582,12 @@@ public class TabletServer extends Abstr
updateMetrics = new TabletServerUpdateMetrics();
scanMetrics = new TabletServerScanMetrics();
mincMetrics = new TabletServerMinCMetrics();
-ceMetrics = new CompactionExecutorsMetrics();
pausedMetrics = new PausedCompactionMetrics();
- metricsInfo.addMetricsProducers(metrics, updateMetrics, scanMetrics,
mincMetrics,
+ metricsInfo.addMetricsProducers(this, metrics, updateMetrics,
scanMetrics, mincMetrics,
-ceMetrics, pausedMetrics);
+pausedMetrics);
metricsInfo.init();
-this.compactionManager = new CompactionManager(() -> Iterators
-.transform(onlineTablets.snapshot().values().iterator(),
Tablet::asCompactable),
-getContext(), ceMetrics);
-compactionManager.start();
-
announceExistence();
try {
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
index 0becd57120,d2cb595b3d..0c9b08e0ab
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
@@@ -48,14 -43,9 +48,15 @@@ import org.apache.accumulo.core.conf.Pr
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.lock.ServiceLock;
+import org.apache.accumulo.core.lock.ServiceLockData;
+import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
import org.apache.accumulo.core.metrics.MetricsProducer;
+import org.apache.accumulo.core.rpc.ThriftUtil;
+import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
+ import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.minicluster.MemoryUnit;
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index ab90bcf870,d7562f78c1..bc10568f7c
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -129,6 -132,11 +130,12 @@@ public class ZombieTServer
HostAndPort.fromParts("0.0.0.0", port));
String addressString = serverPort.address.toString();
+
+ MetricsInfo metricsInfo = context.getMetricsInfo();
-metricsInfo.addServiceTags("zombie.server", serverPort.address);
++metricsInfo.addServiceTags("zombie.server", serverPort.address,
++Constants.DEFAULT_RESOURCE_GROUP_NAME);
+ metricsInfo.init();
+
var zLockPath =
ServiceLock.path(context.getZooKeeperRoot() + Constants.ZTSERVERS +
"/" + addressString);
ZooReaderWriter zoo = context.getZooReaderWriter();
(accumulo) branch elasticity updated (07bf1bc6fd -> 03d3a6d911)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 07bf1bc6fd Merge branch 'main' into elasticity add e682a2f229 Fix test failures after metrics merge (#4500) new 03d3a6d911 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 4 ++-- .../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 1 + .../src/main/java/org/apache/accumulo/tserver/TabletServer.java| 2 +- .../org/apache/accumulo/test/functional/MemoryStarvedScanIT.java | 7 +-- .../java/org/apache/accumulo/test/functional/ZombieTServer.java| 7 +++ 5 files changed, 16 insertions(+), 5 deletions(-)
(accumulo) branch main updated: Fix test failures after metrics merge (#4500)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e682a2f229 Fix test failures after metrics merge (#4500)
e682a2f229 is described below
commit e682a2f2296cf95a32d58250aeef612b61147936
Author: EdColeman
AuthorDate: Sat Apr 27 15:58:24 2024 -0400
Fix test failures after metrics merge (#4500)
---
.../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 4 ++--
.../org/apache/accumulo/coordinator/CompactionCoordinator.java | 1 +
.../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 1 +
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java| 4 ++--
.../org/apache/accumulo/test/functional/MemoryStarvedScanIT.java | 7 +--
.../java/org/apache/accumulo/test/functional/ZombieTServer.java| 6 ++
6 files changed, 17 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index e50d2d9c03..5e43b2a938 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@ -610,8 +610,8 @@ public interface MetricsProducer {
String METRICS_SCAN_CONTINUE = METRICS_SCAN_PREFIX + "continue";
String METRICS_SCAN_CLOSE = METRICS_SCAN_PREFIX + "close";
String METRICS_SCAN_BUSY_TIMEOUT = METRICS_SCAN_PREFIX + "busy.timeout";
- String METRICS_SCAN_PAUSED_FOR_MEM = METRICS_SCAN_PREFIX +
".paused.for.memory";
- String METRICS_SCAN_RETURN_FOR_MEM = METRICS_SCAN_PREFIX +
".return.early.for.memory";
+ String METRICS_SCAN_PAUSED_FOR_MEM = METRICS_SCAN_PREFIX +
"paused.for.memory";
+ String METRICS_SCAN_RETURN_FOR_MEM = METRICS_SCAN_PREFIX +
"return.early.for.memory";
String METRICS_TSERVER_PREFIX = "accumulo.tserver.";
String METRICS_TSERVER_ENTRIES = METRICS_TSERVER_PREFIX + "entries";
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index ec8cdc514c..45570110d6 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -264,6 +264,7 @@ public class CompactionCoordinator extends AbstractServer
MetricsInfo metricsInfo = getContext().getMetricsInfo();
metricsInfo.addServiceTags(getApplicationName(), clientAddress);
+metricsInfo.addMetricsProducers(this);
metricsInfo.init();
// On a re-start of the coordinator it's possible that external
compactions are in-progress.
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index fde338f422..6c07e9bb32 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -494,6 +494,7 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
MetricsInfo metricsInfo = getContext().getMetricsInfo();
metricsInfo.addServiceTags(getApplicationName(),
HostAndPort.fromParts(advertiseHost, livePort));
+metricsInfo.addMetricsProducers(this);
metricsInfo.init();
try {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 89d503ef58..409d1ecada 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -714,8 +714,8 @@ public class TabletServer extends AbstractServer implements
TabletHostingServer
ceMetrics = new CompactionExecutorsMetrics();
pausedMetrics = new PausedCompactionMetrics();
-metricsInfo.addMetricsProducers(metrics, updateMetrics, scanMetrics,
mincMetrics, ceMetrics,
-pausedMetrics);
+metricsInfo.addMetricsProducers(this, metrics, updateMetrics, scanMetrics,
mincMetrics,
+ceMetrics, pausedMetrics);
metricsInfo.init();
this.compactionManager = new CompactionManager(() -> Iterators
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
index 59b9a535b8..d2cb595b3d 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.ja
(accumulo) branch main updated: Fix PausedCompactionMetrics caused by merge issue (#4491)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 65dd34fad4 Fix PausedCompactionMetrics caused by merge issue (#4491) 65dd34fad4 is described below commit 65dd34fad42590197cb242c9cb44a7a382dbccae Author: EdColeman AuthorDate: Thu Apr 25 08:15:18 2024 -0400 Fix PausedCompactionMetrics caused by merge issue (#4491) --- .../src/main/java/org/apache/accumulo/compactor/Compactor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java index 6b0931b3d4..ebd195a22f 100644 --- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java +++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java @@ -135,7 +135,7 @@ public class Compactor extends AbstractServer implements MetricsProducer, Compac private ServiceLock compactorLock; private ServerAddress compactorAddress = null; - private PausedCompactionMetrics pausedMetrics; + private final PausedCompactionMetrics pausedMetrics = new PausedCompactionMetrics(); // Exposed for tests protected volatile boolean shutdown = false;
(accumulo) branch main updated: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6c1e453f53 Merge branch '2.1'
6c1e453f53 is described below
commit 6c1e453f5399dc21bf9889bb78e0ff541ad95878
Author: Ed Coleman
AuthorDate: Wed Apr 24 18:47:12 2024 +
Merge branch '2.1'
- excludes changes to the grep iterator and grep shell command.
---
core/pom.xml | 1 +
.../org/apache/accumulo/core/conf/Property.java| 11 +-
.../core/metrics/MeterRegistryFactory.java | 5 +
.../apache/accumulo/core/metrics/MetricsInfo.java | 114
.../accumulo/core/metrics/MetricsProducer.java | 43 ---
.../apache/accumulo/core/metrics/MetricsUtil.java | 132 -
.../spi/metrics/LoggingMeterRegistryFactory.java | 81 ++
.../core/spi/metrics/MeterRegistryFactory.java | 72 +
.../accumulo/core/util/threads/ThreadPools.java| 38 ++-
.../metrics/LoggingMeterRegistryFactoryTest.java | 52
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 3 +
.../org/apache/accumulo/server/AbstractServer.java | 9 +-
.../org/apache/accumulo/server/ServerContext.java | 13 +
.../server/compaction/PausedCompactionMetrics.java | 5 +-
.../conf/store/impl/PropCacheCaffeineImpl.java | 18 +-
.../server/conf/store/impl/PropStoreMetrics.java | 92 --
.../server/conf/store/impl/ZooPropLoader.java | 19 +-
.../server/conf/store/impl/ZooPropStore.java | 15 +-
.../server/metrics/MeterRegistryEnvPropImpl.java | 75 +
.../accumulo/server/metrics/MetricsInfoImpl.java | 319 +
.../apache/accumulo/server/rpc/TServerUtils.java | 11 +-
.../apache/accumulo/server/rpc/TimedProcessor.java | 6 +-
.../conf/store/impl/PropCacheCaffeineImplTest.java | 6 +-
.../server/conf/store/impl/PropStoreEventTest.java | 8 +-
.../server/conf/store/impl/ZooPropLoaderTest.java | 78 ++---
.../metrics/MeterRegistryEnvPropImplTest.java | 50
.../server/metrics/MetricsInfoImplTest.java| 84 ++
.../accumulo/server/rpc/TServerUtilsTest.java | 19 +-
.../coordinator/CompactionCoordinator.java | 15 +-
.../org/apache/accumulo/compactor/Compactor.java | 18 +-
.../apache/accumulo/gc/SimpleGarbageCollector.java | 19 +-
.../java/org/apache/accumulo/manager/Manager.java | 20 +-
.../accumulo/manager/metrics/ManagerMetrics.java | 24 +-
.../accumulo/manager/metrics/fate/FateMetrics.java | 49 ++--
.../java/org/apache/accumulo/monitor/Monitor.java | 6 +
.../org/apache/accumulo/tserver/ScanServer.java| 19 +-
.../org/apache/accumulo/tserver/TabletServer.java | 34 +--
.../tserver/TabletServerResourceManager.java | 35 +--
.../tserver/metrics/TabletServerMinCMetrics.java | 8 +-
.../tserver/metrics/TabletServerScanMetrics.java | 24 +-
.../accumulo/tserver/tablet/CompactableImpl.java | 9 +
.../test/conf/store/PropCacheCaffeineImplZkIT.java | 10 +-
.../accumulo/test/functional/ZombieTServer.java| 2 +-
.../apache/accumulo/test/metrics/MetricsIT.java| 16 +-
.../test/metrics/TestStatsDRegistryFactory.java| 6 +-
.../accumulo/test/performance/NullTserver.java | 2 +-
test/src/main/resources/log4j2-test.properties | 23 --
47 files changed, 1109 insertions(+), 609 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 05d9a02c69..5d70035fab 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -271,6 +271,7 @@
+
io[.]micrometer[.]core[.]instrument[.]MeterRegistry
io[.]opentelemetry[.]api[.]OpenTelemetry
org[.]apache[.]hadoop[.]io[.]Text
org[.]apache[.]accumulo[.]core[.]client[.].*
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 7d48c05ae1..6dc56205ba 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -322,8 +322,15 @@ public enum Property {
"Enables metrics functionality using Micrometer.", "2.1.0"),
GENERAL_MICROMETER_JVM_METRICS_ENABLED("general.micrometer.jvm.metrics.enabled",
"false",
PropertyType.BOOLEAN, "Enables JVM metrics functionality using
Micrometer.", "2.1.0"),
- GENERAL_MICROMETER_FACTORY("general.micrometer.factory", "",
PropertyType.CLASSNAME,
- "Name of class that implements MeterRegistryFactory.", "2.1.0"),
+ GENERAL_MICROMETER_FACTORY("general.micrometer.factory",
+ "org.apache.accumulo.core.spi.metrics.Log
(accumulo) branch 2.1 updated: Additional metrics changes (#4459)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/2.1 by this push: new 6d0b476852 Additional metrics changes (#4459) 6d0b476852 is described below commit 6d0b4768522187d8c43b9bf0850034d80a684235 Author: EdColeman AuthorDate: Wed Apr 24 08:55:21 2024 -0400 Additional metrics changes (#4459) * Completes removal of static MetricsUtils class (this completes changes started in #4384) * Adds metric initialization to ServiceContext - provides consistent use of tags and service metrics initialization so service name and ip address are applied as common tags. * Provide a metrics SPI to enable metric provider extensions by users - Adds the MeterRegistryFactory to ..core.spi.metrics and deprecates .core.metrics.MeterRegistryFactory - Modified the GENERAL_MICROMETER_FACTORY to be a list of class names. This allows multiple providers to be used. For example, logging and statsd. - Provides a sample metrics implementation using a LoggingProvider * Provides general method that can be used to pass parameters using standard properties, with `general.custom.metrics.opts prefix` to allow for user runtime customization * Removes prop store metrics --- core/pom.xml | 1 + .../org/apache/accumulo/core/conf/Property.java| 11 +- .../core/metrics/MeterRegistryFactory.java | 5 + .../apache/accumulo/core/metrics/MetricsInfo.java | 114 .../accumulo/core/metrics/MetricsProducer.java | 43 --- .../apache/accumulo/core/metrics/MetricsUtil.java | 149 -- .../spi/metrics/LoggingMeterRegistryFactory.java | 81 ++ .../core/spi/metrics/MeterRegistryFactory.java | 72 + .../accumulo/core/util/threads/ThreadPools.java| 38 ++- .../metrics/LoggingMeterRegistryFactoryTest.java | 52 .../miniclusterImpl/MiniAccumuloConfigImpl.java| 3 + .../org/apache/accumulo/server/AbstractServer.java | 9 +- .../org/apache/accumulo/server/ServerContext.java | 13 + .../conf/store/impl/PropCacheCaffeineImpl.java | 18 +- .../server/conf/store/impl/PropStoreMetrics.java | 92 -- .../server/conf/store/impl/ZooPropLoader.java | 19 +- .../server/conf/store/impl/ZooPropStore.java | 15 +- .../server/metrics/MeterRegistryEnvPropImpl.java | 75 + .../accumulo/server/metrics/MetricsInfoImpl.java | 319 + .../apache/accumulo/server/rpc/TServerUtils.java | 11 +- .../apache/accumulo/server/rpc/TimedProcessor.java | 6 +- .../conf/store/impl/PropCacheCaffeineImplTest.java | 6 +- .../server/conf/store/impl/PropStoreEventTest.java | 8 +- .../server/conf/store/impl/ZooPropLoaderTest.java | 78 ++--- .../metrics/MeterRegistryEnvPropImplTest.java | 50 .../server/metrics/MetricsInfoImplTest.java| 84 ++ .../accumulo/server/rpc/TServerUtilsTest.java | 19 +- .../coordinator/CompactionCoordinator.java | 14 +- .../coordinator/CompactionCoordinatorTest.java | 11 + .../org/apache/accumulo/compactor/Compactor.java | 17 +- .../apache/accumulo/compactor/CompactorTest.java | 10 + .../apache/accumulo/gc/SimpleGarbageCollector.java | 19 +- .../java/org/apache/accumulo/manager/Manager.java | 19 +- .../accumulo/manager/metrics/ManagerMetrics.java | 17 +- .../manager/metrics/ReplicationMetrics.java| 12 +- .../accumulo/manager/metrics/fate/FateMetrics.java | 49 ++-- .../java/org/apache/accumulo/monitor/Monitor.java | 6 + .../org/apache/accumulo/tserver/ScanServer.java| 17 +- .../org/apache/accumulo/tserver/TabletServer.java | 17 +- .../tserver/TabletServerResourceManager.java | 35 +-- .../tserver/metrics/TabletServerMinCMetrics.java | 8 +- .../tserver/metrics/TabletServerScanMetrics.java | 19 +- .../test/conf/store/PropCacheCaffeineImplZkIT.java | 10 +- .../accumulo/test/functional/ZombieTServer.java| 2 +- .../apache/accumulo/test/metrics/MetricsIT.java| 16 +- .../test/metrics/TestStatsDRegistryFactory.java| 6 +- .../accumulo/test/performance/NullTserver.java | 2 +- test/src/main/resources/log4j2-test.properties | 23 -- 48 files changed, 1112 insertions(+), 608 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index d8cc050b33..6e4ffc9883 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -277,6 +277,7 @@ + io[.]micrometer[.]core[.]instrument[.]MeterRegistry io[.]opentelemetry[.]api[.]OpenTelemetry org[.]apache[.]hadoop[.]io[.]Text org[.]apache[.]accumulo[.]core[.]client[.].* diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf
(accumulo) branch elasticity updated (8c68e4732e -> 14645fcd87)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8c68e4732e Fixed NPE in CompactionDriver when null passed to Text constructor (#4443) add ac80267fcb Add utility to create an empty wal file. (#4116) add 14645fcd87 Merge remote-tracking branch 'upstream/main' into elasticity No new revisions were added by this update. Summary of changes: .../accumulo/core/file/rfile/CreateEmpty.java | 113 - .../apache/accumulo/tserver/util/CreateEmpty.java | 198 .../accumulo/tserver/util/CreateEmptyTest.java | 263 + .../test/functional/RecoveryWithEmptyRFileIT.java | 3 +- .../apache/accumulo/test/start/KeywordStartIT.java | 2 +- 5 files changed, 464 insertions(+), 115 deletions(-) delete mode 100644 core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java create mode 100644 server/tserver/src/main/java/org/apache/accumulo/tserver/util/CreateEmpty.java create mode 100644 server/tserver/src/test/java/org/apache/accumulo/tserver/util/CreateEmptyTest.java
(accumulo) branch main updated: Add utility to create an empty wal file. (#4116)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ac80267fcb Add utility to create an empty wal file. (#4116)
ac80267fcb is described below
commit ac80267fcb797dd57c5fead1cdd4e96d8a480a00
Author: EdColeman
AuthorDate: Tue Apr 9 11:24:18 2024 -0400
Add utility to create an empty wal file. (#4116)
Updated the CreateEmpty utility with an option to create empty wal file(s).
---
.../accumulo/core/file/rfile/CreateEmpty.java | 113 -
.../apache/accumulo/tserver/util/CreateEmpty.java | 198
.../accumulo/tserver/util/CreateEmptyTest.java | 263 +
.../test/functional/RecoveryWithEmptyRFileIT.java | 3 +-
.../apache/accumulo/test/start/KeywordStartIT.java | 2 +-
5 files changed, 464 insertions(+), 115 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java
deleted file mode 100644
index 5a8d4dc104..00
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.core.file.rfile;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.accumulo.core.cli.Help;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
-import org.apache.accumulo.core.file.FileSKVWriter;
-import org.apache.accumulo.core.file.rfile.bcfile.Compression;
-import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
-import org.apache.accumulo.core.spi.crypto.NoCryptoServiceFactory;
-import org.apache.accumulo.core.spi.file.rfile.compression.NoCompression;
-import org.apache.accumulo.start.spi.KeywordExecutable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-import com.google.auto.service.AutoService;
-
-/**
- * Create an empty RFile for use in recovering from data loss where Accumulo
still refers internally
- * to a path.
- */
-@AutoService(KeywordExecutable.class)
-public class CreateEmpty implements KeywordExecutable {
- private static final Logger log = LoggerFactory.getLogger(CreateEmpty.class);
-
- public static class NamedLikeRFile implements IParameterValidator {
-@Override
-public void validate(String name, String value) throws ParameterException {
- if (!value.endsWith(".rf")) {
-throw new ParameterException("File must end with .rf and '" + value +
"' does not.");
- }
-}
- }
-
- public static class IsSupportedCompressionAlgorithm implements
IParameterValidator {
-@Override
-public void validate(String name, String value) throws ParameterException {
- List algorithms = Compression.getSupportedAlgorithms();
- if (!algorithms.contains(value)) {
-throw new ParameterException("Compression codec must be one of " +
algorithms);
- }
-}
- }
-
- static class Opts extends Help {
-@Parameter(names = {"-c", "--codec"}, description = "the compression codec
to use.",
-validateWith = IsSupportedCompressionAlgorithm.class)
-String codec = new NoCompression().getName();
-@Parameter(
-description = " { ... } Each path given is a URL."
-+ " Relative paths are resolved according to the default
filesystem defined in"
-+ " your Hadoop configuration, which is usually an HDFS instance.",
-required = true, validateWith = NamedLikeRFile.class)
-List files = new ArrayList<>();
- }
-
- public static void main(String[] args) throws Exception {
-new CreateEmpty().execute(args);
- }
-
- @Override
- public String keyword() {
-return
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 2f4b0372512b3cf677e7a66d4b5ee8482a36a0da
Merge: ce83488245 0ad96b1dc0
Author: Ed Coleman
AuthorDate: Tue Mar 19 20:31:36 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../accumulo/core/clientImpl/ClientContext.java| 9 +-
.../core/clientImpl/ConditionalWriterImpl.java | 2 +-
.../core/clientImpl/InstanceOperationsImpl.java| 4 +-
.../core/clientImpl/TableOperationsImpl.java | 4 +-
.../core/clientImpl/TabletServerBatchReader.java | 5 +-
.../core/clientImpl/TabletServerBatchWriter.java | 8 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 8 +-
.../accumulo/core/file/BloomFilterLayer.java | 5 +-
.../file/blockfile/cache/lru/LruBlockCache.java| 4 +-
.../blockfile/cache/tinylfu/TinyLfuBlockCache.java | 2 +-
.../util/compaction/ExternalCompactionUtil.java| 9 +-
.../accumulo/core/util/threads/ThreadPools.java| 379 +
.../core/file/rfile/MultiThreadedRFileTest.java| 7 +-
.../threads/ThreadPoolExecutorBuilderTest.java | 79 +
.../server/conf/ServerConfigurationFactory.java| 4 +-
.../conf/store/impl/PropCacheCaffeineImpl.java | 7 +-
.../server/conf/store/impl/PropStoreWatcher.java | 5 +-
.../accumulo/server/fs/VolumeManagerImpl.java | 4 +-
.../accumulo/server/problems/ProblemReports.java | 7 +-
.../apache/accumulo/server/rpc/TServerUtils.java | 7 +-
.../server/util/RemoveEntriesForMissingFiles.java | 4 +-
.../server/util/VerifyTabletAssignments.java | 5 +-
.../server/conf/store/impl/ReadyMonitorTest.java | 4 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 2 +-
.../compaction/coordinator/QueueMetrics.java | 4 +-
.../accumulo/manager/metrics/fate/FateMetrics.java | 4 +-
.../accumulo/manager/recovery/RecoveryManager.java | 4 +-
.../apache/accumulo/manager/split/Splitter.java| 5 +-
.../manager/tableOps/bulkVer2/TabletRefresher.java | 2 +-
.../manager/upgrade/UpgradeCoordinator.java| 8 +-
.../tserver/TabletServerResourceManager.java | 21 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 5 +-
.../accumulo/tserver/log/TabletServerLogger.java | 4 +-
.../accumulo/test/BalanceWithOfflineTableIT.java | 5 +-
.../test/functional/BatchWriterFlushIT.java| 2 +-
35 files changed, 409 insertions(+), 229 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
index c009e14b14,02aeef6bf6..bf650a077f
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
@@@ -372,8 -372,8 +372,8 @@@ public class ConditionalWriterImpl impl
this.auths = config.getAuthorizations();
this.ve = new VisibilityEvaluator(config.getAuthorizations());
this.threadPool = context.threadPools().createScheduledExecutorService(
- config.getMaxWriteThreads(), this.getClass().getSimpleName(), false);
+ config.getMaxWriteThreads(), this.getClass().getSimpleName());
-this.locator = new SyncingTabletLocator(context, tableId);
+this.locator = new SyncingClientTabletCache(context, tableId);
this.serverQueues = new HashMap<>();
this.tableId = tableId;
this.tableName = tableName;
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 297ed70495,6b94fd81cf..2ea43612f1
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@@ -448,74 -422,109 +448,76 @@@ public class TableOperationsImpl extend
}
}
- private static class SplitEnv {
-private final String tableName;
-private final TableId tableId;
-private final ExecutorService executor;
-private final CountDownLatch latch;
-private final AtomicReference exception;
-
-SplitEnv(String tableName, TableId tableId, ExecutorService executor,
CountDownLatch latch,
-AtomicReference exception) {
- this.tableName = tableName;
- this.tableId = tableId;
- this.executor = executor;
- this.latch = latch;
- this.exception = exception;
-}
- }
-
- private class SplitTask implements Runnable {
-
-private List splits;
-private SplitEnv env;
+ /**
+ * On the server side the fate operation will exit w/o an error if the
tablet requested to split
+ * does not exist. When this happens it will also return an empty string.
In the case where the
+ * fate operation successfully splits the tablet it will return the
following string. This code
+ * uses this return va
(accumulo) branch elasticity updated (ce83488245 -> 2f4b037251)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from ce83488245 Reduced number of compactions in CompactionConfigChangeIT (#4402) add ec8ae122ed Use fluent-style builder for pool creation, replacing overloaded methods (#4384) add 0ad96b1dc0 Merge remote-tracking branch 'upstream/2.1' new 2f4b037251 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/clientImpl/ClientContext.java| 9 +- .../core/clientImpl/ConditionalWriterImpl.java | 2 +- .../core/clientImpl/InstanceOperationsImpl.java| 4 +- .../core/clientImpl/TableOperationsImpl.java | 4 +- .../core/clientImpl/TabletServerBatchReader.java | 5 +- .../core/clientImpl/TabletServerBatchWriter.java | 8 +- .../accumulo/core/clientImpl/bulk/BulkImport.java | 8 +- .../accumulo/core/file/BloomFilterLayer.java | 5 +- .../file/blockfile/cache/lru/LruBlockCache.java| 4 +- .../blockfile/cache/tinylfu/TinyLfuBlockCache.java | 2 +- .../util/compaction/ExternalCompactionUtil.java| 9 +- .../accumulo/core/util/threads/ThreadPools.java| 379 + .../core/file/rfile/MultiThreadedRFileTest.java| 7 +- .../threads/ThreadPoolExecutorBuilderTest.java | 79 + .../server/conf/ServerConfigurationFactory.java| 4 +- .../conf/store/impl/PropCacheCaffeineImpl.java | 7 +- .../server/conf/store/impl/PropStoreWatcher.java | 5 +- .../accumulo/server/fs/VolumeManagerImpl.java | 4 +- .../accumulo/server/problems/ProblemReports.java | 7 +- .../apache/accumulo/server/rpc/TServerUtils.java | 7 +- .../server/util/RemoveEntriesForMissingFiles.java | 4 +- .../server/util/VerifyTabletAssignments.java | 5 +- .../server/conf/store/impl/ReadyMonitorTest.java | 4 +- .../main/java/org/apache/accumulo/gc/GCRun.java| 2 +- .../compaction/coordinator/QueueMetrics.java | 4 +- .../accumulo/manager/metrics/fate/FateMetrics.java | 4 +- .../accumulo/manager/recovery/RecoveryManager.java | 4 +- .../apache/accumulo/manager/split/Splitter.java| 5 +- .../manager/tableOps/bulkVer2/TabletRefresher.java | 2 +- .../manager/upgrade/UpgradeCoordinator.java| 8 +- .../tserver/TabletServerResourceManager.java | 21 +- .../org/apache/accumulo/tserver/log/LogSorter.java | 5 +- .../accumulo/tserver/log/TabletServerLogger.java | 4 +- .../accumulo/test/BalanceWithOfflineTableIT.java | 5 +- .../test/functional/BatchWriterFlushIT.java| 2 +- 35 files changed, 409 insertions(+), 229 deletions(-) create mode 100644 core/src/test/java/org/apache/accumulo/core/util/threads/ThreadPoolExecutorBuilderTest.java
(accumulo) branch main updated (8b0262d5b3 -> 0ad96b1dc0)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8b0262d5b3 Deprecated MiniAccumuloConfig setNumServer methods (#4374) add ec8ae122ed Use fluent-style builder for pool creation, replacing overloaded methods (#4384) new 0ad96b1dc0 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/clientImpl/ClientContext.java| 9 +- .../core/clientImpl/ConditionalWriterImpl.java | 2 +- .../core/clientImpl/InstanceOperationsImpl.java| 4 +- .../core/clientImpl/TableOperationsImpl.java | 3 +- .../core/clientImpl/TabletServerBatchReader.java | 5 +- .../core/clientImpl/TabletServerBatchWriter.java | 8 +- .../accumulo/core/clientImpl/bulk/BulkImport.java | 8 +- .../accumulo/core/file/BloomFilterLayer.java | 5 +- .../file/blockfile/cache/lru/LruBlockCache.java| 4 +- .../blockfile/cache/tinylfu/TinyLfuBlockCache.java | 2 +- .../util/compaction/ExternalCompactionUtil.java| 10 +- .../accumulo/core/util/threads/ThreadPools.java| 386 + .../core/file/rfile/MultiThreadedRFileTest.java| 7 +- .../threads/ThreadPoolExecutorBuilderTest.java | 79 + .../server/conf/ServerConfigurationFactory.java| 4 +- .../conf/store/impl/PropCacheCaffeineImpl.java | 7 +- .../server/conf/store/impl/PropStoreWatcher.java | 5 +- .../accumulo/server/fs/VolumeManagerImpl.java | 4 +- .../accumulo/server/problems/ProblemReports.java | 7 +- .../apache/accumulo/server/rpc/TServerUtils.java | 7 +- .../server/util/RemoveEntriesForMissingFiles.java | 4 +- .../server/util/VerifyTabletAssignments.java | 5 +- .../server/conf/store/impl/ReadyMonitorTest.java | 4 +- .../coordinator/CompactionCoordinator.java | 4 +- .../accumulo/coordinator/CompactionFinalizer.java | 11 +- .../main/java/org/apache/accumulo/gc/GCRun.java| 2 +- .../accumulo/manager/metrics/fate/FateMetrics.java | 4 +- .../accumulo/manager/recovery/RecoveryManager.java | 4 +- .../manager/upgrade/UpgradeCoordinator.java| 8 +- .../tserver/TabletServerResourceManager.java | 31 +- .../tserver/compactions/CompactionService.java | 6 +- .../compactions/InternalCompactionExecutor.java| 9 +- .../org/apache/accumulo/tserver/log/LogSorter.java | 5 +- .../accumulo/tserver/log/TabletServerLogger.java | 4 +- .../metrics/CompactionExecutorsMetrics.java| 2 +- .../accumulo/test/BalanceWithOfflineTableIT.java | 5 +- .../test/functional/BatchWriterFlushIT.java| 2 +- .../accumulo/test/functional/BulkFailureIT.java| 4 +- 38 files changed, 433 insertions(+), 247 deletions(-) create mode 100644 core/src/test/java/org/apache/accumulo/core/util/threads/ThreadPoolExecutorBuilderTest.java
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 0ad96b1dc088070486ccc92a170545a6c91fab7f
Merge: 8b0262d5b3 ec8ae122ed
Author: Ed Coleman
AuthorDate: Tue Mar 19 19:02:30 2024 +
Merge remote-tracking branch 'upstream/2.1'
.../accumulo/core/clientImpl/ClientContext.java| 9 +-
.../core/clientImpl/ConditionalWriterImpl.java | 2 +-
.../core/clientImpl/InstanceOperationsImpl.java| 4 +-
.../core/clientImpl/TableOperationsImpl.java | 3 +-
.../core/clientImpl/TabletServerBatchReader.java | 5 +-
.../core/clientImpl/TabletServerBatchWriter.java | 8 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 8 +-
.../accumulo/core/file/BloomFilterLayer.java | 5 +-
.../file/blockfile/cache/lru/LruBlockCache.java| 4 +-
.../blockfile/cache/tinylfu/TinyLfuBlockCache.java | 2 +-
.../util/compaction/ExternalCompactionUtil.java| 10 +-
.../accumulo/core/util/threads/ThreadPools.java| 386 +
.../core/file/rfile/MultiThreadedRFileTest.java| 7 +-
.../threads/ThreadPoolExecutorBuilderTest.java | 79 +
.../server/conf/ServerConfigurationFactory.java| 4 +-
.../conf/store/impl/PropCacheCaffeineImpl.java | 7 +-
.../server/conf/store/impl/PropStoreWatcher.java | 5 +-
.../accumulo/server/fs/VolumeManagerImpl.java | 4 +-
.../accumulo/server/problems/ProblemReports.java | 7 +-
.../apache/accumulo/server/rpc/TServerUtils.java | 7 +-
.../server/util/RemoveEntriesForMissingFiles.java | 4 +-
.../server/util/VerifyTabletAssignments.java | 5 +-
.../server/conf/store/impl/ReadyMonitorTest.java | 4 +-
.../coordinator/CompactionCoordinator.java | 4 +-
.../accumulo/coordinator/CompactionFinalizer.java | 11 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 2 +-
.../accumulo/manager/metrics/fate/FateMetrics.java | 4 +-
.../accumulo/manager/recovery/RecoveryManager.java | 4 +-
.../manager/upgrade/UpgradeCoordinator.java| 8 +-
.../tserver/TabletServerResourceManager.java | 31 +-
.../tserver/compactions/CompactionService.java | 6 +-
.../compactions/InternalCompactionExecutor.java| 9 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 5 +-
.../accumulo/tserver/log/TabletServerLogger.java | 4 +-
.../metrics/CompactionExecutorsMetrics.java| 2 +-
.../accumulo/test/BalanceWithOfflineTableIT.java | 5 +-
.../test/functional/BatchWriterFlushIT.java| 2 +-
.../accumulo/test/functional/BulkFailureIT.java| 4 +-
38 files changed, 433 insertions(+), 247 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
index 12adfc5c98,147f19e5dd..334fb46a53
--- a/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/threads/ThreadPools.java
@@@ -243,46 -264,105 +261,82 @@@ public class ThreadPools
* @return ExecutorService impl
* @throws IllegalArgumentException if property is not handled
*/
- @SuppressWarnings("deprecation")
public ThreadPoolExecutor createExecutorService(final AccumuloConfiguration
conf,
final Property p, boolean emitThreadPoolMetrics) {
-
+ ThreadPoolExecutorBuilder builder;
switch (p) {
- case GENERAL_SIMPLETIMER_THREADPOOL_SIZE:
-return createScheduledExecutorService(conf.getCount(p),
"SimpleTimer");
case GENERAL_THREADPOOL_SIZE:
return createScheduledExecutorService(conf.getCount(p),
"GeneralExecutor",
emitThreadPoolMetrics);
- case MANAGER_BULK_THREADPOOL_SIZE:
-builder = getPoolBuilder("bulk
import").numCoreThreads(conf.getCount(p)).withTimeOut(
-conf.getTimeInMillis(Property.MANAGER_BULK_THREADPOOL_TIMEOUT),
MILLISECONDS);
-if (emitThreadPoolMetrics) {
- builder.enableThreadPoolMetrics();
-}
-return builder.build();
- case MANAGER_RENAME_THREADS:
-builder = getPoolBuilder("bulk
move").numCoreThreads(conf.getCount(p));
-if (emitThreadPoolMetrics) {
- builder.enableThreadPoolMetrics();
-}
-return builder.build();
case MANAGER_FATE_THREADPOOL_SIZE:
- return createFixedThreadPool(conf.getCount(p), "Repo Runner",
emitThreadPoolMetrics);
+ builder = getPoolBuilder("Repo
Runner").numCoreThreads(conf.getCount(p));
+ if (emitThreadPoolMetrics) {
+ builder.enableThreadPoolMetrics();
+ }
+ return builder.build();
case MANAGER_STATUS_THREAD_POOL_SIZE:
+ builder = getPoolBuilder("GatherTableInformation");
int threads = conf.getCount(p);
(accumulo) branch 2.1 updated: Use fluent-style builder for pool creation, replacing overloaded methods (#4384)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new ec8ae122ed Use fluent-style builder for pool creation, replacing
overloaded methods (#4384)
ec8ae122ed is described below
commit ec8ae122edfd5c25601694d107f18af014e28383
Author: EdColeman
AuthorDate: Tue Mar 19 13:31:38 2024 -0400
Use fluent-style builder for pool creation, replacing overloaded methods
(#4384)
* Use fluent-style builder for pool creation, replacing overloaded methods
* Replaces overloaded createThreadPool methods with a fluent-style builder
(ThreadPoolExecutorBuilderTest).
* Adds ThreadPoolExecutorBuilderTest test,
* Adds `createExecutorService` method that does not have an option to enable
metrics and replaces all occurrences in the code that was calling the
alternate
method with emitThreadPoolMetrics=false to use it.
The `createScheduledExecutorService` will be refactored in a future PR when
service
initialization is reworked. See PR #4342 for an example.
This change isolates where metrics can be enabled and makes finding them
easier. It also will make reviewing future changes easier because those
changes will be isolated to places that currently enable thread pool
metrics.
---
.../accumulo/core/clientImpl/ClientContext.java| 9 +-
.../core/clientImpl/ConditionalWriterImpl.java | 2 +-
.../core/clientImpl/InstanceOperationsImpl.java| 4 +-
.../core/clientImpl/TableOperationsImpl.java | 3 +-
.../core/clientImpl/TabletServerBatchReader.java | 5 +-
.../core/clientImpl/TabletServerBatchWriter.java | 8 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 8 +-
.../accumulo/core/file/BloomFilterLayer.java | 5 +-
.../file/blockfile/cache/lru/LruBlockCache.java| 4 +-
.../blockfile/cache/tinylfu/TinyLfuBlockCache.java | 2 +-
.../util/compaction/ExternalCompactionUtil.java| 10 +-
.../accumulo/core/util/threads/ThreadPools.java| 419 +
.../core/file/rfile/MultiThreadedRFileTest.java| 7 +-
.../threads/ThreadPoolExecutorBuilderTest.java | 79
.../accumulo/server/client/BulkImporter.java | 10 +-
.../server/conf/ServerConfigurationFactory.java| 4 +-
.../conf/store/impl/PropCacheCaffeineImpl.java | 7 +-
.../server/conf/store/impl/PropStoreWatcher.java | 5 +-
.../accumulo/server/fs/VolumeManagerImpl.java | 4 +-
.../accumulo/server/problems/ProblemReports.java | 7 +-
.../apache/accumulo/server/rpc/TServerUtils.java | 7 +-
.../server/util/RemoveEntriesForMissingFiles.java | 4 +-
.../server/util/VerifyTabletAssignments.java | 5 +-
.../server/conf/store/impl/ReadyMonitorTest.java | 4 +-
.../coordinator/CompactionCoordinator.java | 4 +-
.../accumulo/coordinator/CompactionFinalizer.java | 11 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 2 +-
.../java/org/apache/accumulo/manager/Manager.java | 2 +-
.../manager/metrics/ReplicationMetrics.java| 2 +-
.../accumulo/manager/metrics/fate/FateMetrics.java | 4 +-
.../accumulo/manager/recovery/RecoveryManager.java | 4 +-
.../manager/tableOps/bulkVer1/BulkImport.java | 6 +-
.../manager/upgrade/UpgradeCoordinator.java| 9 +-
.../org/apache/accumulo/tserver/TabletServer.java | 2 +-
.../tserver/TabletServerResourceManager.java | 31 +-
.../tserver/compactions/CompactionService.java | 6 +-
.../compactions/InternalCompactionExecutor.java| 9 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 5 +-
.../accumulo/tserver/log/TabletServerLogger.java | 4 +-
.../metrics/CompactionExecutorsMetrics.java| 2 +-
.../accumulo/test/BalanceWithOfflineTableIT.java | 5 +-
.../test/functional/BatchWriterFlushIT.java| 2 +-
42 files changed, 465 insertions(+), 267 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
index 8f7993812a..d02a2c743e 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
@@ -260,8 +260,9 @@ public class ClientContext implements AccumuloClient {
submitScannerReadAheadTask(Callable> c) {
ensureOpen();
if (scannerReadaheadPool == null) {
- scannerReadaheadPool = clientThreadPools.createThreadPool(0,
Integer.MAX_VALUE, 3L, SECONDS,
- "Accumulo scanner read ahead thread", new SynchronousQueue<>(),
true);
+ scannerReadaheadPool = clientThreadPools.getPoolBuilder("Accumulo
scanner read ahead thread")
+ .numCoreThreads(0).numMaxThreads(Inte
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 61ce280c8ebbfe92961bedf76500e59c1f935cf6 Merge: f1914da45a a3616bf46f Author: Ed Coleman AuthorDate: Fri Mar 15 21:21:51 2024 + Merge remote-tracking branch 'upstream/main' into elasticity pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch elasticity updated (f1914da45a -> 61ce280c8e)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from f1914da45a Merge branch 'main' into elasticity add f26a110292 bump zookeeper version from 3.9.1 to 3.9.2 (#4387) add a3616bf46f Merge remote-tracking branch 'upstream/2.1' new 61ce280c8e Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch main updated (fe552af0dd -> a3616bf46f)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from fe552af0dd Merge branch '2.1' add f26a110292 bump zookeeper version from 3.9.1 to 3.9.2 (#4387) new a3616bf46f Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a3616bf46f218d6b6cb39864f334f7c99acef258 Merge: fe552af0dd f26a110292 Author: Ed Coleman AuthorDate: Fri Mar 15 20:33:07 2024 + Merge remote-tracking branch 'upstream/2.1' pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch 2.1 updated: bump zookeeper version from 3.9.1 to 3.9.2 (#4387)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/2.1 by this push: new f26a110292 bump zookeeper version from 3.9.1 to 3.9.2 (#4387) f26a110292 is described below commit f26a110292a8ec57bee7470ae5942612399f4a54 Author: EdColeman AuthorDate: Fri Mar 15 16:12:45 2024 -0400 bump zookeeper version from 3.9.1 to 3.9.2 (#4387) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0191cdaca..5648b4f859 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 2.0.9 2.0.11 0.17.0 -3.9.1 +3.9.2
(accumulo) branch elasticity updated (3fe1115014 -> 8b0d235581)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 3fe1115014 Updates merge code to handle all tablet metadata columns (#4352) add 45bcbd44b9 Change to use ServerContext to get scheduled thread pool (#4353) add 9d9bcc1d48 Merge remote-tracking branch 'upstream/2.1' new 8b0d235581 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/manager/ManagerTime.java | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8b0d2355813300ed09b45920fc1cd7cf091df1be Merge: 3fe1115014 9d9bcc1d48 Author: Ed Coleman AuthorDate: Mon Mar 11 13:05:13 2024 + Merge remote-tracking branch 'upstream/main' into elasticity .../src/main/java/org/apache/accumulo/manager/ManagerTime.java | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-)
(accumulo) branch main updated (311c5fe821 -> 9d9bcc1d48)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 311c5fe821 Merge remote-tracking branch 'upstream/2.1' add 45bcbd44b9 Change to use ServerContext to get scheduled thread pool (#4353) new 9d9bcc1d48 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/manager/ManagerTime.java | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9d9bcc1d4875dfd1405f36ea2fc87eec24b62b4a Merge: 311c5fe821 45bcbd44b9 Author: Ed Coleman AuthorDate: Mon Mar 11 13:03:15 2024 + Merge remote-tracking branch 'upstream/2.1' .../src/main/java/org/apache/accumulo/manager/ManagerTime.java | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-)
(accumulo) branch 2.1 updated: Change to use ServerContext to get scheduled thread pool (#4353)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 45bcbd44b9 Change to use ServerContext to get scheduled thread pool
(#4353)
45bcbd44b9 is described below
commit 45bcbd44b9ae43a69552d2f1863490e75a376a32
Author: EdColeman
AuthorDate: Mon Mar 11 09:01:24 2024 -0400
Change to use ServerContext to get scheduled thread pool (#4353)
---
.../src/main/java/org/apache/accumulo/manager/ManagerTime.java | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java
b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java
index c7eac43ff0..8bd842fc9d 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerTime.java
@@ -64,10 +64,9 @@ public class ManagerTime {
throw new IOException("Error updating manager time", ex);
}
-ThreadPools.watchCriticalScheduledTask(
-
ThreadPools.getServerThreadPools().createGeneralScheduledExecutorService(conf)
-.scheduleWithFixedDelay(Threads.createNamedRunnable("Manager time
keeper", () -> run()),
-0, SECONDS.toMillis(10), MILLISECONDS));
+
ThreadPools.watchCriticalScheduledTask(manager.getContext().getScheduledExecutor()
+.scheduleWithFixedDelay(Threads.createNamedRunnable("Manager time
keeper", () -> run()), 0,
+SECONDS.toMillis(10), MILLISECONDS));
}
/**
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 0148294e35dbcf502fb0f9df6a178d669e4c8f3f
Merge: 7b3e52f1ae 311c5fe821
Author: Ed Coleman
AuthorDate: Fri Mar 8 19:02:50 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
Additional changes from merge
* Unused type variable in initializeFateInstance
* unneeded initialization
.../java/org/apache/accumulo/manager/Manager.java | 23 +++---
1 file changed, 12 insertions(+), 11 deletions(-)
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 95fb00777e,bdf73b9191..6758acfc1f
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -243,20 -231,6 +243,20 @@@ public class Manager extends AbstractSe
return state;
}
+ // ELASTICITIY_TODO it would be nice if this method could take DataLevel as
an argument and only
+ // retrieve information about compactions in that data level. Attempted
this and a lot of
+ // refactoring was needed to get that small bit of information to this
method. Would be best to
+ // address this after issue. May be best to attempt this after #3576.
+ public Map> getCompactionHints() {
- Map allConfig = null;
++Map allConfig;
+try {
+ allConfig = CompactionConfigStorage.getAllConfig(getContext(), tableId
-> true);
+} catch (InterruptedException | KeeperException e) {
+ throw new RuntimeException(e);
+}
+return Maps.transformValues(allConfig,
CompactionConfig::getExecutionHints);
+ }
+
public boolean stillManager() {
return getManagerState() != ManagerState.STOP;
}
@@@ -620,9 -757,9 +620,9 @@@
@Override
public void run() {
- EventCoordinator.Listener eventListener = nextEvent.getListener();
+ EventCoordinator.Tracker eventTracker = nextEvent.getTracker();
while (stillManager()) {
- long wait = DEFAULT_WAIT_FOR_WATCHER;
+ long wait;
try {
switch (getManagerGoalState()) {
case NORMAL:
@@@ -960,7 -1088,7 +960,7 @@@
log.info("Started Manager client service at {}", sa.address);
// block until we can obtain the ZK lock for the manager
--ServiceLockData sld = null;
++ServiceLockData sld;
try {
sld = getManagerLock(ServiceLock.path(zroot + Constants.ZMANAGER_LOCK));
} catch (KeeperException | InterruptedException e) {
@@@ -1072,17 -1197,17 +1072,17 @@@
}
try {
- var metaInstance = initializeFateInstance(context,
FateInstanceType.META,
- final AgeOffStore store = new AgeOffStore<>(
- new org.apache.accumulo.core.fate.ZooStore<>(getZooKeeperRoot() +
Constants.ZFATE,
- context.getZooReaderWriter()),
- HOURS.toMillis(8), System::currentTimeMillis);
-
- Fate f = new Fate<>(this, store, TraceRepo::toLogString,
getConfiguration());
- fateRef.set(f);
++ var metaInstance = initializeFateInstance(context,
+ new ZooStore<>(getZooKeeperRoot() + Constants.ZFATE,
context.getZooReaderWriter()));
- var userInstance = initializeFateInstance(context,
FateInstanceType.USER,
++ var userInstance = initializeFateInstance(context,
+ new AccumuloStore<>(context, AccumuloTable.FATE.tableName()));
+
+ if (!fateRefs.compareAndSet(null,
+ Map.of(FateInstanceType.META, metaInstance, FateInstanceType.USER,
userInstance))) {
+throw new IllegalStateException(
+"Unexpected previous fate reference map already initialized");
+ }
fateReadyLatch.countDown();
-
- ThreadPools.watchCriticalScheduledTask(context.getScheduledExecutor()
- .scheduleWithFixedDelay(store::ageOff, 63000, 63000, MILLISECONDS));
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException("Exception setting up FaTE cleanup
thread", e);
}
@@@ -1117,15 -1242,9 +1117,15 @@@
}
String address = sa.address.toString();
-sld = new ServiceLockData(sld.getServerUUID(ThriftService.MANAGER),
address,
-ThriftService.MANAGER);
-log.info("Setting manager lock data to {}", sld.toString());
+UUID uuid = sld.getServerUUID(ThriftService.MANAGER);
+ServiceDescriptors descriptors = new ServiceDescriptors();
+for (ThriftService svc : new ThriftService[] {ThriftService.MANAGER,
ThriftService.COORDINATOR,
+ThriftService.FATE}) {
+ descriptors.addService(new ServiceDescriptor(uuid, svc, address,
this.getResourceGroup()));
+}
+
+sld = new ServiceLockData(descriptors);
- log.info("Setting manager lock data
(accumulo) branch elasticity updated (7b3e52f1ae -> 0148294e35)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7b3e52f1ae Merge remote-tracking branch 'upstream/main' into elasticity add d3f8a8b2b5 fixes minor qualtity check issues (#4351) add 311c5fe821 Merge remote-tracking branch 'upstream/2.1' new 0148294e35 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/manager/Manager.java | 23 +++--- 1 file changed, 12 insertions(+), 11 deletions(-)
(accumulo) branch main updated (dcb09a726b -> 311c5fe821)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from dcb09a726b Merge remote-tracking branch 'upstream/2.1' add d3f8a8b2b5 fixes minor qualtity check issues (#4351) new 311c5fe821 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../src/main/java/org/apache/accumulo/manager/Manager.java| 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 311c5fe821e19f886334fb58e1b79b46aa4957e0 Merge: dcb09a726b d3f8a8b2b5 Author: Ed Coleman AuthorDate: Fri Mar 8 18:10:56 2024 + Merge remote-tracking branch 'upstream/2.1' .../src/main/java/org/apache/accumulo/manager/Manager.java| 11 ++- 1 file changed, 6 insertions(+), 5 deletions(-)
(accumulo) branch 2.1 updated: fixes minor qualtity check issues (#4351)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d3f8a8b2b5 fixes minor qualtity check issues (#4351)
d3f8a8b2b5 is described below
commit d3f8a8b2b587bed44c978fc6b742e7bfe286148f
Author: EdColeman
AuthorDate: Fri Mar 8 13:09:02 2024 -0500
fixes minor qualtity check issues (#4351)
---
.../src/main/java/org/apache/accumulo/manager/Manager.java| 11 ++-
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 9d71e542f9..1b63f18381 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -595,7 +595,6 @@ public class Manager extends AbstractServer
}
return TabletGoalState.UNASSIGNED;
case UNLOAD_ROOT_TABLET:
-return TabletGoalState.UNASSIGNED;
case STOP:
return TabletGoalState.UNASSIGNED;
default:
@@ -766,7 +765,7 @@ public class Manager extends AbstractServer
public void run() {
EventCoordinator.Listener eventListener = nextEvent.getListener();
while (stillManager()) {
-long wait = DEFAULT_WAIT_FOR_WATCHER;
+long wait;
try {
switch (getManagerGoalState()) {
case NORMAL:
@@ -1497,7 +1496,7 @@ public class Manager extends AbstractServer
}
if (acquiredLock) {
-Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " +
failedToAcquireLock,
+Halt.halt("Zoolock in unexpected state acquiredLock true with FAL " +
failedToAcquireLock,
-1);
}
@@ -1509,7 +1508,9 @@ public class Manager extends AbstractServer
while (!acquiredLock && !failedToAcquireLock) {
try {
wait();
-} catch (InterruptedException e) {}
+} catch (InterruptedException e) {
+ // empty
+}
}
}
}
@@ -1655,7 +1656,7 @@ public class Manager extends AbstractServer
for (TableId tableId : context.getTableIdToNameMap().keySet()) {
TableState state = manager.getTableState(tableId);
- if ((state != null) && (state == TableState.ONLINE)) {
+ if (state == TableState.ONLINE) {
result.add(tableId);
}
}
(accumulo) branch elasticity updated (c7ae69b881 -> 7b3e52f1ae)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from c7ae69b881 fixes bug and removes unneeded wait in delete table (#4346) add efdc4fc3d7 Remove pool metrics instrumentation from LRU caches (#4345) add dcb09a726b Merge remote-tracking branch 'upstream/2.1' new 7b3e52f1ae Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/file/blockfile/cache/lru/LruBlockCache.java | 14 -- .../file/blockfile/cache/tinylfu/TinyLfuBlockCache.java| 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 7b3e52f1ae0f1a02d0804744137db521e96cf201
Merge: c7ae69b881 dcb09a726b
Author: Ed Coleman
AuthorDate: Thu Mar 7 19:42:42 2024 +
Merge remote-tracking branch 'upstream/main' into elasticity
.../core/file/blockfile/cache/lru/LruBlockCache.java | 14 --
.../file/blockfile/cache/tinylfu/TinyLfuBlockCache.java| 4 ++--
2 files changed, 10 insertions(+), 8 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
index feb6a1b5c1,04cab1a38c..3656b99815
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
@@@ -63,15 -62,15 +63,15 @@@ public final class TinyLfuBlockCache im
private final Policy.Eviction policy;
private final int maxSize;
private final ScheduledExecutorService statsExecutor =
ThreadPools.getServerThreadPools()
- .createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor",
true);
+ .createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor",
false);
public TinyLfuBlockCache(Configuration conf, CacheType type) {
-cache = Caffeine.newBuilder()
+cache =
Caches.getInstance().createNewBuilder(CacheName.TINYLFU_BLOCK_CACHE, false)
.initialCapacity((int) Math.ceil(1.2 * conf.getMaxSize(type) /
conf.getBlockSize()))
-.weigher((String blockName, Block block) -> {
+.recordStats().weigher((String blockName, Block block) -> {
int keyWeight = ClassSize.align(blockName.length()) +
ClassSize.STRING;
return keyWeight + block.weight();
-}).maximumWeight(conf.getMaxSize(type)).recordStats().build();
+}).maximumWeight(conf.getMaxSize(type)).build();
policy = cache.policy().eviction().orElseThrow();
maxSize = (int) Math.min(Integer.MAX_VALUE, policy.getMaximum());
ScheduledFuture future =
statsExecutor.scheduleAtFixedRate(this::logStats, STATS_PERIOD_SEC,
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit dcb09a726be6a3bb868cbff13f53e91cf1ac26b8 Merge: f3d5fb01d7 efdc4fc3d7 Author: Ed Coleman AuthorDate: Thu Mar 7 19:41:01 2024 + Merge remote-tracking branch 'upstream/2.1' .../core/file/blockfile/cache/lru/LruBlockCache.java | 14 -- .../file/blockfile/cache/tinylfu/TinyLfuBlockCache.java| 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-)
(accumulo) branch main updated (f3d5fb01d7 -> dcb09a726b)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from f3d5fb01d7 Merge branch '2.1' add efdc4fc3d7 Remove pool metrics instrumentation from LRU caches (#4345) new dcb09a726b Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/file/blockfile/cache/lru/LruBlockCache.java | 14 -- .../file/blockfile/cache/tinylfu/TinyLfuBlockCache.java| 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-)
(accumulo) branch 2.1 updated: Remove pool metrics instrumentation from LRU caches (#4345)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new efdc4fc3d7 Remove pool metrics instrumentation from LRU caches (#4345)
efdc4fc3d7 is described below
commit efdc4fc3d725d9bb2c364db6548c9251ea793a6d
Author: EdColeman
AuthorDate: Thu Mar 7 14:38:40 2024 -0500
Remove pool metrics instrumentation from LRU caches (#4345)
* The LRU metrics use a thread pool with 1 thread to generate cache
statistics. The pool size is not configurable. It seems unnecessary
to produce detail pool metrics for these caches.
* resolve missing isPresent check for weighted size, default to 0 if not
present in TinyLfuBlockCache
* other minor quality check improvements. (final varibles, unneeded
initializations.)
---
.../core/file/blockfile/cache/lru/LruBlockCache.java | 14 --
.../file/blockfile/cache/tinylfu/TinyLfuBlockCache.java| 4 ++--
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
index e3c399ec58..01972f8ffe 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCache.java
@@ -103,7 +103,7 @@ public class LruBlockCache extends
SynchronousLoadingBlockCache implements Block
/** Statistics thread schedule pool (for heavy debugging, could remove) */
private final ScheduledExecutorService scheduleThreadPool =
ThreadPools.getServerThreadPools()
- .createScheduledExecutorService(1, "LRUBlockCacheStats", true);
+ .createScheduledExecutorService(1, "LRUBlockCacheStats", false);
/** Current size of cache */
private final AtomicLong size;
@@ -391,9 +391,9 @@ public class LruBlockCache extends
SynchronousLoadingBlockCache implements Block
*/
private class BlockBucket implements Comparable {
-private CachedBlockQueue queue;
-private long totalSize = 0;
-private long bucketSize;
+private final CachedBlockQueue queue;
+private long totalSize;
+private final long bucketSize;
public BlockBucket(long bytesToFree, long blockSize, long bucketSize) {
this.bucketSize = bucketSize;
@@ -512,7 +512,7 @@ public class LruBlockCache extends
SynchronousLoadingBlockCache implements Block
* Thread is triggered into action by {@link LruBlockCache#runEviction()}
*/
private static class EvictionThread extends AccumuloDaemonThread {
-private WeakReference cache;
+private final WeakReference cache;
private boolean running = false;
public EvictionThread(LruBlockCache cache) {
@@ -532,7 +532,9 @@ public class LruBlockCache extends
SynchronousLoadingBlockCache implements Block
running = true;
try {
this.wait();
- } catch (InterruptedException e) {}
+ } catch (InterruptedException e) {
+// empty
+ }
}
LruBlockCache cache = this.cache.get();
if (cache == null) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
index beabe5a7ad..04cab1a38c 100644
---
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
@@ -62,7 +62,7 @@ public final class TinyLfuBlockCache implements BlockCache {
private final Policy.Eviction policy;
private final int maxSize;
private final ScheduledExecutorService statsExecutor =
ThreadPools.getServerThreadPools()
- .createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor",
true);
+ .createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor",
false);
public TinyLfuBlockCache(Configuration conf, CacheType type) {
cache = Caffeine.newBuilder()
@@ -118,7 +118,7 @@ public final class TinyLfuBlockCache implements BlockCache {
private void logStats() {
double maxMB = ((double) policy.getMaximum()) / ((double) (1024 * 1024));
-double sizeMB = ((double) policy.weightedSize().getAsLong()) / ((double)
(1024 * 1024));
+double sizeMB = ((double) policy.weightedSize().orElse(0)) / ((double)
(1024 * 1024));
double freeMB = maxMB - sizeMB;
log.debug("Cache Size={}MB, Free={}MB, Max={}MB, Blocks={}", sizeMB,
freeMB, maxMB,
cache.estimatedSize());
(accumulo) branch elasticity updated (f75dbf8c5b -> 401237dba1)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from f75dbf8c5b Remove unused argument to setOperationIdOnce (#4294) add c72bf8c00c Remove unneeded SuppressFBWarnings annotation from ZooPropStore (#4304) add c853f64908 Merge remote-tracking branch 'upstream/2.1' add 401237dba1 Merge remote-tracking branch 'upstream/main' into elasticity No new revisions were added by this update. Summary of changes: .../java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java | 4 1 file changed, 4 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c853f6490856f21db39aab7ce80cf8cf820a455c Merge: ea5afb9d85 c72bf8c00c Author: Ed Coleman AuthorDate: Mon Feb 26 14:59:57 2024 + Merge remote-tracking branch 'upstream/2.1' .../java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java | 4 1 file changed, 4 deletions(-)
(accumulo) branch main updated (ea5afb9d85 -> c853f64908)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from ea5afb9d85 Merge branch '2.1' add c72bf8c00c Remove unneeded SuppressFBWarnings annotation from ZooPropStore (#4304) new c853f64908 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java | 4 1 file changed, 4 deletions(-)
(accumulo) branch 2.1 updated: Remove unneeded SuppressFBWarnings annotation from ZooPropStore (#4304)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new c72bf8c00c Remove unneeded SuppressFBWarnings annotation from
ZooPropStore (#4304)
c72bf8c00c is described below
commit c72bf8c00cb196d048d1aa35077327c55d6e58e2
Author: EdColeman
AuthorDate: Mon Feb 26 09:58:19 2024 -0500
Remove unneeded SuppressFBWarnings annotation from ZooPropStore (#4304)
---
.../java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java | 4
1 file changed, 4 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
index aed65fdbbe..a345b58c46 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/ZooPropStore.java
@@ -49,8 +49,6 @@ import org.slf4j.LoggerFactory;
import com.github.benmanes.caffeine.cache.Ticker;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
public class ZooPropStore implements PropStore, PropChangeListener {
private final static Logger log =
LoggerFactory.getLogger(ZooPropStore.class);
@@ -83,8 +81,6 @@ public class ZooPropStore implements PropStore,
PropChangeListener {
* @param watcher a watcher. Optional, if null, one is created.
* @param ticker a synthetic clock used for testing. Optional, if null, one
is created.
*/
- @SuppressFBWarnings(value = "PREDICTABLE_RANDOM",
- justification = "random number not used in secure context")
ZooPropStore(final InstanceId instanceId, final ZooReaderWriter zrw, final
ReadyMonitor monitor,
final PropStoreWatcher watcher, final Ticker ticker) {
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit e3d62043051d22f57b7bd71760bec52352eb160c
Merge: 4886e821ed 00bac7ae56
Author: Ed Coleman
AuthorDate: Wed Feb 21 22:17:04 2024 +
Merge remote-tracking branch 'upstream/2.1'
- merge FateIT into main
- includes minor quality check fixes
.../accumulo/test/fate/zookeeper/FateIT.java | 160 ++---
1 file changed, 139 insertions(+), 21 deletions(-)
diff --cc test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java
index d4da1ebae4,2dde8fabca..a392313bd0
--- a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/FateIT.java
@@@ -121,12 -177,17 +176,17 @@@ public class FateIT
private static ZooKeeperTestingServer szk = null;
private static ZooReaderWriter zk = null;
-- private static final String ZK_ROOT = "/accumulo/" +
UUID.randomUUID().toString();
++ private static final String ZK_ROOT = "/accumulo/" + UUID.randomUUID();
private static final NamespaceId NS = NamespaceId.of("testNameSpace");
private static final TableId TID = TableId.of("testTable");
private static CountDownLatch callStarted;
private static CountDownLatch finishCall;
+ private static CountDownLatch undoLatch;
+
+ private enum ExceptionLocation {
+ CALL, IS_READY
- };
++ }
@BeforeAll
public static void setup() throws Exception {
@@@ -148,9 -209,9 +208,8 @@@
@Timeout(30)
public void testTransactionStatus() throws Exception {
--final ZooStore zooStore = new ZooStore(ZK_ROOT +
Constants.ZFATE, zk);
--final AgeOffStore store =
--new AgeOffStore(zooStore, 3000, System::currentTimeMillis);
++final ZooStore zooStore = new ZooStore<>(ZK_ROOT +
Constants.ZFATE, zk);
++final AgeOffStore store = new AgeOffStore<>(zooStore, 3000,
System::currentTimeMillis);
Manager manager = createMock(Manager.class);
ServerContext sctx = createMock(ServerContext.class);
@@@ -159,15 -220,12 +218,12 @@@
expect(sctx.getZooReaderWriter()).andReturn(zk).anyTimes();
replay(manager, sctx);
-Fate fate = new Fate(manager, store,
TraceRepo::toLogString);
+ConfigurationCopy config = new ConfigurationCopy();
+config.set(Property.GENERAL_THREADPOOL_SIZE, "2");
+config.set(Property.MANAGER_FATE_THREADPOOL_SIZE, "1");
- Fate fate = new Fate(manager, store,
TraceRepo::toLogString, config);
++Fate fate = new Fate<>(manager, store, TraceRepo::toLogString,
config);
try {
- ConfigurationCopy config = new ConfigurationCopy();
- config.set(Property.GENERAL_THREADPOOL_SIZE, "2");
- config.set(Property.MANAGER_FATE_THREADPOOL_SIZE, "1");
- // Wait for the transaction runner to be scheduled.
- Thread.sleep(3000);
-
callStarted = new CountDownLatch(1);
finishCall = new CountDownLatch(1);
@@@ -175,6 -233,11 +231,10 @@@
assertEquals(TStatus.NEW, getTxStatus(zk, txid));
fate.seedTransaction("TestOperation", txid, new TestOperation(NS, TID),
true, "Test Op");
assertEquals(TStatus.SUBMITTED, getTxStatus(zk, txid));
+
- fate.startTransactionRunners(config);
+ // Wait for the transaction runner to be scheduled.
- UtilWaitThread.sleep(3000);
++ Thread.sleep(3000);
+
// wait for call() to be called
callStarted.await();
assertEquals(IN_PROGRESS, getTxStatus(zk, txid));
@@@ -208,9 -271,9 +268,8 @@@
@Test
public void testCancelWhileNew() throws Exception {
--final ZooStore zooStore = new ZooStore(ZK_ROOT +
Constants.ZFATE, zk);
--final AgeOffStore store =
--new AgeOffStore(zooStore, 3000, System::currentTimeMillis);
++final ZooStore zooStore = new ZooStore<>(ZK_ROOT +
Constants.ZFATE, zk);
++final AgeOffStore store = new AgeOffStore<>(zooStore, 3000,
System::currentTimeMillis);
Manager manager = createMock(Manager.class);
ServerContext sctx = createMock(ServerContext.class);
@@@ -219,14 -282,15 +278,13 @@@
expect(sctx.getZooReaderWriter()).andReturn(zk).anyTimes();
replay(manager, sctx);
-Fate fate = new Fate(manager, store,
TraceRepo::toLogString);
+ConfigurationCopy config = new ConfigurationCopy();
+config.set(Property.GENERAL_THREADPOOL_SIZE, "2");
+config.set(Property.MANAGER_FATE_THREADPOOL_SIZE, "1");
- Fate fate = new Fate(manager, store,
TraceRepo::toLogString, config);
++Fate fate = new Fate<>(manager, store, TraceRepo::toLogString,
config);
try {
- ConfigurationCopy config = new ConfigurationCopy();
- config.set(Proper
(accumulo) branch main updated (4886e821ed -> e3d6204305)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4886e821ed Remove unused var added by merge add 00bac7ae56 FateIT Repo Error Test (#4282) new e3d6204305 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/test/fate/zookeeper/FateIT.java | 160 ++--- 1 file changed, 139 insertions(+), 21 deletions(-)
(accumulo-website) branch main updated: Adding myself to contributor. PR apache/accumulo#4192 (#414)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new 8883c65c Adding myself to contributor. PR apache/accumulo#4192 (#414) 8883c65c is described below commit 8883c65ca400e2879ee3fb4758a8f6f7f63a39f4 Author: Arbaaz Khan AuthorDate: Thu Feb 8 15:34:13 2024 -0500 Adding myself to contributor. PR apache/accumulo#4192 (#414) --- pages/people.md | 1 + 1 file changed, 1 insertion(+) diff --git a/pages/people.md b/pages/people.md index a9a85d95..a8378bf4 100644 --- a/pages/people.md +++ b/pages/people.md @@ -83,6 +83,7 @@ GitHub also has a [contributor list][github-contributors] based on commits. | Ali Mustafa | [FAST-NU][FAST-NU] | [PKT][PKT]| | Amisha Sahu | | [IST][IST-India] | | Andrew George Wells | [ClearEdgeIT][CLEAREDGE] | [ET][ET] | +| Arbaaz Khan | [Arctic Slope Regional Corp.][ASRC] | [ET][ET] | | Arshak Navruzyan| [Argyle Data][ARGYLE] | | | Ben Kelly | [Microsoft][MICROSOFT] | [GMT][GMT]/[IST][IST-Ireland] | | Ben Manes | | [PT][PT] |
(accumulo) branch main updated: Update export version to handle imports of files without fenced ranges (#4122)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1db892b00d Update export version to handle imports of files without
fenced ranges (#4122)
1db892b00d is described below
commit 1db892b00dfca083882ef8f90d3baa43dbb59181
Author: EdColeman
AuthorDate: Wed Jan 31 17:58:54 2024 -0500
Update export version to handle imports of files without fenced ranges
(#4122)
Update export version to handle imports of files without fenced ranges
- adds specific export version
- adds import test with 2.1.3 create files (no fenced ranges)
Co-authored-by: Christopher Tubbs
---
core/pom.xml | 2 -
.../accumulo/core/metadata/StoredTabletFile.java | 15
.../core/metadata/StoredTabletFileTest.java| 40 +++
pom.xml| 2 +
.../manager/tableOps/tableExport/ExportTable.java | 10 ++-
.../tableOps/tableExport/WriteExportFiles.java | 2 +-
.../manager/tableOps/tableImport/ImportTable.java | 8 +--
.../tableOps/tableImport/ImportedTableInfo.java| 1 +
.../tableImport/PopulateMetadataTable.java | 16 -
.../accumulo/manager/upgrade/Upgrader11to12.java | 20 ++
.../manager/upgrade/Upgrader11to12Test.java| 25 ++-
test/pom.xml | 17 +
.../org/apache/accumulo/test/ImportExportIT.java | 53 ++
test/src/main/resources/v2_import_test/README.md | 78 +
.../main/resources/v2_import_test/data/A008.rf | Bin 0 -> 229 bytes
.../main/resources/v2_import_test/data/A009.rf | Bin 0 -> 229 bytes
.../main/resources/v2_import_test/data/A00a.rf | Bin 0 -> 228 bytes
.../main/resources/v2_import_test/data/A00b.rf | Bin 0 -> 212 bytes
.../main/resources/v2_import_test/data/distcp.txt | 5 ++
.../v2_import_test/data/exportMetadata.zip | Bin 0 -> 934 bytes
20 files changed, 246 insertions(+), 48 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 3b1ec42ea6..05d9a02c69 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -193,7 +193,6 @@
src/main/java/org/apache/accumulo/core/bloomfilter/*.java
src/main/java/org/apache/accumulo/core/util/HostAndPort.java
src/test/resources/*.jceks
-
src/test/resources/org/apache/accumulo/core/file/rfile/*.rf
@@ -206,7 +205,6 @@
src/main/java/org/apache/accumulo/core/bloomfilter/*.java
src/test/resources/*.jceks
-
src/test/resources/org/apache/accumulo/core/file/rfile/*.rf
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index daae673e75..16bb0458a9 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@ -33,7 +33,9 @@ import
org.apache.accumulo.core.util.json.ByteArrayToBase64TypeAdapter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Text;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
/**
@@ -263,6 +265,19 @@ public class StoredTabletFile extends
AbstractTabletFile {
}
}
+ /**
+ * Quick validation to see if value has been converted by checking if the
candidate looks like
+ * json by checking the candidate starts with "{" and ends with "}".
+ *
+ * @param candidate a possible file: reference.
+ * @return false if a likely a json object, true if not a likely json object
+ */
+ @VisibleForTesting
+ public static boolean fileNeedsConversion(@NonNull final String candidate) {
+String trimmed = candidate.trim();
+return !trimmed.startsWith("{") || !trimmed.endsWith("}");
+ }
+
private static class TabletFileCq {
public final Path path;
public final Range range;
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
new file mode 100644
index 00..998c9eab11
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright
(accumulo) branch main updated (bce2ef2bfc -> 9911fc1576)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from bce2ef2bfc Merge branch '2.1' add b90422cb7e set max files to fix ExternalCompaction4_IT test (#4205) add 9911fc1576 Merge remote-tracking branch 'upstream/2.1' No new revisions were added by this update. Summary of changes: .../accumulo/test/compaction/ExternalCompaction4_IT.java | 11 +++ 1 file changed, 7 insertions(+), 4 deletions(-)
(accumulo) branch 2.1 updated (1531fe5d7e -> b90422cb7e)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1531fe5d7e Fix intermittent failure in AuthenticationTokenSecretManagerTest (#4203) add b90422cb7e set max files to fix ExternalCompaction4_IT test (#4205) No new revisions were added by this update. Summary of changes: .../test/compaction/ExternalCompaction4_IT.java| 41 +- 1 file changed, 25 insertions(+), 16 deletions(-)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 97f9cf80cbc89fefb46aa96ae1cec56af7aeff02
Merge: b701918ec0 e69456534b
Author: Ed Coleman
AuthorDate: Fri Jan 26 19:55:38 2024 +
Merge remote-tracking branch 'upstream/2.1'
.../accumulo/tserver/tablet/CompactableImpl.java | 20 ++--
1 file changed, 10 insertions(+), 10 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 28d6b11977,89c68c5b55..4d29f953eb
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -101,13 -106,17 +101,13 @@@ public class CompactableImpl implement
private final FileManager fileMgr;
- private Set runningJobs = new HashSet<>();
+ private final Set runningJobs = new HashSet<>();
private volatile boolean compactionRunning = false;
- private Supplier> servicesInUse;
+ private final Supplier> servicesInUse;
- private Set servicesUsed = new
ConcurrentSkipListSet<>();
+ private final Set servicesUsed = new
ConcurrentSkipListSet<>();
- enum ChopSelectionStatus {
-SELECTING, SELECTED, NOT_ACTIVE, MARKING
- }
-
// status of special compactions
enum FileSelectionStatus {
NEW, SELECTING, SELECTED, RESERVED, NOT_ACTIVE, CANCELED
(accumulo) branch main updated (b701918ec0 -> 97f9cf80cb)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b701918ec0 Trivial - give variable name to test constant add e69456534b CompactableImpl sync fix and message update (#4198) new 97f9cf80cb Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/tserver/tablet/CompactableImpl.java | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-)
(accumulo) branch 2.1 updated: CompactableImpl sync fix and message update (#4198)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e69456534b CompactableImpl sync fix and message update (#4198)
e69456534b is described below
commit e69456534b12f33178d6372f4905a32f023217f6
Author: EdColeman
AuthorDate: Fri Jan 26 14:30:12 2024 -0500
CompactableImpl sync fix and message update (#4198)
* remove unnecessary sync, clarify error messages
---
.../accumulo/tserver/tablet/CompactableImpl.java | 34 +++---
1 file changed, 17 insertions(+), 17 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 73a2efa0e4..89c68c5b55 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -106,12 +106,12 @@ public class CompactableImpl implements Compactable {
private final FileManager fileMgr;
- private Set runningJobs = new HashSet<>();
+ private final Set runningJobs = new HashSet<>();
private volatile boolean compactionRunning = false;
- private Supplier> servicesInUse;
+ private final Supplier> servicesInUse;
- private Set servicesUsed = new
ConcurrentSkipListSet<>();
+ private final Set servicesUsed = new
ConcurrentSkipListSet<>();
enum ChopSelectionStatus {
SELECTING, SELECTED, NOT_ACTIVE, MARKING
@@ -126,21 +126,21 @@ public class CompactableImpl implements Compactable {
private Long compactionId;
private CompactionConfig compactionConfig;
- private CompactionManager manager;
+ private final CompactionManager manager;
AtomicLong lastSeenCompactionCancelId = new AtomicLong(Long.MIN_VALUE);
private volatile boolean closed = false;
- private Map externalCompactions
=
+ private final Map
externalCompactions =
new ConcurrentHashMap<>();
- private Set externalCompactionsCommitting = new
HashSet<>();
+ private final Set externalCompactionsCommitting = new
HashSet<>();
// This interface exists for two purposes. First it allows abstraction of
new and old
// implementations for user pluggable file selection code. Second it
facilitates placing code
// outside of this class.
- public static interface CompactionHelper {
+ public interface CompactionHelper {
Set
selectFiles(SortedMap allFiles);
Set getFilesToDrop();
@@ -195,17 +195,17 @@ public class CompactableImpl implements Compactable {
// important to track this in order to know if the last compaction is a
full compaction and
// should not propagate deletes.
private boolean initiallySelectedAll = false;
-private Set selectedFiles = new HashSet<>();
+private final Set selectedFiles = new HashSet<>();
protected Set allCompactingFiles = new HashSet<>();
// track files produced by compactions of this tablet, those are
considered chopped
-private Set choppedFiles = new HashSet<>();
+private final Set choppedFiles = new HashSet<>();
private ChopSelectionStatus chopStatus = ChopSelectionStatus.NOT_ACTIVE;
-private Set allFilesWhenChopStarted = new HashSet<>();
+private final Set allFilesWhenChopStarted = new
HashSet<>();
private final KeyExtent extent;
-private Deriver selectionExpirationDeriver;
+private final Deriver selectionExpirationDeriver;
public FileManager(KeyExtent extent, Collection
extCompactingFiles,
Optional extSelInfo, Deriver
selectionExpirationDeriver) {
@@ -314,8 +314,8 @@ public class CompactableImpl implements Compactable {
}
class ChopSelector {
- private Set allFiles;
- private Set filesToExamine;
+ private final Set allFiles;
+ private final Set filesToExamine;
private ChopSelector(Set allFiles,
Set filesToExamine) {
this.allFiles = allFiles;
@@ -1485,7 +1485,7 @@ public class CompactableImpl implements Compactable {
if (dispatcher == null) {
log.error(
-"Failed to dispatch compaction {} kind:{} hints:{}, falling back
to {} service. Unable to instantiate dispatcher plugin. Check server log.",
+"Failed to dispatch compaction, no dispatcher. extent:{} kind:{}
hints:{}, falling back to {} service. Unable to instantiate dispatcher plugin.
Check server log.",
getExtent(), kind, debugHints,
CompactionServicesConfig.DEFAULT_SERVICE);
return CompactionServicesConfig.DEFAULT_SERVICE;
}
@@ -1532,7 +1532,8 @@ public class CompactableImpl implements Compactable {
return dispatch.getService();
} catc
