subject:"\(accumulo\) 01\/01\: Merge remote\-tracking branch 'upstream\/main' into elasticity"

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-08-16 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f5396dfa85b3e127c132a549cd98c53dfaca6b05
Merge: 00cb09f957 eda39bb1c1
Author: Dom Garguilo 
AuthorDate: Fri Aug 16 17:19:54 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  | 28 +++-
 .../gc/GarbageCollectWriteAheadLogsTest.java   | 51 +++---
 2 files changed, 34 insertions(+), 45 deletions(-)

diff --cc 
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 79cbe9535a,c636c34c4b..4d2f1b4dc6
--- 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -72,7 -65,8 +72,7 @@@ public class GarbageCollectWriteAheadLo
private final VolumeManager fs;
private final LiveTServerSet liveServers;
private final WalStateManager walMarker;
-   private final AtomicBoolean hasCollected;
+   private final AtomicBoolean hasCollected = new AtomicBoolean(false);
 -  private final Stream store;
  
/**
 * Creates a new GC WAL object.
@@@ -82,26 -76,44 +82,34 @@@
 */
GarbageCollectWriteAheadLogs(final ServerContext context, final 
VolumeManager fs,
final LiveTServerSet liveServers) {
 -this(context, fs, liveServers, new WalStateManager(context), 
createStore(context));
++this(context, fs, liveServers, new WalStateManager(context));
+   }
+ 
+   /**
+* Creates a new GC WAL object. Meant for testing -- allows for mocked 
objects.
+*
+*
+* @param context the collection server's context
+* @param fs volume manager to use
+* @param liveServers a started LiveTServerSet instance
+* @param walMarker a WalStateManager instance
 -   * @param store a stream of TabletLocationState objects
+*/
+   GarbageCollectWriteAheadLogs(final ServerContext context, final 
VolumeManager fs,
 -  final LiveTServerSet liveServers, final WalStateManager walMarker,
 -  final Stream store) {
++  final LiveTServerSet liveServers, final WalStateManager walMarker) {
  this.context = context;
  this.fs = fs;
  this.liveServers = liveServers;
- this.walMarker = createWalStateManager(context);
- this.hasCollected = new AtomicBoolean(false);
-   }
- 
-   @VisibleForTesting
-   WalStateManager createWalStateManager(ServerContext context) {
- return new WalStateManager(context);
+ this.walMarker = walMarker;
 -this.store = store;
}
  
 -  private static Stream createStore(final ServerContext 
context) {
 -var rootStream = TabletStateStore.getStoreForLevel(DataLevel.ROOT, 
context).stream();
 -var metadataStream = 
TabletStateStore.getStoreForLevel(DataLevel.METADATA, context).stream();
 -var userStream = TabletStateStore.getStoreForLevel(DataLevel.USER, 
context).stream();
 -return Streams.concat(rootStream, metadataStream, userStream).onClose(() 
-> {
 -  try {
 -rootStream.close();
 -  } finally {
 -try {
 -  metadataStream.close();
 -} finally {
 -  userStream.close();
 -}
 -  }
 -});
 +  @VisibleForTesting
 +  Stream createStore(Set liveTServers) {
 +GcWalsFilter walsFilter = new GcWalsFilter(liveTServers);
 +
 +return Stream.of(DataLevel.ROOT, DataLevel.METADATA, DataLevel.USER)
 +.map(dataLevel -> 
context.getAmple().readTablets().forLevel(dataLevel).filter(walsFilter)
 +.fetch(LOCATION, LAST, LOGS, PREV_ROW, SUSPEND).build())
 +.flatMap(TabletsMetadata::stream);
}
  
public void collect(GCStatus status) {
diff --cc 
server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
index 7f4ec5173c,13c264a812..f868fd26d1
--- 
a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
+++ 
b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
@@@ -100,26 -94,14 +100,21 @@@ public class GarbageCollectWriteAheadLo
  marker.removeWalMarker(server1, id);
  EasyMock.expectLastCall().once();
  EasyMock.replay(context, fs, marker, tserverSet);
- GarbageCollectWriteAheadLogs gc = new 
GarbageCollectWriteAheadLogs(context, fs, tserverSet) {
 -var gc =
 -new GarbageCollectWriteAheadLogs(context, fs, tserverSet, marker, 
tabletOnServer1List) {
 -  @Override
 -  protected Map getSortedWALogs() {
 -return Collections.emptyMap();
 -  }
 -};
++var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, 
marker) {
 +  @Override
 +  protected Map getSortedWALogs() {
 +return Collections.emptyMap();
 +  }
 +
-   @Override
-   WalStateManager createWalStateManager(ServerContext serverContext) {
- return

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-08-16 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 00cb09f9572dbc418149eaad34f4f380251b8250
Merge: d8185cdea7 003cdf333d
Author: Dom Garguilo 
AuthorDate: Fri Aug 16 14:47:07 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../core/client/lexicoder/BytesLexicoderTest.java  |   4 +-
 .../core/client/lexicoder/LexicoderTest.java   |   5 +-
 .../core/clientImpl/ClientTabletCacheImplTest.java |   5 +-
 .../core/clientImpl/lexicoder/ByteUtilsTest.java   |  15 +--
 .../accumulo/core/conf/ClientPropertyTest.java |   7 +-
 .../apache/accumulo/core/crypto/CryptoTest.java|   5 +-
 .../org/apache/accumulo/core/data/ColumnTest.java  |   7 +-
 .../org/apache/accumulo/core/data/KeyTest.java |  38 
 .../apache/accumulo/core/data/MutationTest.java| 101 +++--
 .../org/apache/accumulo/core/data/RangeTest.java   |  12 +--
 .../zookeeper/DistributedReadWriteLockTest.java|   3 +-
 .../core/file/rfile/CreateCompatTestFile.java  |   5 +-
 .../core/file/rfile/GenerateSplitsTest.java|   3 +-
 .../accumulo/core/file/rfile/KeyShortenerTest.java |  11 ++-
 .../core/file/rfile/MultiThreadedRFileTest.java|   4 +-
 .../apache/accumulo/core/file/rfile/RFileTest.java |   6 +-
 .../core/iterators/system/ColumnFilterTest.java|   5 +-
 .../iterators/system/VisibilityFilterTest.java |   4 +-
 .../accumulo/core/iterators/user/FilterTest.java   |  23 ++---
 .../iterators/user/IndexedDocIteratorTest.java |  17 ++--
 .../core/iterators/user/RegExFilterTest.java   |   5 +-
 .../iterators/user/RowDeletingIteratorTest.java|   5 +-
 .../iterators/user/VersioningIteratorTest.java |   3 +-
 .../accumulo/core/security/AuthorizationsTest.java |   3 +-
 .../core/security/ColumnVisibilityTest.java|   8 +-
 .../balancer/HostRegexTableLoadBalancerTest.java   |   8 +-
 .../hadoop/its/mapred/AccumuloInputFormatIT.java   |   5 +-
 .../hadoop/its/mapred/AccumuloOutputFormatIT.java  |   9 +-
 .../its/mapred/AccumuloRowInputFormatIT.java   |  11 ++-
 .../hadoop/its/mapred/MultiTableInputFormatIT.java |   5 +-
 .../accumulo/hadoop/its/mapred/TokenFileIT.java|   9 +-
 .../its/mapreduce/AccumuloInputFormatIT.java   |   5 +-
 .../its/mapreduce/AccumuloOutputFormatIT.java  |   9 +-
 .../its/mapreduce/AccumuloRowInputFormatIT.java|  11 ++-
 .../its/mapreduce/MultiTableInputFormatIT.java |   5 +-
 .../accumulo/hadoop/its/mapreduce/RowHashIT.java   |   5 +-
 .../accumulo/hadoop/its/mapreduce/TokenFileIT.java |   9 +-
 .../miniclusterImpl/MiniAccumuloClusterImpl.java   |   2 +-
 .../accumulo/server/data/ServerMutationTest.java   |  21 +++--
 .../security/handler/ZKAuthenticatorTest.java  |   2 +-
 .../org/apache/accumulo/monitor/it/WebViewsIT.java |   3 +-
 .../accumulo/tserver/CheckTabletMetadataTest.java  |   5 +-
 .../accumulo/tserver/TservConstraintEnvTest.java   |   3 +-
 .../shell/commands/HistoryCommandTest.java |   4 +-
 .../apache/accumulo/test/ConditionalWriterIT.java  |   8 +-
 .../org/apache/accumulo/test/InMemoryMapIT.java|   3 +-
 .../apache/accumulo/test/MultiTableRecoveryIT.java |   3 +-
 .../org/apache/accumulo/test/VerifyIngest.java |   6 +-
 .../accumulo/test/functional/AuthsIterator.java|   4 +-
 .../accumulo/test/functional/CombinerIT.java   |   3 +-
 .../accumulo/test/functional/KerberosIT.java   |   3 +-
 .../test/functional/WatchTheWatchCountIT.java  |   5 +-
 .../accumulo/test/metrics/TestStatsDSink.java  |   4 +-
 53 files changed, 266 insertions(+), 208 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-08-09 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit cbc2c16b8966a5da765f9619c00eae630e71ff58
Merge: d1456fcaa4 f1dc4dc05c
Author: Dom Garguilo 
AuthorDate: Fri Aug 9 11:43:32 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../java/org/apache/accumulo/core/util/Timer.java  | 25 
 .../org/apache/accumulo/core/util/TimerTest.java   | 34 ++
 2 files changed, 59 insertions(+)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-07-23 Thread ddanielr

This is an automated email from the ASF dual-hosted git repository.

ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit b674b62dd9380f1dd32e9a93f97cbaefa4a049de
Merge: 0485e42f34 bab64ca7cf
Author: Daniel Roberts 
AuthorDate: Tue Jul 23 20:39:59 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../manager/ManagerClientServiceHandler.java   |  7 +++-
 .../org/apache/accumulo/test/shell/ShellIT.java| 49 ++
 2 files changed, 54 insertions(+), 2 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-07-09 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f33ee9dc3c669acdd4e5b3a3b7dcd6cc682a8f12
Merge: d0b72f5465 5ab0bd8273
Author: Dom Garguilo 
AuthorDate: Tue Jul 9 16:58:25 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../org/apache/accumulo/core/conf/Property.java|  4 +-
 .../accumulo/core/metrics/MetricsProducer.java | 16 +++
 .../spi/balancer/HostRegexTableLoadBalancer.java   |  3 +-
 .../org/apache/accumulo/server/AbstractServer.java | 22 ++
 .../accumulo/server/metrics/ProcessMetrics.java| 18 
 .../org/apache/accumulo/compactor/Compactor.java   |  8 
 .../org/apache/accumulo/tserver/ScanServer.java|  6 +--
 .../compaction/ExternalCompactionProgressIT.java   | 29 ++---
 .../test/functional/IdleProcessMetricsIT.java  | 50 ++
 .../apache/accumulo/test/metrics/MetricsIT.java|  3 +-
 10 files changed, 71 insertions(+), 88 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 933c651297,ee6eb6e891..08d8bc6514
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -652,20 -594,8 +651,19 @@@ public interface MetricsProducer 
  
String METRICS_COMPACTOR_PREFIX = "accumulo.compactor.";
String METRICS_COMPACTOR_MAJC_STUCK = METRICS_COMPACTOR_PREFIX + 
"majc.stuck";
 +  String METRICS_COMPACTOR_QUEUE_PREFIX = METRICS_COMPACTOR_PREFIX + "queue.";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUES = 
METRICS_COMPACTOR_QUEUE_PREFIX + "count";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH = 
METRICS_COMPACTOR_QUEUE_PREFIX + "length";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.dequeued";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.queued";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.rejected";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.priority";
String METRICS_COMPACTOR_ENTRIES_READ = METRICS_COMPACTOR_PREFIX + 
"entries.read";
String METRICS_COMPACTOR_ENTRIES_WRITTEN = METRICS_COMPACTOR_PREFIX + 
"entries.written";
-   String METRICS_COMPACTOR_BUSY = METRICS_COMPACTOR_PREFIX + "busy";
  
String METRICS_FATE_PREFIX = "accumulo.fate.";
String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX + 
"ops.in.progress.by.type";
diff --cc 
server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java
index 17e8e42197,69de547e0b..e110e4d5a8
--- 
a/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java
@@@ -18,6 -18,9 +18,8 @@@
   */
  package org.apache.accumulo.server.metrics;
  
 -import java.util.List;
+ import java.util.concurrent.atomic.AtomicInteger;
+ 
  import org.apache.accumulo.core.metrics.MetricsProducer;
  import org.apache.accumulo.server.ServerContext;
  
@@@ -35,14 -38,8 +37,8 @@@ public class ProcessMetrics implements 
  
@Override
public void registerMetrics(MeterRegistry registry) {
 -registry.gauge(METRICS_LOW_MEMORY, List.of(), this, this::lowMemDetected);
 +registry.gauge(METRICS_LOW_MEMORY, this, this::lowMemDetected);
- idleCounter = registry.counter(METRICS_SERVER_IDLE);
-   }
- 
-   public void incrementIdleCounter() {
- if (idleCounter != null) {
-   idleCounter.increment();
- }
+ registry.gauge(METRICS_SERVER_IDLE, isIdle, AtomicInteger::get);
}
  
private int lowMemDetected(ProcessMetrics processMetrics) {
diff --cc 
test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java
index 72c0fbe560,fb8e2d4ccc..f76b816d73
--- 
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java
+++ 
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java
@@@ -35,9 -36,6 +35,8 @@@ import java.util.EnumSet
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
- import java.util.Objects;
 +import java.util.Optional;
 +import java.util.concurrent.ConcurrentHashMap;
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicBoolean;
  import java.util.concurrent.atomic.AtomicLong;
@@@ -146,12 -216,10 +145,11 @@@ public class ExternalCompactionProgress
  
  final AtomicLong totalEntriesRead = new AtomicLong(0);
  final AtomicLong totalEntriesWritten = new AtomicLong(0);
 -final long expectedEntriesRead = 9216;
 -final long expectedEntriesWritten = 4096;
 +final ConcurrentHashMap compactorBusy =

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-06-24 Thread jmark99

This is an automated email from the ASF dual-hosted git repository.

jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit d8cb61a73f5767ba5fd0a22f3c9eb55cf4a73d6d
Merge: 46d8c0b71d c6e31e2cc2
Author: Mark Owens 
AuthorDate: Mon Jun 24 17:33:44 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/core/data/ConditionalMutation.java| 46 +++
 .../core/data/ConditionalMutationTest.java | 92 ++
 2 files changed, 138 insertions(+)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-06-17 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 8c99aff2d672e3fe0120ca4cfcae0058ec2e7dad
Merge: 063d12057a b63afacfa6
Author: Dom Garguilo 
AuthorDate: Mon Jun 17 15:01:46 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../util/compaction/RunningCompactionInfo.java |  21 ++---
 .../compaction/thrift/TCompactionStatusUpdate.java | 104 -
 core/src/main/thrift/compaction-coordinator.thrift |   1 +
 .../core/metadata/schema/TabletMetadataTest.java   |  27 ++
 .../accumulo/server/compaction/CompactionInfo.java |   3 +-
 .../accumulo/server/compaction/FileCompactor.java  |  17 +++-
 .../org/apache/accumulo/compactor/Compactor.java   |  37 ++--
 .../shell/commands/ActiveCompactionHelper.java |  30 +++---
 .../shell/commands/ListCompactionsCommand.java |   2 +-
 .../compaction/ExternalCompactionProgressIT.java   |  10 +-
 .../compaction/ExternalDoNothingCompactor.java |   6 ++
 11 files changed, 203 insertions(+), 55 deletions(-)

diff --cc 
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index d22925afc3,586f697ae8..9f4ba14def
--- 
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++ 
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -22,20 -22,18 +22,21 @@@ import static java.util.stream.Collecto
  import static org.apache.accumulo.core.metadata.StoredTabletFile.serialize;
  import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_COLUMN;
  import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_VALUE;
 -import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
  import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
  import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
 +import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_NONCE_COLUMN;
  import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
  import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN;
 -import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN;
 -import static 
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN;
 +import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
+ import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
 +import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
 +import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED;
  import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
  import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
 -import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
 +import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED;
  import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
 +import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.UNSPLITTABLE;
 +import static 
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
  import static org.junit.jupiter.api.Assertions.assertEquals;
  import static org.junit.jupiter.api.Assertions.assertFalse;
  import static org.junit.jupiter.api.Assertions.assertNull;
@@@ -431,144 -409,38 +432,170 @@@ public class TabletMetadataTest 
  assertEquals(1, tm2.getScans().size());
  assertThrows(UnsupportedOperationException.class, () -> 
tm2.getScans().add(stf));
  assertEquals(1, tm2.getLoaded().size());
 -assertThrows(UnsupportedOperationException.class, () -> 
tm2.getLoaded().put(stf, 0L));
 +assertThrows(UnsupportedOperationException.class,
 +() -> tm2.getLoaded().put(stf, FateId.from(FateInstanceType.USER, 
UUID.randomUUID(;
  assertEquals(1, tm2.getKeyValues().size());
  assertThrows(UnsupportedOperationException.class, () -> 
tm2.getKeyValues().remove(null));
 +assertEquals(1, tm2.getCompacted().size());
 +assertThrows(UnsupportedOperationException.class,
 +() -> tm2.getCompacted().add(FateId.from(FateInstanceType.USER, 
UUID.randomUUID(;
 +assertEquals(1, tm2.getUserCompactionsRequested(

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-06-08 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit e9a4d86e6c94f4a7627ae66269834a9e956280a0
Merge: b0af344d9a 5256e3462d
Author: Keith Turner 
AuthorDate: Sat Jun 8 15:15:25 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../java/org/apache/accumulo/test/PrintInfoIT.java | 34 +-
 1 file changed, 13 insertions(+), 21 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-06-06 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 69a62fa55007dd6f9f6181f6b9a5f855f91e6937
Merge: f4a857e1fa 000855d4c1
Author: Dom Garguilo 
AuthorDate: Thu Jun 6 12:36:02 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

# Conflicts:
#   
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
#   
test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java

 .../accumulo/core/metrics/MetricsProducer.java | 27 +++-
 .../org/apache/accumulo/compactor/Compactor.java   |  8 +++
 .../compaction/ExternalCompactionProgressIT.java   | 81 +-
 3 files changed, 84 insertions(+), 32 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 8e0c0a8d58,0f8e607ee6..933c651297
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -61,46 -62,27 +62,69 @@@ import io.micrometer.core.instrument.Me
   * 
   * N/A
   * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUES}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED}
 + * Gauge
 + * 
 + * 
 + * 
++ * N/A
++ * N/A
+  * {@value #METRICS_COMPACTOR_ENTRIES_READ}
+  * FunctionCounter
+  * Number of entries read by all threads performing compactions
+  * 
+  * 
+  * N/A
+  * N/A
+  * {@value #METRICS_COMPACTOR_ENTRIES_WRITTEN}
+  * FunctionCounter
+  * Number of entries written by all threads performing compactions
+  * 
+  * 
+  * N/A
+  * N/A
+  * {@value #METRICS_COMPACTOR_BUSY}
+  * Gauge
+  * Indicates if the compactor is busy or not. The value will be 0 when 
idle and 1 when
+  * busy.
+  * 
+  * 
+  * 
   * currentFateOps
   * Gauge
   * {@value #METRICS_FATE_OPS}
@@@ -624,23 -591,11 +648,24 @@@ public interface MetricsProducer 
Logger LOG = LoggerFactory.getLogger(MetricsProducer.class);
  
String METRICS_LOW_MEMORY = "accumulo.detected.low.memory";
 +  String METRICS_SERVER_IDLE = "accumulo.server.idle";
 +
String METRICS_COMPACTOR_PREFIX = "accumulo.compactor.";
String METRICS_COMPACTOR_MAJC_STUCK = METRICS_COMPACTOR_PREFIX + 
"majc.stuck";
 +  String METRICS_COMPACTOR_QUEUE_PREFIX = METRICS_COMPACTOR_PREFIX + "queue.";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUES = 
METRICS_COMPACTOR_QUEUE_PREFIX + "count";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH = 
METRICS_COMPACTOR_QUEUE_PREFIX + "length";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.dequeued";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.queued";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.rejected";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.priority";
String METRICS_COMPACTOR_ENTRIES_READ = METRICS_COMPACTOR_PREFIX + 
"entries.read";
String METRICS_COMPACTOR_ENTRIES_WRITTEN = METRICS_COMPACTOR_PREFIX + 
"entries.written";
+   String METRICS_COMPACTOR_BUSY = METRICS_COMPACTOR_PREFIX + "busy";
  
String METRICS_FATE_PREFIX = "accumulo.fate.";
String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX + 
"ops.in.progress.by.type";
diff --cc 
test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java
index b9b41e0833,a298ae22cd..f7fefbc1f4
--- 
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java
+++ 
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionProgressIT.java
@@@ -34,11 -34,13 +34,12 @@@ import java.util.EnumSet
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
 +import java.util.Optional;
  import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.concurrent.atomic.AtomicInteger;
  import java.util.concurrent.atomic.AtomicLong;
  
 -import org.apache.accumulo.compactor.Compactor;
 -import org.apache.accumulo.coordinator.CompactionCoordinator;
  import org.apache.accumulo.core.client.Accumulo;
  import org.apache.accumulo.core.client.AccumuloClient;
  import org.apache.accumulo.core.client.IteratorSetting;
@@@ -125,17 -123,23

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-06-05 Thread jmark99

This is an automated email from the ASF dual-hosted git repository.

jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 93aac1a800c889a060bc5ce4dad41aecc16145c4
Merge: 01fa10be63 9ec0087069
Author: Mark Owens 
AuthorDate: Wed Jun 5 15:08:41 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../apache/accumulo/core/file/rfile/PrintInfo.java |   2 +-
 .../accumulo/core/summary/SummaryReader.java   |  11 +-
 .../java/org/apache/accumulo/test/PrintInfoIT.java | 202 +
 .../resources/org/apache/accumulo/test/ver_7.rf| Bin 0 -> 14557 bytes
 4 files changed, 213 insertions(+), 2 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-30 Thread domgarguilo

This is an automated email from the ASF dual-hosted git repository.

domgarguilo pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 32bbcbf351da18f590ce99dda64f305e904f1d5e
Merge: 3fcab5e18a 627ed8
Author: Dom Garguilo 
AuthorDate: Thu May 30 11:45:20 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/core/metrics/MetricsProducer.java |   2 +
 .../accumulo/server/compaction/FileCompactor.java  |  88 +--
 .../org/apache/accumulo/compactor/Compactor.java   |  27 -
 .../tserver/metrics/TabletServerMetrics.java   |  24 
 .../compaction/ExternalCompactionProgressIT.java   | 123 -
 5 files changed, 244 insertions(+), 20 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 0ed24a24b4,ca3d90ccf1..8e0c0a8d58
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -624,21 -567,10 +624,23 @@@ public interface MetricsProducer 
Logger LOG = LoggerFactory.getLogger(MetricsProducer.class);
  
String METRICS_LOW_MEMORY = "accumulo.detected.low.memory";
 +  String METRICS_SERVER_IDLE = "accumulo.server.idle";
 +
String METRICS_COMPACTOR_PREFIX = "accumulo.compactor.";
String METRICS_COMPACTOR_MAJC_STUCK = METRICS_COMPACTOR_PREFIX + 
"majc.stuck";
 +  String METRICS_COMPACTOR_QUEUE_PREFIX = METRICS_COMPACTOR_PREFIX + "queue.";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUES = 
METRICS_COMPACTOR_QUEUE_PREFIX + "count";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH = 
METRICS_COMPACTOR_QUEUE_PREFIX + "length";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.dequeued";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.queued";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.rejected";
 +  String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY =
 +  METRICS_COMPACTOR_QUEUE_PREFIX + "jobs.priority";
+   String METRICS_COMPACTOR_ENTRIES_READ = METRICS_COMPACTOR_PREFIX + 
"entries.read";
+   String METRICS_COMPACTOR_ENTRIES_WRITTEN = METRICS_COMPACTOR_PREFIX + 
"entries.written";
  
String METRICS_FATE_PREFIX = "accumulo.fate.";
String METRICS_FATE_TYPE_IN_PROGRESS = METRICS_FATE_PREFIX + 
"ops.in.progress.by.type";
diff --cc 
server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
index 41ff9f5f51,cebdce684d..4d47b63053
--- 
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
+++ 
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
@@@ -63,6 -65,8 +65,7 @@@ import org.apache.accumulo.core.tablets
  import org.apache.accumulo.core.trace.TraceUtil;
  import org.apache.accumulo.core.util.LocalityGroupUtil;
  import 
org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
 -import org.apache.accumulo.core.util.ratelimit.RateLimiter;
+ import org.apache.accumulo.core.util.time.NanoTime;
  import org.apache.accumulo.server.ServerContext;
  import org.apache.accumulo.server.fs.VolumeManager;
  import org.apache.accumulo.server.iterators.SystemIteratorEnvironment;
diff --cc 
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 2922a3999e,b44fcbbca1..4c5357e35a
--- 
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++ 
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -149,13 -150,17 +150,21 @@@ public class Compactor extends Abstract
  
protected Compactor(ConfigOpts opts, String[] args) {
  super("compactor", opts, args);
 -queueName = super.getConfiguration().get(Property.COMPACTOR_QUEUE_NAME);
 +  }
 +
 +  @Override
 +  protected String getResourceGroupPropertyValue(SiteConfiguration conf) {
 +return conf.get(Property.COMPACTOR_GROUP_NAME);
}
  
+   private long getTotalEntriesRead() {
+ return FileCompactor.getTotalEntriesRead();
+   }
+ 
+   private long getTotalEntriesWritten() {
+ return FileCompactor.getTotalEntriesWritten();
+   }
+ 
@Override
public void registerMetrics(MeterRegistry registry) {
  super.registerMetrics(registry);
diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
index 30c8594f6c,70b0c4980b..847c39f572
--- 
a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
+++ 
b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetrics.java
@@@ -32,9 -36,28 +36,29 @@@ public class TabletServerMetrics implem
  util = new TabletServerMetricsUtil(tserver);
}
  
+   private long getTotalEntriesRead() {
+ return FileCompactor.getTotalEnt

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-24 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit f9dabfc2501b893695bb04abf7d387f696851b98
Merge: 14efea2e75 4a5d8da7a5
Author: Ed Coleman 
AuthorDate: Fri May 24 15:23:38 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/core/metrics/MetricsProducer.java |  8 +++--
 .../org/apache/accumulo/tserver/ScanServer.java| 15 +
 .../apache/accumulo/tserver/ScanServerMetrics.java | 38 ++
 .../apache/accumulo/test/metrics/MetricsIT.java|  3 +-
 4 files changed, 47 insertions(+), 17 deletions(-)

diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index f9aed7e2cf,616800168e..323117c0c3
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@@ -90,8 -89,8 +90,9 @@@ import org.apache.accumulo.core.tablets
  import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
  import org.apache.accumulo.core.util.Halt;
  import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.util.cache.Caches.CacheName;
  import org.apache.accumulo.core.util.threads.ThreadPools;
+ import org.apache.accumulo.core.util.time.NanoTime;
  import org.apache.accumulo.server.AbstractServer;
  import org.apache.accumulo.server.ServerContext;
  import org.apache.accumulo.server.client.ClientServiceHandler;
diff --cc test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
index 53f981,0b76970d6d..402ed1ff50
--- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
@@@ -103,13 -96,13 +103,14 @@@ public class MetricsIT extends Configur
@Test
public void confirmMetricsPublished() throws Exception {
  
 -doWorkToGenerateMetrics();
 -cluster.stop();
 -
  Set unexpectedMetrics = Set.of(METRICS_COMPACTOR_MAJC_STUCK, 
METRICS_SCAN_YIELDS);
 -Set flakyMetrics = Set.of(METRICS_FATE_TYPE_IN_PROGRESS, 
METRICS_GC_WAL_ERRORS,
 -METRICS_SCAN_RESERVATION_TOTAL_TIMER, 
METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
 -METRICS_SCAN_TABLET_METADATA_CACHE);
 +Set flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, 
METRICS_FATE_TYPE_IN_PROGRESS,
 +METRICS_TSERVER_TABLETS_ONLINE_ONDEMAND, 
METRICS_TSERVER_TABLETS_ONDEMAND_UNLOADED_FOR_MEM,
 +METRICS_COMPACTOR_MAJC_STUCK, METRICS_MANAGER_ROOT_TGW_ERRORS,
 +METRICS_MANAGER_META_TGW_ERRORS, METRICS_MANAGER_USER_TGW_ERRORS,
 +METRICS_SCAN_TABLET_METADATA_CACHE, METRICS_SCAN_BUSY_TIMEOUT_COUNTER,
- METRICS_SCAN_RESERVATION_TIMER, METRICS_SERVER_IDLE);
++METRICS_SCAN_RESERVATION_TOTAL_TIMER, 
METRICS_SCAN_RESERVATION_WRITEOUT_TIMER,
++METRICS_SERVER_IDLE);
  
  Map expectedMetricNames = this.getMetricFields();
  flakyMetrics.forEach(expectedMetricNames::remove); // might not see these

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-10 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit cdaac866da85449227864caabc52fc00ee83fe56
Merge: ef34a4890a 861d28b64b
Author: Keith Turner 
AuthorDate: Fri May 10 18:20:33 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../java/org/apache/accumulo/core/client/rfile/RFileScanner.java  | 5 +
 .../accumulo/core/file/blockfile/cache/lru/LruBlockCache.java | 5 +
 .../core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java  | 5 +
 .../java/org/apache/accumulo/core/metrics/MetricsProducer.java| 4 
 .../main/java/org/apache/accumulo/core/spi/cache/BlockCache.java  | 7 +++
 .../main/java/org/apache/accumulo/tserver/BlockCacheMetrics.java  | 8 
 .../java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java | 3 +++
 7 files changed, 37 insertions(+)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-10 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit ef34a4890a76b592a805947d7c8f7d99a0363952
Merge: 5e4ec67ac0 8c485e362d
Author: Ed Coleman 
AuthorDate: Fri May 10 21:34:39 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 7 ---
 .../java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 6f82281257,abfdfc9b5a..bdc695e787
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -59,53 -59,12 +59,54 @@@ import io.micrometer.core.instrument.Me
   * 
   * 
   * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUES}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED}
 + * Gauge
 + * 
 + * 
 + * 
 + * N/A
 + * N/A
 + * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED}
 + * Gauge
 + * 
 + * 
 + * 
   * currentFateOps
   * Gauge
-  * {@value #METRICS_FATE_TOTAL_IN_PROGRESS}
+  * {@value #METRICS_FATE_OPS}
   * Gauge
-  * 
+  * Was previously named "accumulo.fate.ops.in.progress". Changed to 
better reflect what the
+  * gauge is actually tracking which is all the current fate ops in any 
state.
   * 
   * 
   * FateTxOpType_{name}
diff --cc 
server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
index 1664dd8d93,2f66aef256..e4e5150671
--- 
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
+++ 
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
@@@ -126,29 -128,31 +126,29 @@@ public class FateMetrics implements Met
  
@Override
public void registerMetrics(final MeterRegistry registry) {
- registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS, totalCurrentOpsGauge);
 -totalCurrentOpsGauge = registry.gauge(METRICS_FATE_OPS, new 
AtomicLong(0));
 -totalOpsGauge = registry.gauge(METRICS_FATE_OPS_ACTIVITY, new 
AtomicLong(0));
 -fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS, 
List.of(Tag.of("type", "zk.connection")),
 -new AtomicLong(0));
 -newTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.NEW.name().toLowerCase())),
 -new AtomicLong(0));
 -submittedTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.SUBMITTED.name().toLowerCase())),
 -new AtomicLong(0));
 -inProgressTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.IN_PROGRESS.name().toLowerCase())),
 -new AtomicLong(0));
 -failedInProgressTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())),
 -new AtomicLong(0));
 -failedTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.FAILED.name().toLowerCase())),
 -new AtomicLong(0));
 -successfulTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.SUCCESSFUL.name().toLowerCase())),
 -new AtomicLong(0));
 -unknownTxGauge = registry.gauge(METRICS_FATE_TX,
 -List.of(Tag.of("state", 
ReadOnlyTStore.TStatus.UNKNOWN.name().toLowerCase())),
 -new AtomicLong(0));
++registry.gauge(METRICS_FATE_OPS, totalCurrentOpsGauge);
 +registry.gauge(METRICS_FATE_OPS_ACTIVITY, totalOpsGauge);
 +registry.gauge(METRICS_FATE_ERRORS, List.of(Tag.of("type", 
"zk.connection")), fateErrorsGauge);
 +registry.gauge(METRICS_FATE_TX,
 +List.of(Tag.of("state", 
ReadOnlyFateStore.TStatus.NEW.name().toLowerCase())), newTxGauge);
 +registry.gauge(METRICS_FATE_TX,
 +List.of(Tag.of("state", 
ReadOnlyFateStore.TStatus.SUBMITTED.name().toLowerCase())),
 +submittedTxGauge);
 +registry.gauge(METRICS_FATE_TX,
 +List.of(Tag.of("state", 
ReadOnlyFateStore.TStatus.IN_PROGRESS.name().toLowerCase())),
 +inProgressTxGauge);
 +registry.gauge(METRICS_FATE_TX,
 +List.of(Tag.of("state", 
ReadOnlyFateStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())),
 +failedInProgressTxGauge);
 +registry.gauge(METRICS_FATE_TX,
 +List.of(Tag.of("st

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-08 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 4542108269fb8da4a523f1e5513974bc45ee13bd
Merge: b32a32cb58 f4c1f24658
Author: Ed Coleman 
AuthorDate: Wed May 8 18:01:13 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/core/metrics/MetricsProducer.java |  9 
 .../apache/accumulo/tserver/BlockCacheMetrics.java | 61 ++
 .../org/apache/accumulo/tserver/ScanServer.java| 18 +++
 .../org/apache/accumulo/tserver/TabletServer.java  |  5 +-
 4 files changed, 82 insertions(+), 11 deletions(-)

diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 7f325c39fa,0590fb8aa2..8aa8f65f4b
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -166,7 -181,9 +166,8 @@@ public class TabletServer extends Abstr
TabletServerUpdateMetrics updateMetrics;
TabletServerScanMetrics scanMetrics;
TabletServerMinCMetrics mincMetrics;
 -  CompactionExecutorsMetrics ceMetrics;
PausedCompactionMetrics pausedMetrics;
+   BlockCacheMetrics blockCacheMetrics;
  
@Override
public TabletServerScanMetrics getScanMetrics() {
@@@ -582,14 -712,21 +583,16 @@@
  updateMetrics = new TabletServerUpdateMetrics();
  scanMetrics = new TabletServerScanMetrics();
  mincMetrics = new TabletServerMinCMetrics();
 -ceMetrics = new CompactionExecutorsMetrics();
  pausedMetrics = new PausedCompactionMetrics();
+ blockCacheMetrics = new 
BlockCacheMetrics(this.resourceManager.getIndexCache(),
+ this.resourceManager.getDataCache(), 
this.resourceManager.getSummaryCache());
  
  metricsInfo.addMetricsProducers(this, metrics, updateMetrics, 
scanMetrics, mincMetrics,
- pausedMetrics);
 -ceMetrics, pausedMetrics, blockCacheMetrics);
++pausedMetrics, blockCacheMetrics);
  metricsInfo.init();
  
 -this.compactionManager = new CompactionManager(() -> Iterators
 -.transform(onlineTablets.snapshot().values().iterator(), 
Tablet::asCompactable),
 -getContext(), ceMetrics);
 -compactionManager.start();
 -
  announceExistence();
 +getContext().setServiceLock(tabletServerLock);
  
  try {
walMarker.initWalMarker(getTabletSession());

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-06 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit c6f7c91d829c6b30ac28f3f18f705913f611e25c
Merge: 0d40533307 d46ef7cdc6
Author: Ed Coleman 
AuthorDate: Mon May 6 21:40:51 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../server/metrics/NoOpDistributionSummary.java| 83 ++
 .../accumulo/server/metrics/ThriftMetrics.java |  4 +-
 .../metrics/NoOpDistributionSummaryTest.java}  | 38 --
 .../accumulo/server/metrics/ThriftMetricsTest.java | 62 
 .../tserver/metrics/TabletServerScanMetrics.java   |  5 +-
 .../tserver/metrics/TabletServerUpdateMetrics.java |  3 +-
 .../accumulo/test/functional/ZombieTServer.java| 10 +--
 7 files changed, 170 insertions(+), 35 deletions(-)

diff --cc 
test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index bc10568f7c,dce3d5dda1..38e93abb4c
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -143,6 -140,10 +138,11 @@@ public class ZombieTServer 
  
  ServiceLock zlock = new ServiceLock(zoo.getZooKeeper(), zLockPath, 
UUID.randomUUID());
  
+ MetricsInfo metricsInfo = context.getMetricsInfo();
 -metricsInfo.addServiceTags("zombie.server", serverPort.address);
++metricsInfo.addServiceTags("zombie.server", serverPort.address,
++Constants.DEFAULT_RESOURCE_GROUP_NAME);
+ metricsInfo.init();
+ 
  LockWatcher lw = new LockWatcher() {
  
@SuppressFBWarnings(value = "DM_EXIT",

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-06 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 0d40533307ab421960fbcc950892a72e72437031
Merge: 82b80d0d62 a41366bbd2
Author: Ed Coleman 
AuthorDate: Mon May 6 20:16:17 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../core/clientImpl/TabletServerBatchReaderIterator.java  | 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
index b95f256786,040a765b1f..3828b2c799
--- 
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchReaderIterator.java
@@@ -716,40 -650,26 +716,41 @@@ public class TabletServerBatchReaderIte
if (serverToUse == null) {
  // no scan server was given so use the tablet server
  serverToUse = extentToTserverMap.get(extent);
 -log.trace("For tablet {} using hints {} scan server selector chose 
tablet_server", tabletId,
 -options.executionHints);
 +if (serverToUse != null) {
-   log.trace("For tablet {} scan server selector chose tablet_server 
{}", tabletId,
-   serverToUse);
++  log.trace("For tablet {} using hints {} scan server selector chose 
tablet_server",
++  tabletId, options.executionHints);
 +} else {
 +  log.trace(
-   "For tablet {} scan server selector chose tablet_server, but 
tablet is not hosted",
-   tabletId);
++  "For tablet {} using hints {} scan server selector chose 
tablet_server, but tablet is not hosted",
++  tabletId, options.executionHints);
 +}
} else {
- log.trace("For tablet {} scan server selector chose scan_server:{}", 
tabletId, serverToUse);
+ log.trace("For tablet {} using hints {} scan server selector chose 
scan_server:{}",
+ tabletId, options.executionHints, serverToUse);
}
  
 -  var rangeMap = binnedRanges2.computeIfAbsent(serverToUse, k -> new 
HashMap<>());
 -  List ranges = extentToRangesMap.get(extent);
 -  rangeMap.put(extent, ranges);
 +  if (serverToUse != null) {
 +var rangeMap = binnedRanges.computeIfAbsent(serverToUse, k -> new 
HashMap<>());
 +List extentRanges = extentToRangesMap.get(extent);
 +rangeMap.put(extent, extentRanges);
 +
 +var server = serverToUse;
 +reporters.computeIfAbsent(serverToUse, k -> 
scanAttempts.createReporter(server, tabletId));
 +  } else {
 +failures.addAll(extentToRangesMap.get(extent));
 +  }
 +}
  
 -  var server = serverToUse;
 -  reporters.computeIfAbsent(serverToUse, k -> 
scanAttempts.createReporter(server, tabletId));
 +if (!failures.isEmpty()) {
 +  // if there are failures at this point its because tablets are not 
hosted, so lets attempt to
 +  // get them hosted
 +  clientTabletCache.findTablets(context, ranges, (cachedTablet, range) -> 
{},
 +  LocationNeed.REQUIRED);
 +  return new ScanServerData(failures);
  }
  
 -ScanServerData ssd = new ScanServerData();
 +ScanServerData ssd = new ScanServerData(actions, reporters);
  
 -ssd.binnedRanges = binnedRanges2;
 -ssd.actions = actions;
 -ssd.reporters = reporters;
  log.trace("Scan server selector chose delay:{} busyTimeout:{}", 
actions.getDelay(),
  actions.getBusyTimeout());
  return ssd;

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-05-06 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 82b80d0d62bd4290ac85a857b566d6cc0f864367
Merge: a198c1fbc4 7f58fc03b2
Author: Ed Coleman 
AuthorDate: Mon May 6 17:29:28 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

includes multiple PRs:

 - 4526 bouncy castle version bump
 - 4522 update scan server metrics for file reservations
 - 4520 log hints to select a scan server

 .../accumulo/core/metrics/MetricsProducer.java | 108 +++--
 pom.xml|   2 +-
 .../accumulo/manager/TabletGroupWatcher.java   |  18 ++--
 .../java/org/apache/accumulo/monitor/Monitor.java  |   6 +-
 .../org/apache/accumulo/tserver/ScanServer.java|  53 --
 .../apache/accumulo/tserver/ScanServerMetrics.java |  59 +++
 .../accumulo/tserver/ThriftScanClientHandler.java  |   4 +-
 .../tserver/metrics/TabletServerScanMetrics.java   |  20 ++--
 .../apache/accumulo/tserver/ScanServerTest.java|   9 ++
 .../apache/accumulo/test/metrics/MetricsIT.java|   8 +-
 10 files changed, 204 insertions(+), 83 deletions(-)

diff --cc 
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 18682789c5,c3a77f2297..060411fcbf
--- 
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++ 
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -972,43 -1449,20 +972,49 @@@ abstract class TabletGroupWatcher exten
  }
  tLists.assignments.addAll(tLists.assigned);
  for (Assignment a : tLists.assignments) {
-   TServerConnection client = manager.tserverSet.getConnection(a.server);
-   if (client != null) {
- client.assignTablet(manager.managerLock, a.tablet);
-   } else {
- Manager.log.warn("Could not connect to server {}", a.server);
+   try {
+ TServerConnection client = manager.tserverSet.getConnection(a.server);
+ if (client != null) {
+   client.assignTablet(manager.managerLock, a.tablet);
+   manager.assignedTablet(a.tablet);
+ } else {
+   Manager.log.warn("Could not connect to server {} for assignment of 
{}", a.server,
+   a.tablet);
+ }
+   } catch (TException tException) {
+ Manager.log.warn("Could not connect to server {} for assignment of 
{}", a.server, a.tablet,
+ tException);
}
-   manager.assignedTablet(a.tablet);
  }
 +
 +replaceVolumes(tLists.volumeReplacements);
 +  }
 +
 +  private void replaceVolumes(List 
volumeReplacementsList) {
 +try (var tabletsMutator = 
manager.getContext().getAmple().conditionallyMutateTablets()) {
 +  for (VolumeUtil.VolumeReplacements vr : volumeReplacementsList) {
 +var tabletMutator =
 +
tabletsMutator.mutateTablet(vr.tabletMeta.getExtent()).requireAbsentOperation()
 +.requireAbsentLocation().requireSame(vr.tabletMeta, FILES, 
LOGS);
 +vr.logsToRemove.forEach(tabletMutator::deleteWal);
 +vr.logsToAdd.forEach(tabletMutator::putWal);
 +
 +vr.filesToRemove.forEach(tabletMutator::deleteFile);
 +vr.filesToAdd.forEach(tabletMutator::putFile);
 +
 +tabletMutator.submit(
 +tm -> tm.getLogs().containsAll(vr.logsToAdd) && 
tm.getFiles().containsAll(vr.filesToAdd
 +
.keySet().stream().map(ReferencedTabletFile::insert).collect(Collectors.toSet(;
 +  }
 +
 +  tabletsMutator.process().forEach((extent, result) -> {
 +if (result.getStatus() == Ample.ConditionalResult.Status.REJECTED) {
 +  // log that failure happened, should try again later
 +  LOG.debug("Failed to update volumes for tablet {}", extent);
 +}
 +  });
 +}
 +
}
  
private static void markDeadServerLogsAsClosed(WalStateManager mgr,
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 51be4e13d8,10ccf19c6c..1fb3a66108
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -486,11 -489,11 +486,11 @@@ public class Monitor extends AbstractSe
  log.error("Unable to get hostname", e);
}
  }
- log.debug("Using {} to advertise monitor location in ZooKeeper", 
advertiseHost);
+ HostAndPort monitorHostAndPort = HostAndPort.fromParts(advertiseHost, 
livePort);
+ log.debug("Using {} to advertise monitor location in ZooKeeper", 
monitorHostAndPort);
  
  MetricsInfo metricsInfo = getContext().getMetricsInfo();
- metricsInfo.addServiceTags(getApplicationName(), 
HostAndPort.fromParts(advertiseHost, livePort),
- getResourceGroup());
 -metricsInfo.addServiceTags(getApplicationName(), monitorHostAndPort);
++

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-04-29 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit a198c1fbc45dbe6203bc662cf667010eb8f8d50d
Merge: 76fffdedf3 09adb11491
Author: Ed Coleman 
AuthorDate: Mon Apr 29 19:32:29 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/test/compaction/ExternalCompactionMetricsIT.java | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-04-29 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 13bb871ac52613d5ca42127b6c36b3e36e23ccb0
Merge: 03d3a6d911 252b46bab3
Author: Ed Coleman 
AuthorDate: Mon Apr 29 16:54:34 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java| 2 --
 1 file changed, 2 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-04-27 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 03d3a6d911f5711e95d5ebc990821bd2a69a85e9
Merge: 07bf1bc6fd e682a2f229
Author: Ed 
AuthorDate: Sat Apr 27 16:40:04 2024 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../java/org/apache/accumulo/core/metrics/MetricsProducer.java | 4 ++--
 .../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 1 +
 .../src/main/java/org/apache/accumulo/tserver/TabletServer.java| 2 +-
 .../org/apache/accumulo/test/functional/MemoryStarvedScanIT.java   | 7 +--
 .../java/org/apache/accumulo/test/functional/ZombieTServer.java| 7 +++
 5 files changed, 16 insertions(+), 5 deletions(-)

diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 0d99ea3f0f,6c07e9bb32..74ec7b89a5
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -488,8 -492,9 +488,9 @@@ public class Monitor extends AbstractSe
  log.debug("Using {} to advertise monitor location in ZooKeeper", 
advertiseHost);
  
  MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -metricsInfo.addServiceTags(getApplicationName(),
 -HostAndPort.fromParts(advertiseHost, livePort));
 +metricsInfo.addServiceTags(getApplicationName(), 
HostAndPort.fromParts(advertiseHost, livePort),
 +getResourceGroup());
+ metricsInfo.addMetricsProducers(this);
  metricsInfo.init();
  
  try {
diff --cc 
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 9947b5e48b,409d1ecada..93f3e58def
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -582,12 -711,18 +582,12 @@@ public class TabletServer extends Abstr
  updateMetrics = new TabletServerUpdateMetrics();
  scanMetrics = new TabletServerScanMetrics();
  mincMetrics = new TabletServerMinCMetrics();
 -ceMetrics = new CompactionExecutorsMetrics();
  pausedMetrics = new PausedCompactionMetrics();
  
- metricsInfo.addMetricsProducers(metrics, updateMetrics, scanMetrics, 
mincMetrics,
+ metricsInfo.addMetricsProducers(this, metrics, updateMetrics, 
scanMetrics, mincMetrics,
 -ceMetrics, pausedMetrics);
 +pausedMetrics);
  metricsInfo.init();
  
 -this.compactionManager = new CompactionManager(() -> Iterators
 -.transform(onlineTablets.snapshot().values().iterator(), 
Tablet::asCompactable),
 -getContext(), ceMetrics);
 -compactionManager.start();
 -
  announceExistence();
  
  try {
diff --cc 
test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
index 0becd57120,d2cb595b3d..0c9b08e0ab
--- 
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
+++ 
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
@@@ -48,14 -43,9 +48,15 @@@ import org.apache.accumulo.core.conf.Pr
  import org.apache.accumulo.core.data.Key;
  import org.apache.accumulo.core.data.Range;
  import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.fate.zookeeper.ZooCache;
  import org.apache.accumulo.core.iterators.WrappingIterator;
 +import org.apache.accumulo.core.lock.ServiceLock;
 +import org.apache.accumulo.core.lock.ServiceLockData;
 +import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
  import org.apache.accumulo.core.metrics.MetricsProducer;
 +import org.apache.accumulo.core.rpc.ThriftUtil;
 +import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
+ import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory;
  import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
  import org.apache.accumulo.harness.SharedMiniClusterBase;
  import org.apache.accumulo.minicluster.MemoryUnit;
diff --cc 
test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index ab90bcf870,d7562f78c1..bc10568f7c
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -129,6 -132,11 +130,12 @@@ public class ZombieTServer 
  HostAndPort.fromParts("0.0.0.0", port));
  
  String addressString = serverPort.address.toString();
+ 
+ MetricsInfo metricsInfo = context.getMetricsInfo();
 -metricsInfo.addServiceTags("zombie.server", serverPort.address);
++metricsInfo.addServiceTags("zombie.server", serverPort.address,
++Constants.DEFAULT_RESOURCE_GROUP_NAME);
+ metricsInfo.init();
+ 
  var zLockPath =
  ServiceLock.path(context.getZooKeeperRoot() + Constants.ZTSERVERS + 
"/" + addressString);
  ZooReaderWriter zoo = context.getZooReaderWriter();

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-03-19 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 2f4b0372512b3cf677e7a66d4b5ee8482a36a0da
Merge: ce83488245 0ad96b1dc0
Author: Ed Coleman 
AuthorDate: Tue Mar 19 20:31:36 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../accumulo/core/clientImpl/ClientContext.java|   9 +-
 .../core/clientImpl/ConditionalWriterImpl.java |   2 +-
 .../core/clientImpl/InstanceOperationsImpl.java|   4 +-
 .../core/clientImpl/TableOperationsImpl.java   |   4 +-
 .../core/clientImpl/TabletServerBatchReader.java   |   5 +-
 .../core/clientImpl/TabletServerBatchWriter.java   |   8 +-
 .../accumulo/core/clientImpl/bulk/BulkImport.java  |   8 +-
 .../accumulo/core/file/BloomFilterLayer.java   |   5 +-
 .../file/blockfile/cache/lru/LruBlockCache.java|   4 +-
 .../blockfile/cache/tinylfu/TinyLfuBlockCache.java |   2 +-
 .../util/compaction/ExternalCompactionUtil.java|   9 +-
 .../accumulo/core/util/threads/ThreadPools.java| 379 +
 .../core/file/rfile/MultiThreadedRFileTest.java|   7 +-
 .../threads/ThreadPoolExecutorBuilderTest.java |  79 +
 .../server/conf/ServerConfigurationFactory.java|   4 +-
 .../conf/store/impl/PropCacheCaffeineImpl.java |   7 +-
 .../server/conf/store/impl/PropStoreWatcher.java   |   5 +-
 .../accumulo/server/fs/VolumeManagerImpl.java  |   4 +-
 .../accumulo/server/problems/ProblemReports.java   |   7 +-
 .../apache/accumulo/server/rpc/TServerUtils.java   |   7 +-
 .../server/util/RemoveEntriesForMissingFiles.java  |   4 +-
 .../server/util/VerifyTabletAssignments.java   |   5 +-
 .../server/conf/store/impl/ReadyMonitorTest.java   |   4 +-
 .../main/java/org/apache/accumulo/gc/GCRun.java|   2 +-
 .../compaction/coordinator/QueueMetrics.java   |   4 +-
 .../accumulo/manager/metrics/fate/FateMetrics.java |   4 +-
 .../accumulo/manager/recovery/RecoveryManager.java |   4 +-
 .../apache/accumulo/manager/split/Splitter.java|   5 +-
 .../manager/tableOps/bulkVer2/TabletRefresher.java |   2 +-
 .../manager/upgrade/UpgradeCoordinator.java|   8 +-
 .../tserver/TabletServerResourceManager.java   |  21 +-
 .../org/apache/accumulo/tserver/log/LogSorter.java |   5 +-
 .../accumulo/tserver/log/TabletServerLogger.java   |   4 +-
 .../accumulo/test/BalanceWithOfflineTableIT.java   |   5 +-
 .../test/functional/BatchWriterFlushIT.java|   2 +-
 35 files changed, 409 insertions(+), 229 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
index c009e14b14,02aeef6bf6..bf650a077f
--- 
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
@@@ -372,8 -372,8 +372,8 @@@ public class ConditionalWriterImpl impl
  this.auths = config.getAuthorizations();
  this.ve = new VisibilityEvaluator(config.getAuthorizations());
  this.threadPool = context.threadPools().createScheduledExecutorService(
- config.getMaxWriteThreads(), this.getClass().getSimpleName(), false);
+ config.getMaxWriteThreads(), this.getClass().getSimpleName());
 -this.locator = new SyncingTabletLocator(context, tableId);
 +this.locator = new SyncingClientTabletCache(context, tableId);
  this.serverQueues = new HashMap<>();
  this.tableId = tableId;
  this.tableName = tableName;
diff --cc 
core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 297ed70495,6b94fd81cf..2ea43612f1
--- 
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@@ -448,74 -422,109 +448,76 @@@ public class TableOperationsImpl extend
  }
}
  
 -  private static class SplitEnv {
 -private final String tableName;
 -private final TableId tableId;
 -private final ExecutorService executor;
 -private final CountDownLatch latch;
 -private final AtomicReference exception;
 -
 -SplitEnv(String tableName, TableId tableId, ExecutorService executor, 
CountDownLatch latch,
 -AtomicReference exception) {
 -  this.tableName = tableName;
 -  this.tableId = tableId;
 -  this.executor = executor;
 -  this.latch = latch;
 -  this.exception = exception;
 -}
 -  }
 -
 -  private class SplitTask implements Runnable {
 -
 -private List splits;
 -private SplitEnv env;
 +  /**
 +   * On the server side the fate operation will exit w/o an error if the 
tablet requested to split
 +   * does not exist. When this happens it will also return an empty string. 
In the case where the
 +   * fate operation successfully splits the tablet it will return the 
following string. This code
 +   * uses this return value to see if it nee

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-03-15 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 61ce280c8ebbfe92961bedf76500e59c1f935cf6
Merge: f1914da45a a3616bf46f
Author: Ed Coleman 
AuthorDate: Fri Mar 15 21:21:51 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-03-11 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 8b0d2355813300ed09b45920fc1cd7cf091df1be
Merge: 3fe1115014 9d9bcc1d48
Author: Ed Coleman 
AuthorDate: Mon Mar 11 13:05:13 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../src/main/java/org/apache/accumulo/manager/ManagerTime.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-03-08 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 0148294e35dbcf502fb0f9df6a178d669e4c8f3f
Merge: 7b3e52f1ae 311c5fe821
Author: Ed Coleman 
AuthorDate: Fri Mar 8 19:02:50 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

Additional changes from merge
 * Unused type variable in initializeFateInstance
 * unneeded initialization

 .../java/org/apache/accumulo/manager/Manager.java  | 23 +++---
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 95fb00777e,bdf73b9191..6758acfc1f
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -243,20 -231,6 +243,20 @@@ public class Manager extends AbstractSe
  return state;
}
  
 +  // ELASTICITIY_TODO it would be nice if this method could take DataLevel as 
an argument and only
 +  // retrieve information about compactions in that data level. Attempted 
this and a lot of
 +  // refactoring was needed to get that small bit of information to this 
method. Would be best to
 +  // address this after issue. May be best to attempt this after #3576.
 +  public Map> getCompactionHints() {
- Map allConfig = null;
++Map allConfig;
 +try {
 +  allConfig = CompactionConfigStorage.getAllConfig(getContext(), tableId 
-> true);
 +} catch (InterruptedException | KeeperException e) {
 +  throw new RuntimeException(e);
 +}
 +return Maps.transformValues(allConfig, 
CompactionConfig::getExecutionHints);
 +  }
 +
public boolean stillManager() {
  return getManagerState() != ManagerState.STOP;
}
@@@ -620,9 -757,9 +620,9 @@@
  
  @Override
  public void run() {
 -  EventCoordinator.Listener eventListener = nextEvent.getListener();
 +  EventCoordinator.Tracker eventTracker = nextEvent.getTracker();
while (stillManager()) {
- long wait = DEFAULT_WAIT_FOR_WATCHER;
+ long wait;
  try {
switch (getManagerGoalState()) {
  case NORMAL:
@@@ -960,7 -1088,7 +960,7 @@@
  log.info("Started Manager client service at {}", sa.address);
  
  // block until we can obtain the ZK lock for the manager
--ServiceLockData sld = null;
++ServiceLockData sld;
  try {
sld = getManagerLock(ServiceLock.path(zroot + Constants.ZMANAGER_LOCK));
  } catch (KeeperException | InterruptedException e) {
@@@ -1072,17 -1197,17 +1072,17 @@@
  }
  
  try {
-   var metaInstance = initializeFateInstance(context, 
FateInstanceType.META,
 -  final AgeOffStore store = new AgeOffStore<>(
 -  new org.apache.accumulo.core.fate.ZooStore<>(getZooKeeperRoot() + 
Constants.ZFATE,
 -  context.getZooReaderWriter()),
 -  HOURS.toMillis(8), System::currentTimeMillis);
 -
 -  Fate f = new Fate<>(this, store, TraceRepo::toLogString, 
getConfiguration());
 -  fateRef.set(f);
++  var metaInstance = initializeFateInstance(context,
 +  new ZooStore<>(getZooKeeperRoot() + Constants.ZFATE, 
context.getZooReaderWriter()));
-   var userInstance = initializeFateInstance(context, 
FateInstanceType.USER,
++  var userInstance = initializeFateInstance(context,
 +  new AccumuloStore<>(context, AccumuloTable.FATE.tableName()));
 +
 +  if (!fateRefs.compareAndSet(null,
 +  Map.of(FateInstanceType.META, metaInstance, FateInstanceType.USER, 
userInstance))) {
 +throw new IllegalStateException(
 +"Unexpected previous fate reference map already initialized");
 +  }
fateReadyLatch.countDown();
 -
 -  ThreadPools.watchCriticalScheduledTask(context.getScheduledExecutor()
 -  .scheduleWithFixedDelay(store::ageOff, 63000, 63000, MILLISECONDS));
  } catch (KeeperException | InterruptedException e) {
throw new IllegalStateException("Exception setting up FaTE cleanup 
thread", e);
  }
@@@ -1117,15 -1242,9 +1117,15 @@@
  }
  
  String address = sa.address.toString();
 -sld = new ServiceLockData(sld.getServerUUID(ThriftService.MANAGER), 
address,
 -ThriftService.MANAGER);
 -log.info("Setting manager lock data to {}", sld.toString());
 +UUID uuid = sld.getServerUUID(ThriftService.MANAGER);
 +ServiceDescriptors descriptors = new ServiceDescriptors();
 +for (ThriftService svc : new ThriftService[] {ThriftService.MANAGER, 
ThriftService.COORDINATOR,
 +ThriftService.FATE}) {
 +  descriptors.addService(new ServiceDescriptor(uuid, svc, address, 
this.getResourceGroup()));
 +}
 +
 +sld = new ServiceLockData(descriptors);
- log.info("Setting manager lock data to {}", sld.toString());
++log.info("Setting manager lock data to {}", sld);
  try {

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-03-07 Thread edcoleman

This is an automated email from the ASF dual-hosted git repository.

edcoleman pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 7b3e52f1ae0f1a02d0804744137db521e96cf201
Merge: c7ae69b881 dcb09a726b
Author: Ed Coleman 
AuthorDate: Thu Mar 7 19:42:42 2024 +

Merge remote-tracking branch 'upstream/main' into elasticity

 .../core/file/blockfile/cache/lru/LruBlockCache.java   | 14 --
 .../file/blockfile/cache/tinylfu/TinyLfuBlockCache.java|  4 ++--
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
index feb6a1b5c1,04cab1a38c..3656b99815
--- 
a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/tinylfu/TinyLfuBlockCache.java
@@@ -63,15 -62,15 +63,15 @@@ public final class TinyLfuBlockCache im
private final Policy.Eviction policy;
private final int maxSize;
private final ScheduledExecutorService statsExecutor = 
ThreadPools.getServerThreadPools()
-   .createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor", 
true);
+   .createScheduledExecutorService(1, "TinyLfuBlockCacheStatsExecutor", 
false);
  
public TinyLfuBlockCache(Configuration conf, CacheType type) {
 -cache = Caffeine.newBuilder()
 +cache = 
Caches.getInstance().createNewBuilder(CacheName.TINYLFU_BLOCK_CACHE, false)
  .initialCapacity((int) Math.ceil(1.2 * conf.getMaxSize(type) / 
conf.getBlockSize()))
 -.weigher((String blockName, Block block) -> {
 +.recordStats().weigher((String blockName, Block block) -> {
int keyWeight = ClassSize.align(blockName.length()) + 
ClassSize.STRING;
return keyWeight + block.weight();
 -}).maximumWeight(conf.getMaxSize(type)).recordStats().build();
 +}).maximumWeight(conf.getMaxSize(type)).build();
  policy = cache.policy().eviction().orElseThrow();
  maxSize = (int) Math.min(Integer.MAX_VALUE, policy.getMaximum());
  ScheduledFuture future = 
statsExecutor.scheduleAtFixedRate(this::logStats, STATS_PERIOD_SEC,

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2024-01-16 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 1f218de444eb1aff5654c55fc84659d0491ae8b8
Merge: f674e9a0b0 f26c702a87
Author: Keith Turner 
AuthorDate: Tue Jan 16 12:51:49 2024 -0500

Merge remote-tracking branch 'upstream/main' into elasticity

 .../core/clientImpl/TableOperationsImpl.java   | 71 +-
 .../clientImpl/bulk/ConcurrentKeyExtentCache.java  | 17 ++---
 .../fate/zookeeper/DistributedReadWriteLock.java   | 33 +
 .../org/apache/accumulo/core/summary/Gatherer.java | 22 +++---
 .../java/org/apache/accumulo/core/util/Merge.java  | 62 
 .../org/apache/accumulo/core/util/MergeTest.java   | 31 +++-
 .../manager/balancer/BalancerEnvironmentImpl.java  | 12 ++--
 .../accumulo/server/metadata/ServerAmpleImpl.java  |  2 +-
 .../main/java/org/apache/accumulo/gc/GCRun.java|  4 +-
 .../accumulo/gc/GarbageCollectionAlgorithm.java| 84 +++---
 .../apache/accumulo/manager/tableOps/Utils.java| 15 +++-
 .../manager/tableOps/bulkVer2/LoadFiles.java   | 23 +++---
 .../manager/tableOps/bulkVer2/PrepBulkImport.java  | 42 ---
 .../tableOps/bulkVer2/PrepBulkImportTest.java  | 31 +---
 .../org/apache/accumulo/tserver/ScanServer.java| 10 ++-
 .../java/org/apache/accumulo/test/AmpleIT.java | 16 +++--
 .../java/org/apache/accumulo/test/GCRunIT.java | 13 ++--
 .../accumulo/test/ScanServerMetadataEntriesIT.java |  6 +-
 .../test/compaction/ExternalCompaction4_IT.java| 33 +
 .../test/compaction/ExternalCompaction_2_IT.java   |  8 +--
 .../test/functional/GarbageCollectorTrashBase.java | 17 ++---
 .../accumulo/test/functional/MetadataIT.java   | 15 ++--
 22 files changed, 335 insertions(+), 232 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 09e4174dd7,3b7bf91612..157df582ac
--- 
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@@ -1306,37 -1328,38 +1303,41 @@@ public class TableOperationsImpl extend
Text continueRow = null;
MapCounter serverCounts = new MapCounter<>();
  
-   for (TabletMetadata tablet : tablets) {
- total++;
- Location loc = tablet.getLocation();
- TabletHostingGoal goal = tablet.getHostingGoal();
- 
- if ((expectedState == TableState.ONLINE
- && (goal == TabletHostingGoal.ALWAYS
- || (goal == TabletHostingGoal.ONDEMAND) && 
tablet.getHostingRequested())
- && (loc == null || loc.getType() == LocationType.FUTURE))
- || (expectedState == TableState.OFFLINE && loc != null)) {
-   if (continueRow == null) {
- continueRow = tablet.getExtent().toMetaRow();
+   try (TabletsMetadata tablets = 
TabletsMetadata.builder(context).scanMetadataTable()
 -  .overRange(range).fetch(LOCATION, PREV_ROW).build()) {
++  .overRange(range).fetch(HOSTING_GOAL, HOSTING_REQUESTED, LOCATION, 
PREV_ROW).build()) {
+ 
+ for (TabletMetadata tablet : tablets) {
+   total++;
+   Location loc = tablet.getLocation();
++  TabletHostingGoal goal = tablet.getHostingGoal();
+ 
+   if ((expectedState == TableState.ONLINE
++  && (goal == TabletHostingGoal.ALWAYS
++  || (goal == TabletHostingGoal.ONDEMAND) && 
tablet.getHostingRequested())
+   && (loc == null || loc.getType() == LocationType.FUTURE))
+   || (expectedState == TableState.OFFLINE && loc != null)) {
+ if (continueRow == null) {
+   continueRow = tablet.getExtent().toMetaRow();
+ }
+ waitFor++;
+ lastRow = tablet.getExtent().toMetaRow();
+ 
+ if (loc != null) {
+   serverCounts.increment(loc.getHostPortSession(), 1);
+ }
}
-   waitFor++;
-   lastRow = tablet.getExtent().toMetaRow();
  
-   if (loc != null) {
- serverCounts.increment(loc.getHostPortSession(), 1);
+   if (!tablet.getExtent().tableId().equals(tableId)) {
+ throw new AccumuloException(
+ "Saw unexpected table Id " + tableId + " " + 
tablet.getExtent());
}
- }
  
- if (!tablet.getExtent().tableId().equals(tableId)) {
-   throw new AccumuloException(
-   "Saw unexpected table Id " + tableId + " " + 
tablet.getExtent());
- }
+   if (lastExtent != null && 
!tablet.getExtent().isPreviousExtent(lastExtent)) {
+ holes++;
+   }
  
- if (lastExtent != null && 
!tablet.getExtent().isPreviousExtent(lastExtent)) {
-   holes++;
+   lastExtent = tablet.getExtent();
  }
- 
-

[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2023-10-10 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 123ce967869621d527aff453bdf43f3617bc76ec
Merge: 31323e5e5e bc932655b3
Author: Keith Turner 
AuthorDate: Tue Oct 10 17:51:13 2023 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../org/apache/accumulo/core/conf/Property.java|   6 +-
 .../accumulo/core/metadata/StoredTabletFile.java   |   8 +
 .../util/compaction/ExternalCompactionUtil.java|  16 +-
 pom.xml|  49 ---
 .../constraints/MetadataConstraintsTest.java   | 210 +++---
 server/compactor/pom.xml   |  38 +-
 .../apache/accumulo/compactor/CompactorTest.java   | 455 -
 .../src/test/resources/log4j2-test.properties  |  35 --
 server/native/pom.xml  |   5 -
 src/build/ci/find-unapproved-junit.sh  |   8 +-
 start/pom.xml  |  17 +-
 .../accumulo/test/functional/MonitorSslIT.java |   2 +-
 .../accumulo/test/functional/ReadWriteIT.java  |   2 +-
 .../org/apache/accumulo/test/shell/MockShell.java  |  17 +-
 .../apache/accumulo/test/shell/ShellServerIT.java  |  68 +--
 15 files changed, 250 insertions(+), 686 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index e4eda5713d,cbda041d41..697f079d40
--- 
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@@ -99,15 -99,9 +99,9 @@@ public class ExternalCompactionUtil 
 * @return Optional HostAndPort of Coordinator node if found
 */
public static Optional findCompactionCoordinator(ClientContext 
context) {
 -final String lockPath = context.getZooKeeperRoot() + 
Constants.ZCOORDINATOR_LOCK;
 +final String lockPath = context.getZooKeeperRoot() + 
Constants.ZMANAGER_LOCK;
- try {
-   var zk = ZooSession.getAnonymousSession(context.getZooKeepers(),
-   context.getZooKeepersSessionTimeOut());
-   return ServiceLock.getLockData(zk, ServiceLock.path(lockPath))
-   .map(sld -> sld.getAddress(ThriftService.COORDINATOR));
- } catch (KeeperException | InterruptedException e) {
-   throw new IllegalStateException(e);
- }
+ return ServiceLock.getLockData(context.getZooCache(), 
ServiceLock.path(lockPath), new ZcStat())
+ .map(sld -> sld.getAddress(ThriftService.COORDINATOR));
}
  
/**
@@@ -115,14 -109,14 +109,14 @@@
 */
public static Map> getCompactorAddrs(ClientContext 
context) {
  try {
 -  final Map> queuesAndAddresses = new 
HashMap<>();
 -  final String compactorQueuesPath = context.getZooKeeperRoot() + 
Constants.ZCOMPACTORS;
 +  final Map> groupsAndAddresses = new 
HashMap<>();
 +  final String compactorGroupsPath = context.getZooKeeperRoot() + 
Constants.ZCOMPACTORS;
ZooReader zooReader = context.getZooReader();
 -  List queues = zooReader.getChildren(compactorQueuesPath);
 -  for (String queue : queues) {
 -queuesAndAddresses.putIfAbsent(queue, new ArrayList<>());
 +  List groups = zooReader.getChildren(compactorGroupsPath);
 +  for (String group : groups) {
- groupsAndAddresses.putIfAbsent(group, new ArrayList());
++groupsAndAddresses.putIfAbsent(group, new ArrayList<>());
  try {
 -  List compactors = zooReader.getChildren(compactorQueuesPath 
+ "/" + queue);
 +  List compactors = zooReader.getChildren(compactorGroupsPath 
+ "/" + group);
for (String compactor : compactors) {
  // compactor is the address, we are checking to see if there is a 
child node which
  // represents the compactor's lock as a check that it's alive.
diff --cc pom.xml
index 7a5a7f1fde,602809f869..4fd8a15edd
--- a/pom.xml
+++ b/pom.xml
@@@ -322,12 -322,11 +321,6 @@@
  commons-logging
  1.2

--  
- 
- junit
- junit
- 4.13.2
 -org.apache.accumulo
 -accumulo-compaction-coordinator
 -${project.version}
--  

  org.apache.accumulo
  accumulo-compactor
@@@ -623,37 -622,6 +616,11 @@@

  

-   
- 
- org.objenesis
- objenesis
- 3.3
-   
 +  
 +org.opentest4j
 +opentest4j
 +1.2.0
 +  
-   
- org.powermock
- powermock-api-easymock
- ${version.powermock}
-   
-   
- org.powermock
- powermock-core
- ${version.powermock}
-   
-   
- org.powermock
- powermock-module-junit4
- ${version.powermock}
-   
-   
- org.powermock
- powermock-refl

[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2023-05-04 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 899a95a18be20433038691df7aa0e8515a60c402
Merge: 950c806e20 de73759c24
Author: Keith Turner 
AuthorDate: Thu May 4 13:08:34 2023 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .github/ISSUE_TEMPLATE/post_vote_checklist.md  |   5 +-
 .github/workflows/scripts.yaml |  16 +
 LICENSE|   3 +-
 assemble/bin/accumulo-cluster  |   8 -
 assemble/bin/accumulo-service  |   6 -
 assemble/build.sh  |   2 +-
 assemble/conf/accumulo-env.sh  |   4 +-
 assemble/src/main/resources/LICENSE|   3 +-
 contrib/ci/install-thrift.sh   |  31 ++
 contrib/ci/run-thrift.sh   |  35 ++
 .../accumulo/core/clientImpl/ClientContext.java|   6 +-
 .../core/clientImpl/TableOperationsImpl.java   |  35 +-
 .../accumulo/core/conf/ConfigurationDocGen.java|   4 +-
 .../org/apache/accumulo/core/conf/Property.java|  11 +-
 .../java/org/apache/accumulo/core/data/Range.java  |  22 +-
 .../blockfile/cache/tinylfu/TinyLfuBlockCache.java |   2 +-
 .../core/iterators/IteratorEnvironment.java|  13 -
 .../core/iteratorsImpl/system/MapFileIterator.java |  16 +-
 .../system/SourceSwitchingIterator.java|  12 +-
 .../apache/accumulo/core/logging/TabletLogger.java |   2 +-
 .../manager/balancer/AssignmentParamsImpl.java |   2 +-
 .../core/manager/balancer/BalanceParamsImpl.java   |   2 +-
 .../core/manager/balancer/TServerStatusImpl.java   |   2 +-
 .../core/manager/balancer/TableStatisticsImpl.java |   2 +-
 .../core/metadata/schema/MetadataSchema.java   |  17 +
 .../core/metadata/schema/TabletMetadata.java   |   4 +-
 .../core/metadata/schema/TabletsMetadata.java  |   2 +
 .../accumulo/core/metrics/MetricsProducer.java |  10 +-
 .../accumulo/core/rpc/clients/TServerClient.java   |   2 +-
 .../core/spi/balancer/SimpleLoadBalancer.java  |   6 -
 .../accumulo/core/spi/crypto/AESCryptoService.java |   4 +-
 .../spi/crypto/PerTableCryptoServiceFactory.java   |   2 +-
 .../accumulo/core/spi/fs/DelegatingChooser.java|   4 +-
 .../core/spi/fs/PreferredVolumeChooser.java|   2 +-
 .../org/apache/accumulo/core/util/Encoding.java|   6 +-
 .../core/util/compaction/CompactionJobImpl.java|   2 +-
 .../util/compaction/ExternalCompactionUtil.java|   2 +-
 .../core/util/interpret/ScanInterpreter.java   |  41 ---
 core/src/main/scripts/generate-thrift.sh   |   2 +-
 core/src/main/spotbugs/exclude-filter.xml  |   6 -
 .../thrift/BulkImportState.java|   2 +-
 .../thrift/BulkImportStatus.java   |   6 +-
 .../{master => manager}/thrift/Compacting.java |   2 +-
 .../accumulo/core/manager/thrift/FateService.java  |  76 ++--
 .../core/manager/thrift/ManagerClientService.java  |  32 +-
 .../core/manager/thrift/ManagerMonitorInfo.java| 304 +++
 .../core/manager/thrift/RecoveryException.java | 406 -
 .../{master => manager}/thrift/RecoveryStatus.java |   2 +-
 .../core/{master => manager}/thrift/TableInfo.java |   2 +-
 .../thrift/TabletServerStatus.java | 122 +++
 .../thrift/TabletServerClientService.java  |  44 +--
 core/src/main/thrift/manager.thrift|  79 +++-
 core/src/main/thrift/master.thrift |  92 -
 core/src/main/thrift/tabletingest.thrift   |   1 -
 core/src/main/thrift/tabletscan.thrift |   1 -
 core/src/main/thrift/tabletserver.thrift   |   3 +-
 .../org/apache/accumulo/core/WithTestNames.java|   2 +-
 .../core/conf/AccumuloConfigurationTest.java   |  32 +-
 .../core/iterators/DefaultIteratorEnvironment.java |  14 -
 .../system/SourceSwitchingIteratorTest.java|   7 +-
 .../BaseHostRegexTableLoadBalancerTest.java|  34 +-
 .../core/spi/balancer/GroupBalancerTest.java   |   2 +-
 .../core/spi/balancer/SimpleLoadBalancerTest.java  |   6 +-
 .../core/spi/balancer/TableLoadBalancerTest.java   |   6 +-
 .../mapreduce/FileOutputFormatBuilderImpl.java |  12 +-
 .../mapreduce/InputFormatBuilderImpl.java  |   6 +-
 .../mapreduce/OutputFormatBuilderImpl.java |   2 +-
 .../src/main/spotbugs/exclude-filter.xml   |   5 -
 .../apache/accumulo/hadoop/its/WithTestNames.java  |   2 +-
 .../mapreduce/InputFormatBuilderTest.java  |   2 +-
 .../accumulo/minicluster/MiniAccumuloRunner.java   |   2 +-
 .../MiniAccumuloClusterControl.java|   2 +-
 minicluster/src/main/spotbugs/exclude-filter.xml   |  31 --
 .../apache/accumulo/minicluster/WithTestNames.java |   2 +-
 pom.xml|  40 +-
 ...

[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

2023-04-05 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 41a3053d539629faf1c3342dd4a85da8cb072215
Merge: 7a54f64282 8b76290644
Author: Keith Turner 
AuthorDate: Wed Apr 5 16:22:21 2023 -0400

Merge remote-tracking branch 'upstream/main' into elasticity

 .../apache/accumulo/core/client/ScannerBase.java   |   1 +
 .../core/clientImpl/RootTabletLocator.java |   6 +-
 .../core/clientImpl/SyncingTabletLocator.java  |  23 +--
 .../core/clientImpl/TableOperationsImpl.java   |  14 +-
 .../accumulo/core/clientImpl/TabletLocator.java|  44 +++---
 .../core/clientImpl/TabletLocatorImpl.java |  65 
 .../accumulo/core/clientImpl/ThriftScanner.java|  59 +++
 .../apache/accumulo/core/clientImpl/Writer.java|  10 +-
 .../core/metadata/MetadataLocationObtainer.java|  20 +--
 .../accumulo/core/security/Authorizations.java |  52 +-
 .../core/clientImpl/TabletLocatorImplTest.java |  18 +--
 .../hadoop/mapreduce/InputFormatBuilder.java   |   2 +
 .../accumulo/server/client/BulkImporter.java   |  14 +-
 .../server/conf/ServerConfigurationFactory.java|  23 ++-
 .../accumulo/server/util/MetadataTableUtil.java|   4 -
 .../accumulo/server/client/BulkImporterTest.java   |  19 +--
 .../accumulo/test/functional/BulkFailureIT.java|   2 +-
 .../accumulo/test/manager/SuspendedTabletsIT.java  |   4 +-
 .../apache/accumulo/test/shell/ShellServerIT.java  | 175 -
 19 files changed, 306 insertions(+), 249 deletions(-)

diff --cc 
core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
index 6fa7716dc6,6d0f2aa735..76afd84fc1
--- 
a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
@@@ -87,9 -87,9 +87,9 @@@ public class MetadataLocationObtainer i
OpTimer timer = null;
  
if (log.isTraceEnabled()) {
 -log.trace("tid={} Looking up in {} row={} extent={} tserver={}",
 +log.trace("tid={} Looking up in {} row={} stopRow={} extent={} 
tserver={}",
- Thread.currentThread().getId(), src.tablet_extent.tableId(), 
TextUtil.truncate(row),
- TextUtil.truncate(stopRow), src.tablet_extent, 
src.tablet_location);
+ Thread.currentThread().getId(), src.getExtent().tableId(), 
TextUtil.truncate(row),
 -src.getExtent(), src.getTserverLocation());
++TextUtil.truncate(stopRow), src.getExtent(), 
src.getTserverLocation());
  timer = new OpTimer().start();
}

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity

30 matches

Site Navigation

Mail list logo

Footer information