(accumulo) branch elasticity updated: Fix Compactors table on EC Monitor page (#4744)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 9a7af44338 Fix Compactors table on EC Monitor page (#4744)
9a7af44338 is described below
commit 9a7af44338753539cf2605610af0b8cf3d7a3dc4
Author: Christopher L. Shannon
AuthorDate: Sat Jul 20 15:36:04 2024 -0400
Fix Compactors table on EC Monitor page (#4744)
compactor.queue was renamed to compactor.group in #3591 and this updates
the Monitor to use the new name so Compactors can be displayed
correctly in the table on the external compations page. Before this
change there the table was empty and there were errors in the javascript
console.
---
.../src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js | 2 +-
.../src/main/resources/org/apache/accumulo/monitor/templates/ec.ftl | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js
index 545c2559df..cf034a8123 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/ec.js
@@ -61,7 +61,7 @@ $(document).ready(function () {
"data": "server"
},
{
-"data": "queueName"
+"data": "groupName"
},
{
"data": "lastContact"
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/ec.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/ec.ftl
index b69b6ef017..37ac8b4d51 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/ec.ftl
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/ec.ftl
@@ -49,7 +49,7 @@
Server
- Queue
+ Group
Last Contact
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 47a53ae91d223021155e05a71f492729a56c012d Merge: 2a52d5d8e1 fc309ccd6d Author: Christopher L. Shannon AuthorDate: Fri Jun 21 19:03:05 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch elasticity updated (2a52d5d8e1 -> 47a53ae91d)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 2a52d5d8e1 Merge branch 'main' into elasticity add e86af0d3bd Fix MemoryStarvedScanIT (#4690) add fc309ccd6d Modified ScanServer to not allow scans on Fate table as well new 47a53ae91d Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/02: Fix MemoryStarvedScanIT (#4690)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit e86af0d3bdfcdc4ea320d524f3fd20b2bc9281ab
Author: Christopher L. Shannon
AuthorDate: Fri Jun 21 18:48:44 2024 -0400
Fix MemoryStarvedScanIT (#4690)
Backport a combination of #4270 and part of #4163 to exclude all tables
in the accumulo namespace from being blocked on low memory detection
---
.../java/org/apache/accumulo/core/metadata/AccumuloTable.java | 11 +++
.../java/org/apache/accumulo/tserver/tablet/TabletBase.java | 3 ++-
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
b/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
index 929444fb3c..568959d83b 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
@@ -18,6 +18,10 @@
*/
package org.apache.accumulo.core.metadata;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.data.TableId;
@@ -43,4 +47,11 @@ public enum AccumuloTable {
this.name = Namespace.ACCUMULO.name() + "." + name;
this.tableId = TableId.of(id);
}
+
+ private static final Set ALL_IDS =
+
Arrays.stream(values()).map(AccumuloTable::tableId).collect(Collectors.toUnmodifiableSet());
+
+ public static Set allTableIds() {
+return ALL_IDS;
+ }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
index 9b0545a5c7..c7c6f9ae3d 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
@@ -43,6 +43,7 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.YieldCallback;
import
org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException;
import org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
@@ -93,7 +94,7 @@ public abstract class TabletBase {
this.context = server.getContext();
this.server = server;
this.extent = extent;
-this.isUserTable = !extent.isMeta();
+this.isUserTable = !AccumuloTable.allTableIds().contains(extent.tableId());
TableConfiguration tblConf =
context.getTableConfiguration(extent.tableId());
if (tblConf == null) {
(accumulo) branch main updated (59f560dd74 -> fc309ccd6d)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 59f560dd74 Fix typo (#4690) new e86af0d3bd Fix MemoryStarvedScanIT (#4690) new fc309ccd6d Modified ScanServer to not allow scans on Fate table as well The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/core/dataImpl/KeyExtent.java | 4 .../java/org/apache/accumulo/core/metadata/AccumuloTable.java | 11 +++ .../src/main/java/org/apache/accumulo/tserver/ScanServer.java | 4 ++-- .../java/org/apache/accumulo/tserver/tablet/TabletBase.java | 3 ++- .../test/java/org/apache/accumulo/tserver/ScanServerTest.java | 10 +- 5 files changed, 24 insertions(+), 8 deletions(-)
(accumulo) 02/02: Modified ScanServer to not allow scans on Fate table as well
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit fc309ccd6de99e95520f0ffdfd445bc1b97fe261
Author: Dave Marion
AuthorDate: Mon May 20 14:18:55 2024 +
Modified ScanServer to not allow scans on Fate table as well
(cherry picked from commit 6afdc88f8e6ebfdd2a5b14c2837f1ff03c062a21)
This now applies with Scan ref tables (#4690)
---
.../main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java | 4
.../src/main/java/org/apache/accumulo/tserver/ScanServer.java | 4 ++--
.../test/java/org/apache/accumulo/tserver/ScanServerTest.java | 10 +-
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
b/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
index b5bdb1f744..1069a5e3bc 100644
--- a/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
@@ -535,6 +535,10 @@ public class KeyExtent implements Comparable {
return prevExtent.endRow().equals(prevEndRow());
}
+ public boolean isSystemTable() {
+return AccumuloTable.allTableIds().contains(tableId());
+ }
+
public boolean isMeta() {
return tableId().equals(AccumuloTable.METADATA.tableId()) ||
isRootTablet();
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 3ba3a1c306..ae226db339 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@ -945,7 +945,7 @@ public class ScanServer extends AbstractServer
KeyExtent extent = getKeyExtent(textent);
-if (extent.isMeta() && !isSystemUser(credentials)) {
+if (extent.isSystemTable() && !isSystemUser(credentials)) {
throw new TException(
"Only the system user can perform eventual consistency scans on the
root and metadata tables");
}
@@ -1012,7 +1012,7 @@ public class ScanServer extends AbstractServer
for (Entry> entry : tbatch.entrySet()) {
KeyExtent extent = getKeyExtent(entry.getKey());
- if (extent.isMeta() &&
!context.getSecurityOperation().isSystemUser(credentials)) {
+ if (extent.isSystemTable() && !isSystemUser(credentials)) {
throw new TException(
"Only the system user can perform eventual consistency scans on
the root and metadata tables");
}
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/ScanServerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/ScanServerTest.java
index f93f095a87..bbe8ffb3fc 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/ScanServerTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/ScanServerTest.java
@@ -190,7 +190,7 @@ public class ScanServerTest {
Map execHints = new HashMap<>();
ScanReservation reservation = createMock(ScanReservation.class);
-expect(extent.isMeta()).andReturn(false).anyTimes();
+expect(extent.isSystemTable()).andReturn(false).anyTimes();
expect(extent.toThrift()).andReturn(textent).anyTimes();
expect(reservation.getFailures()).andReturn(Map.of(textent, ranges));
reservation.close();
@@ -242,7 +242,7 @@ public class ScanServerTest {
};
TestScanServer ss = partialMockBuilder(TestScanServer.class).createMock();
-expect(extent.isMeta()).andReturn(false).anyTimes();
+expect(extent.isSystemTable()).andReturn(false).anyTimes();
expect(reservation.newTablet(ss, extent)).andReturn(tablet);
expect(reservation.getTabletMetadataExtents()).andReturn(Set.of(extent));
expect(reservation.getFailures()).andReturn(Map.of());
@@ -305,7 +305,7 @@ public class ScanServerTest {
};
TestScanServer ss = partialMockBuilder(TestScanServer.class).createMock();
-expect(extent.isMeta()).andReturn(false).anyTimes();
+expect(extent.isSystemTable()).andReturn(false).anyTimes();
expect(reservation.newTablet(ss, extent)).andReturn(tablet).anyTimes();
expect(reservation.getTabletMetadataExtents()).andReturn(Set.of());
expect(reservation.getFailures()).andReturn(Map.of(textent,
ranges)).anyTimes();
@@ -395,7 +395,7 @@ public class ScanServerTest {
TabletResolver resolver = createMock(TabletResolver.class);
TestScanServer ss = partialMockBuilder(TestScanServer.class).createMock();
-expect(sextent.isMeta()).andReturn(true).anyTimes();
+expect(sextent.isSystemTable()).andReturn(true).anyTimes();
expect(reservation.newTablet(ss, sextent)).andReturn(tablet);
expect(reservation.getFailures()).andReturn(Map.of()).anyTimes(
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 2a52d5d8e1fdc2f9d1d73296c9d6275160e8e8a0 Merge: 026fe416d5 59f560dd74 Author: Christopher L. Shannon AuthorDate: Fri Jun 21 18:07:35 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch elasticity updated (026fe416d5 -> 2a52d5d8e1)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 026fe416d5 Merge branch 'main' into elasticity add 59f560dd74 Fix typo (#4690) new 2a52d5d8e1 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch main updated: Fix typo (#4690)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 59f560dd74 Fix typo (#4690)
59f560dd74 is described below
commit 59f560dd7443295857304a6d5af956ffb5e0062f
Author: Christopher L. Shannon
AuthorDate: Fri Jun 21 18:02:48 2024 -0400
Fix typo (#4690)
---
.../apache/accumulo/test/functional/TabletStateChangeIteratorIT.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 727f13e74b..ed34aaf7cb 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -272,7 +272,7 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
// metadata should be stable with only 6 rows (2 for each table)
log.debug("Gathered {} rows to create copy {}", mutations.size(), copy);
assertEquals(7, mutations.size(),
-"Metadata should have 7 rows (2 for each table + " + "Scan ref
table)");
+"Metadata should have 7 rows (2 for each table Scan ref table)");
client.tableOperations().create(copy);
try (BatchWriter writer = client.createBatchWriter(copy)) {
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 026fe416d522313cd14aea5a35085f9f4aacd94e Merge: e4e9e8c94d 9d9175aea7 Author: Christopher L. Shannon AuthorDate: Fri Jun 21 17:54:55 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch elasticity updated (e4e9e8c94d -> 026fe416d5)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from e4e9e8c94d Merge branch 'main' into elasticity add 9d9175aea7 Test fixes to account for new Scan ref table (#4690) new 026fe416d5 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch main updated: Test fixes to account for new Scan ref table (#4690)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9d9175aea7 Test fixes to account for new Scan ref table (#4690)
9d9175aea7 is described below
commit 9d9175aea7a7894a43261302e749a04a169d0df5
Author: Christopher L. Shannon
AuthorDate: Fri Jun 21 17:50:51 2024 -0400
Test fixes to account for new Scan ref table (#4690)
---
test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java| 6 --
.../accumulo/test/functional/TabletStateChangeIteratorIT.java | 3 ++-
.../main/java/org/apache/accumulo/test/manager/MergeStateIT.java| 4 +++-
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
index c9aed19795..7bd28961df 100644
--- a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
@@ -189,8 +189,10 @@ public class MetaSplitIT extends AccumuloClusterHarness {
private void verifyMetadataTableScan(AccumuloClient client) throws Exception
{
var tables = client.tableOperations().tableIdMap();
var expectedExtents = tables.entrySet().stream()
-.filter(e ->
!e.getKey().startsWith("accumulo.")).map(Map.Entry::getValue).map(TableId::of)
-.map(tid -> new KeyExtent(tid, null,
null)).collect(Collectors.toSet());
+.filter(e -> !e.getKey().equals(AccumuloTable.ROOT.tableName())
+&& !e.getKey().equals(AccumuloTable.METADATA.tableName()))
+.map(Map.Entry::getValue).map(TableId::of).map(tid -> new
KeyExtent(tid, null, null))
+.collect(Collectors.toSet());
// Verify we have 11 tablets for metadata
assertEquals(11, expectedExtents.size());
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 09290f82ad..727f13e74b 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -271,7 +271,8 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
// metadata should be stable with only 6 rows (2 for each table)
log.debug("Gathered {} rows to create copy {}", mutations.size(), copy);
-assertEquals(6, mutations.size(), "Metadata should have 6 rows (2 for each
table)");
+assertEquals(7, mutations.size(),
+"Metadata should have 7 rows (2 for each table + " + "Scan ref
table)");
client.tableOperations().create(copy);
try (BatchWriter writer = client.createBatchWriter(copy)) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java
b/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java
index 3b0e648002..196beb6e7e 100644
--- a/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/manager/MergeStateIT.java
@@ -154,7 +154,9 @@ public class MergeStateIT extends ConfigurableMacBase {
count++;
}
}
- assertEquals(0, count); // the normal case is to skip tablets in a good
state
+
+ // There should be 1 because of the Scan Ref table
+ assertEquals(1, count); // the normal case is to skip tablets in a good
state
// Create the hole
// Split the tablet at one end of the range
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8449b155d2376923963ff232cd473c6c99e3212c Merge: 20681b37a0 0b220c1d86 Author: Christopher L. Shannon AuthorDate: Fri Jun 21 17:22:10 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch elasticity updated (20681b37a0 -> 8449b155d2)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 20681b37a0 Minor test updates and cleanup related to #4650 add d32d10a9f3 Validate the tablets form a linked list during merge (#4679) add 5b72725950 Merge branch '2.1' add 0b220c1d86 Move Scan Server File refs to their own table (#4690) new 8449b155d2 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch main updated: Move Scan Server File refs to their own table (#4690)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 0b220c1d86 Move Scan Server File refs to their own table (#4690)
0b220c1d86 is described below
commit 0b220c1d86e32123697d94e704309fef48c42dbf
Author: Christopher L. Shannon
AuthorDate: Fri Jun 21 16:48:12 2024 -0400
Move Scan Server File refs to their own table (#4690)
This change moves scan server refs to a separate table from metadata in
order to improve performance. The prefix was dropped as nothing else is
stored in the table anymore.
This is a backport of the change in elasticity that was done in #4650
into 3.1
---
.../accumulo/core/metadata/AccumuloTable.java | 2 +-
.../accumulo/core/metadata/ScanServerRefStore.java | 64
.../core/metadata/ScanServerRefTabletFile.java | 4 +-
.../accumulo/core/metadata/schema/Ample.java | 44 +---
.../core/metadata/schema/MetadataSchema.java | 13 ---
.../MiniAccumuloClusterImplTest.java | 5 +-
.../server/init/FileSystemInitializer.java | 27 -
.../accumulo/server/init/InitialConfiguration.java | 51 ++
.../accumulo/server/init/ZooKeeperInitializer.java | 11 +-
.../server/metadata/ScanServerRefStoreImpl.java| 113 +
.../accumulo/server/metadata/ServerAmpleImpl.java | 77 ++
.../server/util/ScanServerMetadataEntries.java | 8 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 2 +-
.../org/apache/accumulo/tserver/ScanServer.java| 8 +-
.../java/org/apache/accumulo/test/MetaSplitIT.java | 4 +-
.../org/apache/accumulo/test/NamespacesIT.java | 2 +-
.../test/ScanServerMetadataEntriesCleanIT.java | 8 +-
.../accumulo/test/ScanServerMetadataEntriesIT.java | 38 ---
.../apache/accumulo/test/ScanServerShutdownIT.java | 6 +-
.../accumulo/test/functional/WALSunnyDayIT.java| 6 +-
20 files changed, 294 insertions(+), 199 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
b/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
index 14b8b0cf30..929444fb3c 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/AccumuloTable.java
@@ -26,7 +26,7 @@ import org.apache.accumulo.core.data.TableId;
*/
public enum AccumuloTable {
- ROOT("root", "+r"), METADATA("metadata", "!0");
+ ROOT("root", "+r"), METADATA("metadata", "!0"), SCAN_REF("scanref",
"+scanref");
private final String name;
private final TableId tableId;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefStore.java
b/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefStore.java
new file mode 100644
index 00..461427fcd2
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefStore.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import java.util.Collection;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+public interface ScanServerRefStore {
+
+ /**
+ * Insert ScanServer references to Tablet files
+ *
+ * @param scanRefs set of scan server ref table file objects
+ */
+ default void put(Collection scanRefs) {
+throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get ScanServer references to Tablet files
+ *
+ * @return stream of scan server references
+ */
+ default Stream list() {
+throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Delete the set of scan server references
+ *
+ * @param refsToDelete set of scan server references to delete
+ */
+ default void delete(Collection refsToDelete) {
+throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Delete scan server references for this server
+
(accumulo) branch elasticity updated: Minor test updates and cleanup related to #4650
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 20681b37a0 Minor test updates and cleanup related to #4650
20681b37a0 is described below
commit 20681b37a01e97738d5c45b1c475f233aa071bb0
Author: Christopher L. Shannon
AuthorDate: Fri Jun 21 11:29:10 2024 -0400
Minor test updates and cleanup related to #4650
---
.../org/apache/accumulo/server/init/InitialConfiguration.java | 1 -
.../java/org/apache/accumulo/test/functional/WALSunnyDayIT.java | 8 ++--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
b/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
index 98a3ac8217..4b7adfe176 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/InitialConfiguration.java
@@ -94,7 +94,6 @@ class InitialConfiguration {
FateSchema.TxColumnFamily.STR_NAME);
initialFateTableConf.put(Property.TABLE_LOCALITY_GROUPS.getKey(),
"status");
-// TODO: What other configs do we want for this table?
initialScanRefTableConf.putAll(commonConfig);
int max = hadoopConf.getInt("dfs.replication.max", 512);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
index 5fb10db0c6..2d085c8893 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@ -149,14 +149,18 @@ public class WALSunnyDayIT extends ConfigurableMacBase {
Map> markers = getRecoveryMarkers(c);
// log.debug("markers " + markers);
// There should be markers for the created table and also the Fate and
ScanRef tables
- assertEquals(3, markers.size(), "two tablets should have markers");
+ assertEquals(3, markers.size(), "three tablets should have markers");
assertTrue(
markers.keySet().stream().anyMatch(extent ->
extent.tableId().canonical().equals("1")),
"tableId of the keyExtent should be 1");
assertTrue(
markers.keySet().stream()
.anyMatch(extent ->
extent.tableId().equals(AccumuloTable.FATE.tableId())),
- "tableId of the FateTable can't be found");
+ "tableId of the Fate table can't be found");
+ assertTrue(
+ markers.keySet().stream()
+ .anyMatch(extent ->
extent.tableId().equals(AccumuloTable.SCAN_REF.tableId())),
+ "tableId of the ScanRef table can't be found");
// put some data in the WAL
assertEquals(0, cluster.exec(SetGoalState.class,
"NORMAL").getProcess().waitFor());
(accumulo) branch main updated (32e4f3fda2 -> 5b72725950)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 32e4f3fda2 Merge branch '2.1' add d32d10a9f3 Validate the tablets form a linked list during merge (#4679) new 5b72725950 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/manager/TabletGroupWatcher.java | 78 +++--- 1 file changed, 68 insertions(+), 10 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5b7272595086ae92521f8f7e5195cf574a6dea13
Merge: 32e4f3fda2 d32d10a9f3
Author: Christopher L. Shannon
AuthorDate: Fri Jun 21 08:00:28 2024 -0400
Merge branch '2.1'
.../accumulo/manager/TabletGroupWatcher.java | 78 +++---
1 file changed, 68 insertions(+), 10 deletions(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 443df6c8f3,827f688c49..148882ae83
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -860,17 -797,18 +860,27 @@@ abstract class TabletGroupWatcher exten
AccumuloClient client = manager.getContext();
+KeyExtent stopExtent = KeyExtent.fromMetaRow(stop.toMetaRow());
++
+ // Used when scanning the table to track the extent of the previous
column.
+ // This value is updated for every column read at the end of the loop
below
+ // with the extent for the column. We scan multiple columns for each
tablet,
+ // so this is useful to detect when we have reached a different tablet.
+ KeyExtent prevColumnExtent = null;
+
+ // Used when scanning the table to track the previous tablet from the
+ // current one. This value will update whenever the current extent for
+ // the column read in the loop is different from the previously read
column,
+ // which is tracked by prevColumnExtent
KeyExtent previousKeyExtent = null;
- KeyExtent lastExtent = null;
+// Check if we have already previously fenced the tablets
+if (highTablet.isMerged()) {
+ Manager.log.debug("tablet metadata already fenced for merge {}", range);
+ // Return as we already fenced the files
+ return;
+}
+
try (BatchWriter bw = client.createBatchWriter(targetSystemTable)) {
long fileCount = 0;
// Make file entries in highest tablet
@@@ -892,79 -828,48 +902,100 @@@
Key key = entry.getKey();
Value value = entry.getValue();
+// Verify that Tablet is offline
+if (isTabletAssigned(key)) {
+ throw new IllegalStateException(
+ "Tablet " + key.getRow() + " is assigned during a merge!");
+ // Verify that Tablet has no WALs
+} else if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
+ throw new IllegalStateException("Tablet " + key.getRow() + " has
walogs during a merge!");
+}
+
final KeyExtent keyExtent = KeyExtent.fromMetaRow(key.getRow());
- // Keep track of the last Key Extent seen so we can use it to fence
- // of RFiles when merging the metadata
- if (lastExtent != null && !keyExtent.equals(lastExtent)) {
- previousKeyExtent = lastExtent;
-// Keep track of extents to verify the linked list.
++// Keep track of extents to verify the linked list and also we need
the
++// prevColumnExtent seen so we can use it to fence off RFiles when
merging
+ // 'keyExtent' represents the current tablet for this colum
+ // 'prevColumnExtent' is the extent seen from the previous column
read.
+ // 'previousKeyExtent' is the extent for the previous tablet
+ //
+ // If 'prevColumnExtent' is different from 'keyExtent' then we have
reached a new tablet
+ // and we can update 'previousKeyExtent' with the value from
'prevColumnExtent'
+ if (prevColumnExtent != null && !keyExtent.equals(prevColumnExtent)) {
+ previousKeyExtent = prevColumnExtent;
}
-// Verify that Tablet is offline
-if (isTabletAssigned(key)) {
- throw new IllegalStateException(
- "Tablet " + key.getRow() + " is assigned during a merge!");
- // Verify that Tablet has no WALs
-} else if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
- throw new IllegalStateException("Tablet " + key.getRow() + " has
walogs during a merge!");
-} else if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- m.put(key.getColumnFamily(), key.getColumnQualifier(), value);
+// Special case to handle the highest/stop tablet, which is where
files are
+// merged to. The existing merge code won't delete files from this
tablet
+// so we need to handle the deletes in this tablet when fencing files.
+// We may be able to make this simpler in the future.
+if (keyExtent.equals(stopExtent)) {
+ if (previousKeyExtent != null
+ && key.getColumnFamily().equals(DataFileColumnFami
(accumulo) branch 2.1 updated (2cc38dff5e -> d32d10a9f3)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 2cc38dff5e Back ported Timeout value to 2.1 (#4671) add d32d10a9f3 Validate the tablets form a linked list during merge (#4679) No new revisions were added by this update. Summary of changes: .../accumulo/manager/TabletGroupWatcher.java | 80 -- 1 file changed, 76 insertions(+), 4 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 28ddceead3547afeb0b171691bfbddc3bf107c97 Merge: ca407eff84 91db8a98d1 Author: Christopher L. Shannon AuthorDate: Fri Jun 14 13:53:38 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch elasticity updated (ca407eff84 -> 28ddceead3)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from ca407eff84 logs special handling of conditional mutations in ample (#4674) add 1867db49e7 Backport fix to make Import tablet idempotent (#4676) add 930a841885 Backport fix to make Import tablet idempotent (#4676) add 91db8a98d1 Merge branch '2.1' new 28ddceead3 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch main updated (1867db49e7 -> 91db8a98d1)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1867db49e7 Backport fix to make Import tablet idempotent (#4676) add 930a841885 Backport fix to make Import tablet idempotent (#4676) new 91db8a98d1 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch 2.1 updated: Backport fix to make Import tablet idempotent (#4676)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 930a841885 Backport fix to make Import tablet idempotent (#4676)
930a841885 is described below
commit 930a84188566f771d65ad36924cdada8dd5ad868
Author: Christopher L. Shannon
AuthorDate: Fri Jun 14 13:48:30 2024 -0400
Backport fix to make Import tablet idempotent (#4676)
This backports the fix from #4646 to make MoveExportedFiles fate step
idempotent.
This closes #4655
---
.../tableOps/tableImport/MoveExportedFiles.java| 18 +++---
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
index 87b4a62ef9..c717527619 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -71,24 +72,27 @@ class MoveExportedFiles extends ManagerRepo {
Function fileStatusName = fstat ->
fstat.getPath().getName();
- Set importing = Arrays.stream(exportedFiles).map(fileStatusName)
- .map(fileNameMappings::get).collect(Collectors.toSet());
+ Set importing =
+
Arrays.stream(exportedFiles).map(fileStatusName).map(fileNameMappings::get)
+
.filter(Objects::nonNull).map(Path::new).collect(Collectors.toSet());
- Set imported =
-
Arrays.stream(importedFiles).map(fileStatusName).collect(Collectors.toSet());
+ Set imported =
+
Arrays.stream(importedFiles).map(FileStatus::getPath).collect(Collectors.toSet());
if (log.isDebugEnabled()) {
log.debug("{} files already present in imported (target) directory:
{}", fmtTid,
-String.join(",", imported));
+
imported.stream().map(Path::getName).collect(Collectors.joining(",")));
}
- Set missingFiles = Sets.difference(new
HashSet<>(fileNameMappings.values()),
+ Set missingFiles = Sets.difference(
+
fileNameMappings.values().stream().map(Path::new).collect(Collectors.toSet()),
new HashSet<>(Sets.union(importing, imported)));
if (!missingFiles.isEmpty()) {
throw new
AcceptableThriftTableOperationException(tableInfo.tableId.canonical(),
tableInfo.tableName, TableOperation.IMPORT,
TableOperationExceptionType.OTHER,
-"Missing source files corresponding to files " + String.join(",",
missingFiles));
+"Missing source files corresponding to files "
++
missingFiles.stream().map(Path::getName).collect(Collectors.joining(",")));
}
for (FileStatus fileStatus : exportedFiles) {
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 91db8a98d1c738b038c55f022799a99ce3815bb8 Merge: 1867db49e7 930a841885 Author: Christopher L. Shannon AuthorDate: Fri Jun 14 13:51:23 2024 -0400 Merge branch '2.1'
(accumulo) branch main updated: Backport fix to make Import tablet idempotent (#4676)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1867db49e7 Backport fix to make Import tablet idempotent (#4676)
1867db49e7 is described below
commit 1867db49e7478041cdc0edc6f47c130dbb853c68
Author: Christopher L. Shannon
AuthorDate: Fri Jun 14 13:48:30 2024 -0400
Backport fix to make Import tablet idempotent (#4676)
This backports the fix from #4646 to make MoveExportedFiles fate step
idempotent.
This closes #4655
---
.../tableOps/tableImport/MoveExportedFiles.java| 18 +++---
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
index 87b4a62ef9..c717527619 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/MoveExportedFiles.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -71,24 +72,27 @@ class MoveExportedFiles extends ManagerRepo {
Function fileStatusName = fstat ->
fstat.getPath().getName();
- Set importing = Arrays.stream(exportedFiles).map(fileStatusName)
- .map(fileNameMappings::get).collect(Collectors.toSet());
+ Set importing =
+
Arrays.stream(exportedFiles).map(fileStatusName).map(fileNameMappings::get)
+
.filter(Objects::nonNull).map(Path::new).collect(Collectors.toSet());
- Set imported =
-
Arrays.stream(importedFiles).map(fileStatusName).collect(Collectors.toSet());
+ Set imported =
+
Arrays.stream(importedFiles).map(FileStatus::getPath).collect(Collectors.toSet());
if (log.isDebugEnabled()) {
log.debug("{} files already present in imported (target) directory:
{}", fmtTid,
-String.join(",", imported));
+
imported.stream().map(Path::getName).collect(Collectors.joining(",")));
}
- Set missingFiles = Sets.difference(new
HashSet<>(fileNameMappings.values()),
+ Set missingFiles = Sets.difference(
+
fileNameMappings.values().stream().map(Path::new).collect(Collectors.toSet()),
new HashSet<>(Sets.union(importing, imported)));
if (!missingFiles.isEmpty()) {
throw new
AcceptableThriftTableOperationException(tableInfo.tableId.canonical(),
tableInfo.tableName, TableOperation.IMPORT,
TableOperationExceptionType.OTHER,
-"Missing source files corresponding to files " + String.join(",",
missingFiles));
+"Missing source files corresponding to files "
++
missingFiles.stream().map(Path::getName).collect(Collectors.joining(",")));
}
for (FileStatus fileStatus : exportedFiles) {
(accumulo) branch elasticity updated: Fix TabletManagementIteratorIT after changes in (#4650)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 02a28f1374 Fix TabletManagementIteratorIT after changes in (#4650)
02a28f1374 is described below
commit 02a28f1374b4b9d9dea4151c3b477141ccb08113
Author: Christopher L. Shannon
AuthorDate: Sat Jun 8 14:40:17 2024 -0400
Fix TabletManagementIteratorIT after changes in (#4650)
---
.../org/apache/accumulo/test/functional/TabletManagementIteratorIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
index ab703d2753..fdf4a72cb6 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
@@ -527,7 +527,7 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
// metadata should be stable with only 9 rows (2 for each table)
// + 2 for the FateTable and ScanRef table
log.debug("Gathered {} rows to create copy {}", mutations.size(), copy);
-assertEquals(9, mutations.size(),
+assertEquals(10, mutations.size(),
"Metadata should have 8 rows (2 for each table) + one row for "
+ AccumuloTable.FATE.tableId().canonical());
client.tableOperations().create(copy);
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a064d05538c55f55984b14a0fedc368e2ff86331 Merge: a545e1e45e 7724f6ad7b Author: Christopher L. Shannon AuthorDate: Sat Jun 8 12:47:21 2024 -0400 Merge branch 'main' into elasticity .../apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java | 9 - 1 file changed, 4 insertions(+), 5 deletions(-) diff --cc minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java index 36f242f59a,1cd63b4889..b5c6667519 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java @@@ -75,12 -75,14 +75,11 @@@ public class MiniAccumuloConfigImpl private Map siteConfig = new HashMap<>(); private Map configuredSiteConig = new HashMap<>(); private Map clientProps = new HashMap<>(); - private int numTservers = 2; - private int numScanServers = 0; - private int numCompactors = 1; private Map memoryConfig = new HashMap<>(); - private final EnumMap> serverTypeClasses = new EnumMap>( - Map.of(MANAGER, Manager.class, GARBAGE_COLLECTOR, SimpleGarbageCollector.class, MONITOR, - Monitor.class, ZOOKEEPER, ZooKeeperServerMain.class, TABLET_SERVER, - TabletServer.class, SCAN_SERVER, ScanServer.class, COMPACTOR, Compactor.class)); + private final EnumMap> serverTypeClasses = + new EnumMap<>(Map.of(MANAGER, Manager.class, GARBAGE_COLLECTOR, SimpleGarbageCollector.class, + MONITOR, Monitor.class, ZOOKEEPER, ZooKeeperServerMain.class, TABLET_SERVER, + TabletServer.class, SCAN_SERVER, ScanServer.class, COMPACTOR, Compactor.class)); private boolean jdwpEnabled = false; private Map systemProperties = new HashMap<>();
(accumulo) branch elasticity updated (a545e1e45e -> a064d05538)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from a545e1e45e Update ITs to account for new scan ref table (#4650) add 10f45c4112 Backport FlakyFate and idempotent tests for Merge and DeleteRows (#4656) add 7724f6ad7b Merge branch '2.1' new a064d05538 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java | 9 - 1 file changed, 4 insertions(+), 5 deletions(-)
(accumulo) branch elasticity updated: Update ITs to account for new scan ref table (#4650)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new a545e1e45e Update ITs to account for new scan ref table (#4650)
a545e1e45e is described below
commit a545e1e45e37d6ffcf3b571f5ed599d27ce4f4c8
Author: Christopher L. Shannon
AuthorDate: Sat Jun 8 12:44:44 2024 -0400
Update ITs to account for new scan ref table (#4650)
---
test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java | 4 ++--
test/src/main/java/org/apache/accumulo/test/NamespacesIT.java | 2 +-
.../apache/accumulo/test/functional/TabletManagementIteratorIT.java | 3 ++-
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
index 5bbab1b173..885721a5e2 100644
--- a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
@@ -193,8 +193,8 @@ public class MetaSplitIT extends AccumuloClusterHarness {
&& !e.getKey().equals(AccumuloTable.METADATA.tableName()))
.map(Map.Entry::getValue).map(TableId::of).map(tid -> new
KeyExtent(tid, null, null))
.collect(Collectors.toSet());
-// Verify we have 11 tablets for metadata (Includes FateTable)
-assertEquals(11, expectedExtents.size());
+// Verify we have 12 tablets for metadata (Includes FateTable and ScanRef
table)
+assertEquals(12, expectedExtents.size());
// Scan each tablet to verify data exists
var ample = ((ClientContext) client).getAmple();
diff --git a/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
b/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
index 80d47f23d3..0835c43346 100644
--- a/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/NamespacesIT.java
@@ -134,7 +134,7 @@ public class NamespacesIT extends SharedMiniClusterBase {
c.tableOperations().delete(t);
}
}
-assertEquals(3, c.tableOperations().list().size());
+assertEquals(4, c.tableOperations().list().size());
for (String n : c.namespaceOperations().list()) {
if (!n.equals(Namespace.ACCUMULO.name()) &&
!n.equals(Namespace.DEFAULT.name())) {
c.namespaceOperations().delete(n);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
index fc6d42fe35..ab703d2753 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java
@@ -524,7 +524,8 @@ public class TabletManagementIteratorIT extends
AccumuloClusterHarness {
}
}
-// metadata should be stable with only 9 rows (2 for each table) + 1 for
the FateTable
+// metadata should be stable with only 9 rows (2 for each table)
+// + 2 for the FateTable and ScanRef table
log.debug("Gathered {} rows to create copy {}", mutations.size(), copy);
assertEquals(9, mutations.size(),
"Metadata should have 8 rows (2 for each table) + one row for "
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 7724f6ad7bb3a4cb00c8d4c8af042f8bc5efcf0c
Merge: 69584e2835 10f45c4112
Author: Christopher L. Shannon
AuthorDate: Sat Jun 8 12:34:22 2024 -0400
Merge branch '2.1'
.../java/org/apache/accumulo/core/fate/Fate.java | 7 +++
.../MiniAccumuloClusterControl.java| 30 +--
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 40 +++
.../java/org/apache/accumulo/manager/Manager.java | 10 +++-
.../org/apache/accumulo/test/fate/FlakyFate.java | 59 ++
.../accumulo/test/fate/FlakyFateManager.java | 49 ++
.../test/functional/DeleteRowsFlakyFateIT.java | 35 +
.../accumulo/test/functional/MergeFlakyFateIT.java | 36 +
8 files changed, 248 insertions(+), 18 deletions(-)
diff --cc
minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index 9e395bb47c,e40679ebb0..68321cdf4b
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@@ -42,19 -41,15 +42,14 @@@ import org.apache.accumulo.core.conf.Pr
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
- import org.apache.accumulo.gc.SimpleGarbageCollector;
- import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.minicluster.ServerType;
import
org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo;
- import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.tserver.ScanServer;
- import org.apache.accumulo.tserver.TabletServer;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransportException;
- import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -205,6 -195,7 +200,7 @@@ public class MiniAccumuloClusterContro
start(server, Collections.emptyMap(), Integer.MAX_VALUE);
}
- @SuppressWarnings(value = {"removal", "unchecked"})
++ @SuppressWarnings("unchecked")
public synchronized void start(ServerType server, Map
configOverrides, int limit)
throws IOException {
if (limit <= 0) {
@@@ -222,9 -215,10 +220,9 @@@
}
}
break;
- case MASTER:
case MANAGER:
if (managerProcess == null) {
- managerProcess = cluster._exec(Manager.class, server,
configOverrides).getProcess();
+ managerProcess = cluster._exec(classToUse, server,
configOverrides).getProcess();
}
break;
case ZOOKEEPER:
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index a778744c0f,92c3453e45..82cf1ea91a
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -72,6 -69,10 +72,7 @@@ import org.apache.accumulo.core.data.Va
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.fate.AgeOffStore;
import org.apache.accumulo.core.fate.Fate;
+ import org.apache.accumulo.core.fate.TStore;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock.LockLossReason;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock.ServiceLockPath;
import org.apache.accumulo.core.fate.zookeeper.ZooCache.ZcStat;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
@@@ -422,7 -416,7 +423,7 @@@ public class Manager extends AbstractSe
}
}
- Manager(ConfigOpts opts, String[] args) throws IOException {
- protected Manager(ServerOpts opts, String[] args) throws IOException {
++ protected Manager(ConfigOpts opts, String[] args) throws IOException {
super("manager", opts, args);
ServerContext context = super.getContext();
balancerEnvironment = new BalancerEnvironmentImpl(context);
@@@ -1253,7 -1240,8 +1254,7 @@@
context.getZooReaderWriter()),
HOURS.toMillis(8), System::currentTimeMillis);
- Fate f = new Fate<>(this, store, TraceRepo::toLogString,
getConfiguration());
- Fate f = initializeFateInstance(store);
- f.startTransactionRunners(getConfiguration());
++ Fate f = initializeFateInstance(
(accumulo) branch main updated (69584e2835 -> 7724f6ad7b)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 69584e2835 Merge branch '2.1'
add 10f45c4112 Backport FlakyFate and idempotent tests for Merge and
DeleteRows (#4656)
new 7724f6ad7b Merge branch '2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/accumulo/core/fate/Fate.java | 7 +++
.../MiniAccumuloClusterControl.java| 30 +--
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 40 +++
.../java/org/apache/accumulo/manager/Manager.java | 10 +++-
.../org/apache/accumulo/test/fate/FlakyFate.java | 59 ++
.../FlakyFateManager.java} | 32 ++--
.../functional/DeleteRowsFlakyFateIT.java} | 27 --
.../functional/MergeFlakyFateIT.java} | 26 --
8 files changed, 163 insertions(+), 68 deletions(-)
create mode 100644
test/src/main/java/org/apache/accumulo/test/fate/FlakyFate.java
copy
test/src/main/java/org/apache/accumulo/test/{compaction/ExternalCompactionTServer.java
=> fate/FlakyFateManager.java} (53%)
copy
test/src/main/java/org/apache/accumulo/{harness/MiniClusterConfigurationCallback.java
=> test/functional/DeleteRowsFlakyFateIT.java} (60%)
copy
test/src/main/java/org/apache/accumulo/{harness/MiniClusterConfigurationCallback.java
=> test/functional/MergeFlakyFateIT.java} (61%)
(accumulo) branch 2.1 updated: Backport FlakyFate and idempotent tests for Merge and DeleteRows (#4656)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 10f45c4112 Backport FlakyFate and idempotent tests for Merge and
DeleteRows (#4656)
10f45c4112 is described below
commit 10f45c4112458786df137d4125f0e1fd32ca8262
Author: Christopher L. Shannon
AuthorDate: Sat Jun 8 12:32:23 2024 -0400
Backport FlakyFate and idempotent tests for Merge and DeleteRows (#4656)
This backports the FlakyFate and FlakyFateManager impl from elasticity
that was added in #4643 so that fate operations can be easily tested to
check if they are idempotent. DeleteRowsFlakyFateIT and MergeFlakyFateIT
were also backported and pass verifying the operations are idempotent.
---
.../java/org/apache/accumulo/core/fate/Fate.java | 7 +++
.../MiniAccumuloClusterControl.java| 31 ++--
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 40 +++
.../java/org/apache/accumulo/manager/Manager.java | 9 +++-
.../org/apache/accumulo/test/fate/FlakyFate.java | 57 ++
.../accumulo/test/fate/FlakyFateManager.java | 47 ++
.../test/functional/DeleteRowsFlakyFateIT.java | 35 +
.../accumulo/test/functional/MergeFlakyFateIT.java | 36 ++
8 files changed, 243 insertions(+), 19 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
b/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
index 8dadac916e..4eb690e5b2 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
@@ -215,7 +215,14 @@ public class Fate {
log.warn("Failed to undo Repo, " + FateTxId.formatTid(tid), e);
}
}
+ }
+
+ protected long executeIsReady(Long tid, Repo op) throws Exception {
+return op.isReady(tid, environment);
+ }
+ protected Repo executeCall(Long tid, Repo op) throws Exception {
+return op.call(tid, environment);
}
/**
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index 69f6f64297..e40679ebb0 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@ -44,17 +44,12 @@ import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
-import org.apache.accumulo.gc.SimpleGarbageCollector;
-import org.apache.accumulo.manager.Manager;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl.ProcessInfo;
-import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.tserver.ScanServer;
-import org.apache.accumulo.tserver.TabletServer;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransportException;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -200,13 +195,15 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
start(server, Collections.emptyMap(), Integer.MAX_VALUE);
}
- @SuppressWarnings("removal")
+ @SuppressWarnings(value = {"removal", "unchecked"})
public synchronized void start(ServerType server, Map
configOverrides, int limit)
throws IOException {
if (limit <= 0) {
return;
}
+Class classToUse = cluster.getConfig().getServerClass(server);
+
switch (server) {
case TABLET_SERVER:
synchronized (tabletServerProcesses) {
@@ -214,31 +211,31 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
for (int i = tabletServerProcesses.size();
count < limit && i < cluster.getConfig().getNumTservers(); i++,
++count) {
tabletServerProcesses
-.add(cluster._exec(TabletServer.class, server,
configOverrides).getProcess());
+.add(cluster._exec(classToUse, server,
configOverrides).getProcess());
}
}
break;
case MASTER:
case MANAGER:
if (managerProcess == null) {
- managerProcess = cluster._exec(Manager.class, server,
configOverrides).getProcess();
+ managerProcess = cluster._exec(classToUse, server,
configOverrides).getProcess();
}
break;
case ZOOKEEPER:
if (zooKeeperProce
(accumulo) branch elasticity updated (9d46575487 -> f1a200aaaa)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9d46575487 Added caching for CompactionServiceId (#4325) add f1a200 Move Scan Server File refs to their own table (#4650) No new revisions were added by this update. Summary of changes: .../accumulo/core/metadata/AccumuloTable.java | 5 +- .../accumulo/core/metadata/ScanServerRefStore.java | 64 .../core/metadata/ScanServerRefTabletFile.java | 4 +- .../accumulo/core/metadata/schema/Ample.java | 45 +--- .../core/metadata/schema/MetadataSchema.java | 13 --- .../MiniAccumuloClusterImplTest.java | 5 +- .../server/init/FileSystemInitializer.java | 18 +++- .../accumulo/server/init/InitialConfiguration.java | 9 ++ .../accumulo/server/init/ZooKeeperInitializer.java | 14 +-- .../server/metadata/ScanServerRefStoreImpl.java| 113 + .../accumulo/server/metadata/ServerAmpleImpl.java | 83 ++- .../server/util/ScanServerMetadataEntries.java | 8 +- .../main/java/org/apache/accumulo/gc/GCRun.java| 2 +- .../org/apache/accumulo/tserver/ScanServer.java| 8 +- .../test/ScanServerMetadataEntriesCleanIT.java | 8 +- .../accumulo/test/ScanServerMetadataEntriesIT.java | 38 --- .../apache/accumulo/test/ScanServerShutdownIT.java | 6 +- .../accumulo/test/functional/WALSunnyDayIT.java| 4 +- 18 files changed, 263 insertions(+), 184 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/metadata/ScanServerRefStore.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metadata/ScanServerRefStoreImpl.java
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c76f5f75782375a7bc58a49905c0b3834992f3a6 Merge: d7fcb8493a 73125e90ef Author: Christopher L. Shannon AuthorDate: Sun Jun 2 11:15:29 2024 -0400 Merge branch 'main' into elasticity test/src/main/java/org/apache/accumulo/test/ImportExportIT.java | 9 ++--- test/src/main/java/org/apache/accumulo/test/SampleIT.java| 6 +++--- .../apache/accumulo/test/compaction/ExternalCompaction_1_IT.java | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-)
(accumulo) branch elasticity updated (d7fcb8493a -> c76f5f7578)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from d7fcb8493a Merge branch 'main' into elasticity add 73125e90ef Update tests to handle new range validation added in (#4626) new c76f5f7578 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: test/src/main/java/org/apache/accumulo/test/ImportExportIT.java | 9 ++--- test/src/main/java/org/apache/accumulo/test/SampleIT.java| 6 +++--- .../apache/accumulo/test/compaction/ExternalCompaction_1_IT.java | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-)
(accumulo) branch main updated: Update tests to handle new range validation added in (#4626)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 73125e90ef Update tests to handle new range validation added in (#4626)
73125e90ef is described below
commit 73125e90efda2a79796172915e534a0216361764
Author: Christopher L. Shannon
AuthorDate: Sun Jun 2 11:14:35 2024 -0400
Update tests to handle new range validation added in (#4626)
---
test/src/main/java/org/apache/accumulo/test/ImportExportIT.java | 9 ++---
test/src/main/java/org/apache/accumulo/test/SampleIT.java| 6 +++---
.../apache/accumulo/test/compaction/ExternalCompaction_1_IT.java | 6 +++---
3 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
index 4fc909c4d5..8e5b076557 100644
--- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
@@ -463,8 +463,11 @@ public class ImportExportIT extends AccumuloClusterHarness
{
private Set createRanges() {
// Split file into ranges of 1, 2, and 5000 for a total of 35000
return Set.of(
-new Range("row_" + String.format("%010d", 100), "row_" +
String.format("%010d", 199)),
-new Range("row_" + String.format("%010d", 300), "row_" +
String.format("%010d", 499)),
-new Range("row_" + String.format("%010d", 700), "row_" +
String.format("%010d", 749)));
+new Range("row_" + String.format("%010d", 99), false, "row_" +
String.format("%010d", 199),
+true),
+new Range("row_" + String.format("%010d", 299), false, "row_" +
String.format("%010d", 499),
+true),
+new Range("row_" + String.format("%010d", 699), false, "row_" +
String.format("%010d", 749),
+true));
}
}
diff --git a/test/src/main/java/org/apache/accumulo/test/SampleIT.java
b/test/src/main/java/org/apache/accumulo/test/SampleIT.java
index c4f4a5fbd6..30908254db 100644
--- a/test/src/main/java/org/apache/accumulo/test/SampleIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/SampleIT.java
@@ -160,8 +160,8 @@ public class SampleIT extends AccumuloClusterHarness {
client.tableOperations().flush(tableName, null, null, true);
// Fence off the data to a Range that is a subset of the original data
-Range fenced = new Range(new Text(String.format("r_%06d", 3000)),
-new Text(String.format("r_%06d", 6000)));
+Range fenced = new Range(new Text(String.format("r_%06d", 2999)),
false,
+new Text(String.format("r_%06d", 6000)), true);
FileMetadataUtil.splitFilesIntoRanges(getServerContext(), tableName,
Set.of(fenced));
assertEquals(1, countFiles(getServerContext(), tableName));
@@ -620,7 +620,7 @@ public class SampleIT extends AccumuloClusterHarness {
for (int i = 0; i < splits; i++) {
Text start = i > 0 ? new Text(String.format("r_%06d", i * 1000)) : null;
Text end = i < splits - 1 ? new Text(String.format("r_%06d", (i + 1) *
1000)) : null;
- ranges.add(new Range(start, end));
+ ranges.add(new Range(start, false, end, true));
}
return ranges;
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
index a1a85e7b1c..f712777e85 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_1_IT.java
@@ -473,13 +473,13 @@ public class ExternalCompaction_1_IT extends
SharedMiniClusterBase {
// Split file in table1 into two files each fenced off by 100 rows for a
total of 200
splitFilesIntoRanges(getCluster().getServerContext(), table1,
- Set.of(new Range(new Text(row(100)), new Text(row(199))),
- new Range(new Text(row(300)), new Text(row(399);
+ Set.of(new Range(new Text(row(99)), false, new Text(row(199)), true),
+ new Range(new Text(row(299)), false, new Text(row(399)), true)));
assertEquals(2, countFencedFiles(getCluster().getServerContext(),
table1));
// Fence file in table2 to 600 rows
splitFilesIntoRanges(getCluster().getServerContext(), table2,
- Set.of(new Range(new Text(row(200)), new Text(row(799);
+ Set.of(new Range(new Text(row(199)), false, new Text(row(799)),
true)));
assertEquals(1, countFencedFiles(getCluster().getServerContext(),
table2));
// Verify that a subset of the data is now seen after fencing
(accumulo) branch elasticity updated (7139931de4 -> d7fcb8493a)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7139931de4 adds test to cover all conditional updates and fixes bug (#4624) add 53683e4701 Implement estimateOverlappingEntries in fenced files (#4626) new d7fcb8493a Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/core/client/rfile/RFile.java | 4 +- .../apache/accumulo/core/dataImpl/KeyExtent.java | 61 .../org/apache/accumulo/core/file/rfile/RFile.java | 8 +- .../accumulo/core/metadata/AbstractTabletFile.java | 48 ++-- .../accumulo/core/metadata/StoredTabletFile.java | 7 +- .../apache/accumulo/core/util/RowRangeUtil.java| 85 ++ .../core/client/rfile/RFileClientTest.java | 11 +-- .../apache/accumulo/core/data/KeyExtentTest.java | 43 +++ .../core/file/rfile/AbstractRFileTest.java | 37 +- .../accumulo/core/file/rfile/FencedRFileTest.java | 72 +- .../apache/accumulo/core/file/rfile/RFileTest.java | 33 + .../metadata/schema/ReferencedTabletFileTest.java | 22 -- .../core/metadata/schema/TabletMetadataTest.java | 12 +-- .../constraints/MetadataConstraintsTest.java | 15 ++-- .../java/org/apache/accumulo/test/CloneIT.java | 3 +- .../java/org/apache/accumulo/test/VolumeIT.java| 7 +- 16 files changed, 389 insertions(+), 79 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/util/RowRangeUtil.java
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit d7fcb8493ab3be238a4bb567eed7552aade802aa
Merge: 7139931de4 53683e4701
Author: Christopher L. Shannon
AuthorDate: Sat Jun 1 15:57:26 2024 -0400
Merge branch 'main' into elasticity
.../apache/accumulo/core/client/rfile/RFile.java | 4 +-
.../apache/accumulo/core/dataImpl/KeyExtent.java | 61
.../org/apache/accumulo/core/file/rfile/RFile.java | 8 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 48 ++--
.../accumulo/core/metadata/StoredTabletFile.java | 7 +-
.../apache/accumulo/core/util/RowRangeUtil.java| 85 ++
.../core/client/rfile/RFileClientTest.java | 11 +--
.../apache/accumulo/core/data/KeyExtentTest.java | 43 +++
.../core/file/rfile/AbstractRFileTest.java | 37 +-
.../accumulo/core/file/rfile/FencedRFileTest.java | 72 +-
.../apache/accumulo/core/file/rfile/RFileTest.java | 33 +
.../metadata/schema/ReferencedTabletFileTest.java | 22 --
.../core/metadata/schema/TabletMetadataTest.java | 12 +--
.../constraints/MetadataConstraintsTest.java | 15 ++--
.../java/org/apache/accumulo/test/CloneIT.java | 3 +-
.../java/org/apache/accumulo/test/VolumeIT.java| 7 +-
16 files changed, 389 insertions(+), 79 deletions(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 06571407e3,8c3d84de2c..d22925afc3
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -431,142 -408,10 +431,142 @@@ public class TabletMetadataTest
assertEquals(1, tm2.getScans().size());
assertThrows(UnsupportedOperationException.class, () ->
tm2.getScans().add(stf));
assertEquals(1, tm2.getLoaded().size());
-assertThrows(UnsupportedOperationException.class, () ->
tm2.getLoaded().put(stf, 0L));
+assertThrows(UnsupportedOperationException.class,
+() -> tm2.getLoaded().put(stf, FateId.from(FateInstanceType.USER,
UUID.randomUUID(;
assertEquals(1, tm2.getKeyValues().size());
assertThrows(UnsupportedOperationException.class, () ->
tm2.getKeyValues().remove(null));
+assertEquals(1, tm2.getCompacted().size());
+assertThrows(UnsupportedOperationException.class,
+() -> tm2.getCompacted().add(FateId.from(FateInstanceType.USER,
UUID.randomUUID(;
+assertEquals(1, tm2.getUserCompactionsRequested().size());
+assertThrows(UnsupportedOperationException.class, () ->
tm2.getUserCompactionsRequested()
+.add(FateId.from(FateInstanceType.USER, UUID.randomUUID(;
+ }
+
+ @Test
+ public void testCompactionRequestedColumn() {
+KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+FateInstanceType type = FateInstanceType.fromTableId(extent.tableId());
+FateId userCompactFateId1 = FateId.from(type, UUID.randomUUID());
+FateId userCompactFateId2 = FateId.from(type, UUID.randomUUID());
+
+// Test column set
+Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId1.canonical(), "");
+mutation.put(UserCompactionRequestedColumnFamily.STR_NAME,
userCompactFateId2.canonical(), "");
+
+TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+assertEquals(2, tm.getUserCompactionsRequested().size());
+assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId1));
+assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId2));
+
+// Column not set
+mutation = TabletColumnFamily.createPrevRowMutation(extent);
+tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(USER_COMPACTION_REQUESTED), true, false);
+assertTrue(tm.getUserCompactionsRequested().isEmpty());
+
+// Column not fetched
+mutation = TabletColumnFamily.createPrevRowMutation(extent);
+tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.PREV_ROW), true, false);
+assertThrows(IllegalStateException.class,
tm::getUserCompactionsRequested);
+ }
+
+ @Test
+ public void testUnsplittableColumn() {
+KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+StoredTabletFile sf1 = StoredTabletFile.of(new
Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf"));
+StoredTabletFile sf2 = StoredTabletFile.of(new
Path
(accumulo) branch main updated: Implement estimateOverlappingEntries in fenced files (#4626)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 53683e4701 Implement estimateOverlappingEntries in fenced files (#4626)
53683e4701 is described below
commit 53683e4701d1f049e6c7698489a3b8716ad5b143
Author: Christopher L. Shannon
AuthorDate: Sat Jun 1 15:56:58 2024 -0400
Implement estimateOverlappingEntries in fenced files (#4626)
This adds support for estimating overlapping entries in fenced RFiles.
This works by clipping the KeyExtent provided with the fenced file range
to find the overlapping section to use for computing the estimated
entries. Row range validation has been narrowed and renamed to
validate that the range is what will be generated by a KeyExtent
toDataRange
This closes #4611
-
Co-authored-by: Keith Turner
---
.../apache/accumulo/core/client/rfile/RFile.java | 4 +-
.../apache/accumulo/core/dataImpl/KeyExtent.java | 61
.../org/apache/accumulo/core/file/rfile/RFile.java | 8 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 48 ++--
.../accumulo/core/metadata/StoredTabletFile.java | 7 +-
.../apache/accumulo/core/util/RowRangeUtil.java| 85 ++
.../core/client/rfile/RFileClientTest.java | 11 +--
.../apache/accumulo/core/data/KeyExtentTest.java | 43 +++
.../core/file/rfile/AbstractRFileTest.java | 37 +-
.../accumulo/core/file/rfile/FencedRFileTest.java | 72 +-
.../apache/accumulo/core/file/rfile/RFileTest.java | 33 +
.../metadata/schema/ReferencedTabletFileTest.java | 22 --
.../constraints/MetadataConstraintsTest.java | 15 ++--
.../java/org/apache/accumulo/test/CloneIT.java | 3 +-
.../java/org/apache/accumulo/test/VolumeIT.java| 7 +-
15 files changed, 383 insertions(+), 73 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
index 9b61e53c21..64956dcbc2 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
@@ -36,8 +36,8 @@ import
org.apache.accumulo.core.client.summary.Summary.FileStatistics;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.metadata.AbstractTabletFile;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.RowRangeUtil;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -100,7 +100,7 @@ public class RFile {
public FencedPath(Path path, Range fence) {
this.path = Objects.requireNonNull(path);
-this.fence = AbstractTabletFile.requireRowRange(fence);
+this.fence = RowRangeUtil.requireKeyExtentDataRange(fence);
}
public Path getPath() {
diff --git
a/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
b/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
index 434c0f7b89..b5bdb1f744 100644
--- a/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/dataImpl/KeyExtent.java
@@ -20,6 +20,8 @@ package org.apache.accumulo.core.dataImpl;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
+import static
org.apache.accumulo.core.util.RowRangeUtil.requireKeyExtentDataRange;
+import static org.apache.accumulo.core.util.RowRangeUtil.stripZeroTail;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
@@ -388,6 +390,65 @@ public class KeyExtent implements Comparable {
return new Range(metadataPrevRow, prevEndRow() == null, toMetaRow(), true);
}
+ /**
+ * Creates a KeyExtent which represents the intersection of this KeyExtent
and the passed in
+ * range.
+ *
+ * Note: The range provided must be a range that is derived from a
KeyExtent. This means
+ * the range must be in the format of a row range and also requires an
exclusive start key, which
+ * is the format that {@link #toDataRange()} uses
+ *
+ * @param range range to clip to
+ * @return the intersection of this KeyExtent and the given range
+ * @throws IllegalArgumentException if the KeyExtent and range do not overlap
+ */
+ public KeyExtent clip(Range range) {
+return clip(range, false);
+ }
+
+ /**
+ * Creates a KeyExtent which represents the intersection of this KeyExtent
and the passed in
+ * range. Unlike {@link #clip(Range)}, this method can optionally return
null if the given range
+ * and this KeyExtent do
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f5885481e32f6e7ffc41d391530654156e1a8522
Merge: 8e5d967f24 5d3ccec0df
Author: Christopher L. Shannon
AuthorDate: Sat May 25 08:08:36 2024 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 11 +++-
.../org/apache/accumulo/tserver/ScanServer.java| 37 +++-
.../test/ScanServerConcurrentTabletScanIT.java | 69 --
3 files changed, 107 insertions(+), 10 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 463b37cbcf,fe4b8f4c18..aa6533bb53
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -428,17 -445,19 +428,26 @@@ public enum Property
SSERV_DEFAULT_BLOCKSIZE("sserver.default.blocksize", "1M",
PropertyType.BYTES,
"Specifies a default blocksize for the scan server caches.", "2.1.0"),
@Experimental
+ SSERV_GROUP_NAME("sserver.group",
ScanServerSelector.DEFAULT_SCAN_SERVER_GROUP_NAME,
+ PropertyType.STRING,
+ "Optional group name that will be made available to the "
+ + "ScanServerSelector client plugin. Groups support at least two
use cases:"
+ + " dedicating resources to scans and/or using different hardware
for scans.",
+ "3.0.0"),
+ @Experimental
SSERV_CACHED_TABLET_METADATA_EXPIRATION("sserver.cache.metadata.expiration",
"5m",
- PropertyType.TIMEDURATION, "The time after which cached tablet metadata
will be refreshed.",
+ PropertyType.TIMEDURATION,
+ "The time after which cached tablet metadata will be expired if not
previously refreshed.",
"2.1.0"),
@Experimental
+
SSERV_CACHED_TABLET_METADATA_REFRESH_PERCENT("sserver.cache.metadata.refresh.percent",
".75",
+ PropertyType.FRACTION,
+ "The time after which cached tablet metadata will be refreshed,
expressed as a "
+ + "percentage of the expiration time. Cache hits after this time,
but before the "
+ + "expiration time, will trigger a background refresh for future
hits. "
+ + "Value must be less than 100%. Set to 0 will disable refresh.",
+ "2.1.3"),
+ @Experimental
SSERV_PORTSEARCH("sserver.port.search", "true", PropertyType.BOOLEAN,
"if the ports above are in use, search higher ports until one is
available.", "2.1.0"),
@Experimental
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 616800168e,3b52ecf0fd..201bfada4f
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@@ -416,7 -439,12 +442,12 @@@ public class ScanServer extends Abstrac
LOG.warn("Failed to close filesystem : {}", e.getMessage(), e);
}
+ if (tmCacheExecutor != null) {
+ LOG.debug("Shutting down TabletMetadataCache executor");
+ tmCacheExecutor.shutdownNow();
+ }
+
- gcLogger.logGCInfo(getConfiguration());
+ context.getLowMemoryDetector().logGCInfo(getConfiguration());
LOG.info("stop requested. exiting ... ");
try {
if (null != lock) {
(accumulo) branch main updated (8e5d967f24 -> f5885481e3)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8e5d967f24 Merge remote-tracking branch 'upstream/2.1' add 5d3ccec0df Enable background refresh for the scan server tablet metadata cache (#4551) new f5885481e3 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/conf/Property.java| 11 +++- .../org/apache/accumulo/tserver/ScanServer.java| 37 +++- .../test/ScanServerConcurrentTabletScanIT.java | 69 -- 3 files changed, 107 insertions(+), 10 deletions(-)
(accumulo) branch 2.1 updated: Enable background refresh for the scan server tablet metadata cache (#4551)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 5d3ccec0df Enable background refresh for the scan server tablet
metadata cache (#4551)
5d3ccec0df is described below
commit 5d3ccec0dfed42667608a574ae2cb4b12ffa1987
Author: Christopher L. Shannon
AuthorDate: Sat May 25 08:07:35 2024 -0400
Enable background refresh for the scan server tablet metadata cache (#4551)
This adds a property to configure the scan server tablet metadata
Caffeine cache to refresh cached tablet metadata in the background on
cache hits after the refresh time has passed. The refresh time is
expressed as a percentage of the expiration time. This allows the cached
entries to refresh before expiration if they are frequently used so that
scans will not be blocked waiting on a refresh on expiration. Entries
still expire if no cache hits come after the refresh time and expiration
time passes.
See: https://github.com/ben-manes/caffeine/wiki/Refresh
This closes #4544
---
.../org/apache/accumulo/core/conf/Property.java| 11 +++-
.../org/apache/accumulo/tserver/ScanServer.java| 37 +++-
.../test/ScanServerConcurrentTabletScanIT.java | 69 --
3 files changed, 107 insertions(+), 10 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index fc5a52f239..fe4b8f4c18 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -446,9 +446,18 @@ public enum Property {
"Specifies a default blocksize for the scan server caches.", "2.1.0"),
@Experimental
SSERV_CACHED_TABLET_METADATA_EXPIRATION("sserver.cache.metadata.expiration",
"5m",
- PropertyType.TIMEDURATION, "The time after which cached tablet metadata
will be refreshed.",
+ PropertyType.TIMEDURATION,
+ "The time after which cached tablet metadata will be expired if not
previously refreshed.",
"2.1.0"),
@Experimental
+
SSERV_CACHED_TABLET_METADATA_REFRESH_PERCENT("sserver.cache.metadata.refresh.percent",
".75",
+ PropertyType.FRACTION,
+ "The time after which cached tablet metadata will be refreshed,
expressed as a "
+ + "percentage of the expiration time. Cache hits after this time,
but before the "
+ + "expiration time, will trigger a background refresh for future
hits. "
+ + "Value must be less than 100%. Set to 0 will disable refresh.",
+ "2.1.3"),
+ @Experimental
SSERV_PORTSEARCH("sserver.port.search", "true", PropertyType.BOOLEAN,
"if the ports above are in use, search higher ports until one is
available.", "2.1.0"),
@Experimental
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 2ddb76e2cb..3b52ecf0fd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@ -39,6 +39,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
@@ -179,6 +180,7 @@ public class ScanServer extends AbstractServer
private UUID serverLockUUID;
private final TabletMetadataLoader tabletMetadataLoader;
private final LoadingCache tabletMetadataCache;
+ private final ThreadPoolExecutor tmCacheExecutor;
// tracks file reservations that are in the process of being added or
removed from the metadata
// table
private final Set influxFiles = new HashSet<>();
@@ -242,14 +244,38 @@ public class ScanServer extends AbstractServer
if (cacheExpiration == 0L) {
LOG.warn("Tablet metadata caching disabled, may cause excessive scans on
metadata table.");
tabletMetadataCache = null;
+ tmCacheExecutor = null;
} else {
if (cacheExpiration < 6) {
LOG.warn(
"Tablet metadata caching less than one minute, may cause excessive
scans on metadata table.");
}
- tabletMetadataCache =
- Caffeine.newBuilder().expireAfterWrite(cacheExpiration,
TimeUnit.MILLISECONDS)
-
.scheduler(Scheduler.systemScheduler()).recordStats().build(tabletMetadataLoade
(accumulo) branch elasticity updated: Allow system compactions to run if zero user compaction jobs have run (#4480)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 9d4dc21abc Allow system compactions to run if zero user compaction
jobs have run (#4480)
9d4dc21abc is described below
commit 9d4dc21abce1e55daf21f06fdf87f1cffac743ef
Author: Christopher L. Shannon
AuthorDate: Sat May 18 12:00:12 2024 -0400
Allow system compactions to run if zero user compaction jobs have run
(#4480)
This change will allow system compactions to postpone user compactions
that have had no jobs run yet. Before this, if a user compaction was in
the queue and had selected files that overlapped it would block system
compactions from running. Now if there are selected files, but the user
compaction is not running and hasn't had any jobs completed, the
coordinator will clear the selectedFiles column so that the system
compaction can run if the expiration time has passed. The fate operation
will reset the column again while trying to make progress.
This closes #4454
Co-authored-by: Keith Turner
---
.../core/metadata/schema/SelectedFiles.java| 57 +--
.../core/metadata/schema/SelectedFilesTest.java| 47 +++---
.../core/metadata/schema/TabletMetadataTest.java | 3 +-
.../server/compaction/CompactionJobGenerator.java | 25 ++-
.../manager/state/TabletManagementIterator.java| 4 +-
.../manager/state/TabletManagementParameters.java | 14 +-
.../constraints/MetadataConstraintsTest.java | 3 +-
.../state/TabletManagementParametersTest.java | 6 +-
.../java/org/apache/accumulo/manager/Manager.java | 2 +-
.../accumulo/manager/TabletGroupWatcher.java | 8 +-
.../coordinator/CompactionCoordinator.java | 50 +-
.../coordinator/commit/CommitCompaction.java | 7 +-
.../manager/tableOps/compact/CompactionDriver.java | 15 +-
.../compaction/CompactionCoordinatorTest.java | 151 +--
.../manager/tableOps/merge/MergeTabletsTest.java | 5 +-
.../test/functional/AmpleConditionalWriterIT.java | 32 ++--
.../accumulo/test/functional/CompactionIT.java | 167 +
.../functional/TabletManagementIteratorIT.java | 5 +-
18 files changed, 483 insertions(+), 118 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/SelectedFiles.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/SelectedFiles.java
index f2dd1ad861..7cb4c3277e 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/SelectedFiles.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/SelectedFiles.java
@@ -24,10 +24,12 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.util.time.SteadyTime;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
@@ -44,27 +46,48 @@ public class SelectedFiles {
private final Set files;
private final boolean initiallySelectedAll;
private final FateId fateId;
+ private final int completedJobs;
+ private final SteadyTime selectedTime;
private String metadataValue;
private static final Gson GSON = new GsonBuilder()
.registerTypeAdapter(SelectedFiles.class, new
SelectedFilesTypeAdapter()).create();
- public SelectedFiles(Set files, boolean
initiallySelectedAll, FateId fateId) {
+ public SelectedFiles(Set files, boolean
initiallySelectedAll, FateId fateId,
+ SteadyTime selectedTime) {
+this(files, initiallySelectedAll, fateId, 0, selectedTime);
+ }
+
+ public SelectedFiles(Set files, boolean
initiallySelectedAll, FateId fateId,
+ int completedJobs, SteadyTime selectedTime) {
Preconditions.checkArgument(files != null && !files.isEmpty());
+Preconditions.checkArgument(completedJobs >= 0);
this.files = Set.copyOf(files);
this.initiallySelectedAll = initiallySelectedAll;
-this.fateId = fateId;
+this.fateId = Objects.requireNonNull(fateId);
+this.completedJobs = completedJobs;
+this.selectedTime = Objects.requireNonNull(selectedTime);
}
private static class SelectedFilesTypeAdapter extends
TypeAdapter {
+// These fields could be moved to an enum but for now just using static
Strings
+// seems better to avoid having to construct an enum each time the string
is read
+private static final String FATE_ID = "fateId";
+private static final String SELECTED_ALL = "selAll";
+private static final String COMPLETED_JOBS = "compJobs";
+private static final St
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit fc878c19d7cb9db7ca00f84b18131ee6712f3752
Merge: 8f68c3bade 8fe933a671
Author: Christopher L. Shannon
AuthorDate: Fri May 17 16:50:27 2024 -0400
Merge branch 'main' into elasticity
.../server/compaction/PausedCompactionMetrics.java | 20 +++---
.../server/constraints/MetadataConstraints.java| 70 ++
.../constraints/MetadataConstraintsTest.java | 33 +
.../java/org/apache/accumulo/manager/Manager.java | 51 -
.../apache/accumulo/tserver/ScanServerMetrics.java | 10 +--
.../accumulo/tserver/ThriftScanClientHandler.java | 16 ++---
.../tserver/metrics/TabletServerScanMetrics.java | 84 +-
.../tserver/metrics/TabletServerUpdateMetrics.java | 27 ---
.../apache/accumulo/tserver/tablet/TabletBase.java | 2 +-
9 files changed, 198 insertions(+), 115 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index c6a30040a1,1b99a5b307..7f1f5c21bc
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -255,38 -235,6 +256,38 @@@ public class MetadataConstraints implem
} else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
violations = validateDataFileMetadata(violations,
new String(columnUpdate.getColumnQualifier(), UTF_8));
+ } else if (TabletColumnFamily.AVAILABILITY_COLUMN.equals(columnFamily,
columnQualifier)) {
+try {
+ TabletAvailabilityUtil.fromValue(new
Value(columnUpdate.getValue()));
+} catch (IllegalArgumentException e) {
- violations = addViolation(violations, 10);
++ violations = addViolation(violations, 16);
+}
+ } else if (ServerColumnFamily.OPID_COLUMN.equals(columnFamily,
columnQualifier)) {
+try {
+ TabletOperationId.validate(new String(columnUpdate.getValue(),
UTF_8));
+} catch (IllegalArgumentException e) {
+ violations = addViolation(violations, 9);
+}
+ } else if (ServerColumnFamily.SELECTED_COLUMN.equals(columnFamily,
columnQualifier)) {
+try {
+ SelectedFiles.from(new String(columnUpdate.getValue(), UTF_8));
+} catch (RuntimeException e) {
+ violations = addViolation(violations, 11);
+}
+ } else if (CompactedColumnFamily.NAME.equals(columnFamily)) {
+if (!FateId.isFateId(columnQualifier.toString())) {
+ violations = addViolation(violations, 13);
+}
+ } else if
(UserCompactionRequestedColumnFamily.NAME.equals(columnFamily)) {
+if (!FateId.isFateId(columnQualifier.toString())) {
+ violations = addViolation(violations, 14);
+}
+ } else if (SplitColumnFamily.UNSPLITTABLE_COLUMN.equals(columnFamily,
columnQualifier)) {
+try {
+ UnSplittableMetadata.toUnSplittable(new
String(columnUpdate.getValue(), UTF_8));
+} catch (RuntimeException e) {
+ violations = addViolation(violations, 15);
+}
} else if (columnFamily.equals(BulkFileColumnFamily.NAME)) {
if (!columnUpdate.isDeleted() && !checkedBulk) {
/*
@@@ -438,19 -388,9 +448,21 @@@
case 8:
return "Bulk load mutation contains either inconsistent files or
multiple fateTX ids";
case 9:
-return "Invalid data file metadata format";
+return "Malformed operation id";
case 10:
- return "Malformed availability value";
+ return "Suspended timestamp is not valid";
+ case 11:
+return "Malformed file selection value";
+ case 12:
+return "Invalid data file metadata format";
+ case 13:
+return "Invalid compacted column";
+ case 14:
+return "Invalid user compaction requested column";
+ case 15:
+return "Invalid unsplittable column";
++ case 16:
++return "Malformed availability value";
}
return null;
}
diff --cc
server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
index b36fb31583,b7e2bd2a11..71e1ada06e
---
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
@@@ -27,31 -25,25 +27,36 @@@ import static org.junit.jupiter.api.Ass
import java.lang.reflect.Method;
import java.util.Base64;
import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+ import java.ut
(accumulo) branch elasticity updated (8f68c3bade -> fc878c19d7)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8f68c3bade Minor update to fix flaky test add 4b5234bd87 Added ZK cleanup thread to Manager for Scan Server nodes (#4562) add a42c431506 Merge remote-tracking branch 'upstream/2.1' add 1261ec2472 Convert micrometer Counters to FunctionCounters (#4555) add 8fe933a671 Add a constraint check for suspend column (#4546) new fc878c19d7 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../server/compaction/PausedCompactionMetrics.java | 20 +++--- .../server/constraints/MetadataConstraints.java| 70 ++ .../constraints/MetadataConstraintsTest.java | 33 + .../java/org/apache/accumulo/manager/Manager.java | 51 - .../apache/accumulo/tserver/ScanServerMetrics.java | 10 +-- .../accumulo/tserver/ThriftScanClientHandler.java | 16 ++--- .../tserver/metrics/TabletServerScanMetrics.java | 84 +- .../tserver/metrics/TabletServerUpdateMetrics.java | 27 --- .../apache/accumulo/tserver/tablet/TabletBase.java | 2 +- 9 files changed, 198 insertions(+), 115 deletions(-)
(accumulo) branch main updated: Add a constraint check for suspend column (#4546)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8fe933a671 Add a constraint check for suspend column (#4546)
8fe933a671 is described below
commit 8fe933a6710066bb88c63401d12e9ff0d88cdf0f
Author: Christopher L. Shannon
AuthorDate: Fri May 17 16:40:22 2024 -0400
Add a constraint check for suspend column (#4546)
This adds a new metadata constraint check for the suspend metadata
column to ensure that the suspension time is not negative as we should
never have a negative value.
This is a follow on to #4494
---
.../server/constraints/MetadataConstraints.java| 66 +-
.../constraints/MetadataConstraintsTest.java | 33 +++
2 files changed, 72 insertions(+), 27 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 0f4a361312..1b99a5b307 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.SuspendingTServer;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
@@ -306,40 +307,49 @@ public class MetadataConstraints implements Constraint {
} else {
if (!isValidColumn(columnUpdate)) {
violations = addViolation(violations, 2);
-} else if (new
ColumnFQ(columnUpdate).equals(TabletColumnFamily.PREV_ROW_COLUMN)
-&& columnUpdate.getValue().length > 0
-&& (violations == null || !violations.contains((short) 4))) {
- KeyExtent ke = KeyExtent.fromMetaRow(new Text(mutation.getRow()));
+} else {
+ final var column = new ColumnFQ(columnUpdate);
+ if (column.equals(TabletColumnFamily.PREV_ROW_COLUMN)
+ && columnUpdate.getValue().length > 0
+ && (violations == null || !violations.contains((short) 4))) {
+KeyExtent ke = KeyExtent.fromMetaRow(new Text(mutation.getRow()));
- Text per = TabletColumnFamily.decodePrevEndRow(new
Value(columnUpdate.getValue()));
+Text per = TabletColumnFamily.decodePrevEndRow(new
Value(columnUpdate.getValue()));
- boolean prevEndRowLessThanEndRow =
- per == null || ke.endRow() == null || per.compareTo(ke.endRow())
< 0;
+boolean prevEndRowLessThanEndRow =
+per == null || ke.endRow() == null ||
per.compareTo(ke.endRow()) < 0;
- if (!prevEndRowLessThanEndRow) {
-violations = addViolation(violations, 3);
- }
-} else if (new
ColumnFQ(columnUpdate).equals(ServerColumnFamily.LOCK_COLUMN)) {
- if (zooCache == null) {
-zooCache = new ZooCache(context.getZooReader(), null);
-CleanerUtil.zooCacheClearer(this, zooCache);
- }
+if (!prevEndRowLessThanEndRow) {
+ violations = addViolation(violations, 3);
+}
+ } else if (column.equals(ServerColumnFamily.LOCK_COLUMN)) {
+if (zooCache == null) {
+ zooCache = new ZooCache(context.getZooReader(), null);
+ CleanerUtil.zooCacheClearer(this, zooCache);
+}
- if (zooRoot == null) {
-zooRoot = context.getZooKeeperRoot();
- }
+if (zooRoot == null) {
+ zooRoot = context.getZooKeeperRoot();
+}
- boolean lockHeld = false;
- String lockId = new String(columnUpdate.getValue(), UTF_8);
+boolean lockHeld = false;
+String lockId = new String(columnUpdate.getValue(), UTF_8);
- try {
-lockHeld = ServiceLock.isLockHeld(zooCache, new
ZooUtil.LockID(zooRoot, lockId));
- } catch (Exception e) {
-log.debug("Failed to verify lock was held {} {}", lockId,
e.getMessage());
- }
+try {
+ lockHeld = ServiceLock.isLockHeld(zooCache, new
ZooUtil.LockID(zooRoot, lockId));
+} catch (Exception e) {
+ log.debug("Failed to verify lock was held {} {}", lockId,
e.getMessage());
+
(accumulo) branch elasticity updated (ff951d7f99 -> 8f68c3bade)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from ff951d7f99 Add compaction IT that verifies queues are cleared when tablets no longer need to compact (#4466) add 8f68c3bade Minor update to fix flaky test No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/accumulo/test/functional/CompactionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) branch elasticity updated: Move TestAmple to test module (#4557)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 4146721be2 Move TestAmple to test module (#4557)
4146721be2 is described below
commit 4146721be2b25e688db7d57b628e59142206f4de
Author: Christopher L. Shannon
AuthorDate: Mon May 13 17:39:23 2024 -0400
Move TestAmple to test module (#4557)
This moves the TestAmple implementation out of the server/base module
test jar and into the test module. TestAmple and associated classes were
originally put there to match the same package as Ample but this does
not work due to jars being sealed. This changes the visibility of some
of the constructors and methods in order to be able to override now that
TestAmple is in a new package.
This is a follow on to #4415
---
pom.xml| 7 ---
server/base/pom.xml| 18 --
.../metadata/AsyncConditionalTabletsMutatorImpl.java | 7 +--
.../accumulo/server/metadata/ServerAmpleImpl.java | 5 -
.../accumulo/server/metadata/TabletsMutatorImpl.java | 4 +++-
test/pom.xml | 6 --
.../org/apache/accumulo/test/ample/TestAmpleIT.java| 6 +++---
.../org/apache/accumulo/test/ample/TestAmpleUtil.java | 4 ++--
.../ample}/metadata/ConditionalWriterDelegator.java| 2 +-
.../ample}/metadata/ConditionalWriterInterceptor.java | 2 +-
.../accumulo/test/ample}/metadata/TestAmple.java | 9 ++---
.../org/apache/accumulo/test/fate/ManagerRepoIT.java | 6 +++---
12 files changed, 28 insertions(+), 48 deletions(-)
diff --git a/pom.xml b/pom.xml
index ea8923d947..aeacdf925e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -624,13 +624,6 @@
snakeyaml
2.2
-
-org.apache.accumulo
-accumulo-server-base
-${project.version}
-test-jar
-test
-
diff --git a/server/base/pom.xml b/server/base/pom.xml
index d7f23eb028..c20b3e6edc 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -136,24 +136,6 @@
src/test/resources
-
-
-org.apache.maven.plugins
-maven-jar-plugin
-
-
-
- test-jar
-
-
-
-**/metadata/**
-
-
-
-
-
-
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/AsyncConditionalTabletsMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/AsyncConditionalTabletsMutatorImpl.java
index 46532681d9..fb3a43dcd7 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/AsyncConditionalTabletsMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/AsyncConditionalTabletsMutatorImpl.java
@@ -33,6 +33,8 @@ import
org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.server.ServerContext;
+import com.google.common.annotations.VisibleForTesting;
+
public class AsyncConditionalTabletsMutatorImpl implements
Ample.AsyncConditionalTabletsMutator {
private final Consumer resultsConsumer;
private final ExecutorService executor;
@@ -43,8 +45,9 @@ public class AsyncConditionalTabletsMutatorImpl implements
Ample.AsyncConditiona
public static final int BATCH_SIZE = 1000;
private final Function tableMapper;
- AsyncConditionalTabletsMutatorImpl(ServerContext context,
Function tableMapper,
- Consumer resultsConsumer) {
+ @VisibleForTesting
+ public AsyncConditionalTabletsMutatorImpl(ServerContext context,
+ Function tableMapper,
Consumer resultsConsumer) {
this.resultsConsumer = Objects.requireNonNull(resultsConsumer);
this.context = context;
this.bufferingMutator = new ConditionalTabletsMutatorImpl(context,
tableMapper);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index abb297ad23..67f9d96e44 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -65,6 +65,8 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
public class ServerAmpleImpl extends AmpleImpl implements Ample {
private static Logger log = LoggerFactory.getLogger(ServerAmpleImpl.class);
@@ -347,7 +349,8 @@ public class ServerAmpleImpl extends AmpleImpl
(accumulo) branch elasticity updated: Create a test Ample implementation (#4415)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 0edc48f910 Create a test Ample implementation (#4415)
0edc48f910 is described below
commit 0edc48f910f25e5273d62b1d91a7993bf9529b8e
Author: Christopher L. Shannon
AuthorDate: Sat May 11 09:08:36 2024 -0400
Create a test Ample implementation (#4415)
This commit adds a new test Ample implementation that
can be used to interact with Ample against a different table than the
table that the Accumulo instance uses. This will allow testing Ample
without having to deal with metadata being written to the table by the
system.
The test Ample also will support injecting custom behavior such as
changing the return status of a conditional mutation in order to test.
This closes #4376
---
.../accumulo/core/metadata/schema/AmpleImpl.java | 29 ++-
.../core/metadata/schema/TabletsMetadata.java | 15 +-
pom.xml| 7 +
server/base/pom.xml| 18 ++
.../AsyncConditionalTabletsMutatorImpl.java| 11 +-
.../metadata/ConditionalTabletsMutatorImpl.java| 17 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 48 ++--
.../server/metadata/TabletsMutatorImpl.java| 14 +-
.../metadata/ConditionalWriterDelegator.java | 54 +
.../metadata/ConditionalWriterInterceptor.java | 86 +++
.../apache/accumulo/server/metadata/TestAmple.java | 264 +
test/pom.xml | 6 +
.../apache/accumulo/test/ample/TestAmpleIT.java| 190 +++
.../apache/accumulo/test/ample/TestAmpleUtil.java | 38 +++
.../apache/accumulo/test/fate/ManagerRepoIT.java | 225 ++
15 files changed, 992 insertions(+), 30 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/AmpleImpl.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/AmpleImpl.java
index 89ce240cad..cdad8708e6 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/AmpleImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/AmpleImpl.java
@@ -21,23 +21,34 @@ package org.apache.accumulo.core.metadata.schema;
import static com.google.common.collect.MoreCollectors.onlyElement;
import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.function.Function;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata.Options;
+import org.apache.accumulo.core.metadata.schema.TabletsMetadata.TableOptions;
+
+import com.google.common.base.Preconditions;
public class AmpleImpl implements Ample {
private final AccumuloClient client;
+ private final Function tableMapper;
public AmpleImpl(AccumuloClient client) {
+this(client, DataLevel::metaTable);
+ }
+
+ public AmpleImpl(AccumuloClient client, Function
tableMapper) {
this.client = client;
+this.tableMapper = Objects.requireNonNull(tableMapper);
}
@Override
public TabletMetadata readTablet(KeyExtent extent, ReadConsistency
readConsistency,
ColumnType... colsToFetch) {
-Options builder = TabletsMetadata.builder(client).forTablet(extent);
+Options builder = newBuilder().forTablet(extent);
if (colsToFetch.length > 0) {
builder.fetch(colsToFetch);
}
@@ -53,7 +64,21 @@ public class AmpleImpl implements Ample {
@Override
public TabletsMetadata.TableOptions readTablets() {
-return TabletsMetadata.builder(this.client);
+return newBuilder();
+ }
+
+ protected TableOptions newBuilder() {
+return TabletsMetadata.builder(this.client, getTableMapper());
+ }
+
+ protected String getMetadataTableName(Ample.DataLevel dataLevel) {
+final String metadataTable = getTableMapper().apply(dataLevel);
+Preconditions.checkArgument(metadataTable != null,
+"A metadata table for %s has not been registered", dataLevel);
+return metadataTable;
}
+ protected Function getTableMapper() {
+return tableMapper;
+ }
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 48a1160513..c91e2d54de 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -117,9 +117,11 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
private ReadConsistency re
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5e4ec67ac0ba337d21b815e5a8c5cb95f49a5f89
Merge: 5220f009cc 5aca4f67bc
Author: Christopher L. Shannon
AuthorDate: Fri May 10 17:05:27 2024 -0400
Merge branch 'main' into elasticity
.../org/apache/accumulo/core/logging/TabletLogger.java | 7 +++
.../accumulo/core/metadata/SuspendingTServer.java | 17 ++---
.../org/apache/accumulo/core/metadata/schema/Ample.java | 3 ++-
.../core/metadata/schema/TabletMetadataBuilder.java | 3 ++-
.../core/metadata/schema/TabletMutatorBase.java | 3 ++-
.../org/apache/accumulo/core/util/time/SteadyTime.java | 14 ++
.../core/metadata/schema/TabletMetadataTest.java| 16 ++--
.../server/manager/state/AbstractTabletStateStore.java | 9 +
.../server/manager/state/LoggingTabletStateStore.java | 6 +++---
.../server/manager/state/MetaDataStateStore.java| 7 ---
.../accumulo/server/manager/state/TabletStateStore.java | 5 +++--
.../server/manager/state/ZooTabletStateStore.java | 5 +++--
.../org/apache/accumulo/manager/TabletGroupWatcher.java | 4 ++--
.../apache/accumulo/tserver/TabletClientHandler.java| 5 +++--
.../apache/accumulo/tserver/UnloadTabletHandler.java| 14 --
15 files changed, 74 insertions(+), 44 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
index e76c62a6c9,2209e41c27..40dc0b5eb1
--- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
@@@ -23,15 -23,12 +23,14 @@@ import static java.util.stream.Collecto
import java.util.Collection;
import java.util.List;
import java.util.Optional;
+import java.util.SortedSet;
import java.util.UUID;
- import java.util.concurrent.TimeUnit;
-import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletFile;
@@@ -40,8 -36,8 +39,9 @@@ import org.apache.accumulo.core.metadat
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
+ import org.apache.accumulo.core.util.time.SteadyTime;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 27856c899b,931e415774..7d7086649a
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@@ -342,87 -265,50 +343,87 @@@ public interface Ample
/**
* Interface for changing a tablets persistent data.
*/
- interface TabletMutator {
-TabletMutator putPrevEndRow(Text per);
+ interface TabletUpdates {
+T putPrevEndRow(Text per);
+
+T putFile(ReferencedTabletFile path, DataFileValue dfv);
+
+T putFile(StoredTabletFile path, DataFileValue dfv);
-TabletMutator putFile(ReferencedTabletFile path, DataFileValue dfv);
+T deleteFile(StoredTabletFile path);
-TabletMutator putFile(StoredTabletFile path, DataFileValue dfv);
+T putScan(StoredTabletFile path);
-TabletMutator deleteFile(StoredTabletFile path);
+T deleteScan(StoredTabletFile path);
-TabletMutator putScan(StoredTabletFile path);
+T putFlushId(long flushId);
-TabletMutator deleteScan(StoredTabletFile path);
+T putFlushNonce(long flushNonce);
-TabletMutator putCompactionId(long compactionId);
+T putLocation(Location location);
-TabletMutator putFlushId(long flushId);
+T deleteLocation(Location location);
-TabletMutator putLocation(Location location);
+T putDirName(String dirName);
-TabletMutator deleteLocation(Location location);
+T putWal(LogEntry logEntry);
-TabletMutator putZooLock(ServiceLock zooLock);
+T deleteWal(LogEntry logEntry);
-TabletMutator putDirName(String dirName);
+T putTime(MetadataTime time);
-TabletMutator putWal(LogEntry logEntry);
+T putBulkFile(ReferencedTabletFile bulkref, FateId fateId);
-TabletMutator deleteWal(LogEntry wal);
+T deleteBulkFile(StoredTabletFile bulkref);
- T
(accumulo) branch elasticity updated (5220f009cc -> 5e4ec67ac0)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5220f009cc Merge branch 'main' into elasticity add 368cdb36ae Update Ample to use SteadyTime for suspension (#4545) add 5aca4f67bc simplifies handling of time in unload handler (#4548) new 5e4ec67ac0 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/logging/TabletLogger.java | 7 +++ .../accumulo/core/metadata/SuspendingTServer.java | 17 ++--- .../org/apache/accumulo/core/metadata/schema/Ample.java | 3 ++- .../core/metadata/schema/TabletMetadataBuilder.java | 3 ++- .../core/metadata/schema/TabletMutatorBase.java | 3 ++- .../org/apache/accumulo/core/util/time/SteadyTime.java | 14 ++ .../core/metadata/schema/TabletMetadataTest.java| 16 ++-- .../server/manager/state/AbstractTabletStateStore.java | 9 + .../server/manager/state/LoggingTabletStateStore.java | 6 +++--- .../server/manager/state/MetaDataStateStore.java| 7 --- .../accumulo/server/manager/state/TabletStateStore.java | 5 +++-- .../server/manager/state/ZooTabletStateStore.java | 5 +++-- .../org/apache/accumulo/manager/TabletGroupWatcher.java | 4 ++-- .../apache/accumulo/tserver/TabletClientHandler.java| 5 +++-- .../apache/accumulo/tserver/UnloadTabletHandler.java| 14 -- 15 files changed, 74 insertions(+), 44 deletions(-)
(accumulo) branch elasticity updated (edda158a54 -> 5220f009cc)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from edda158a54 Modified SetEncodingIterator to include Value (#4486) add 6dcf84ed00 Create a SteadyTime type in the Manager (#4494) new 5220f009cc Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/core/util/time/SteadyTime.java | 84 + .../accumulo/core/util/time/SteadyTimeTest.java| 56 + server/manager/pom.xml | 5 + .../java/org/apache/accumulo/manager/Manager.java | 9 +- .../org/apache/accumulo/manager/ManagerTime.java | 120 --- .../accumulo/manager/TabletGroupWatcher.java | 6 +- .../apache/accumulo/manager/ManagerTimeTest.java | 130 + 7 files changed, 389 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java create mode 100644 core/src/test/java/org/apache/accumulo/core/util/time/SteadyTimeTest.java create mode 100644 server/manager/src/test/java/org/apache/accumulo/manager/ManagerTimeTest.java
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5220f009ccf2816f195b103d8545707cd2b54793
Merge: edda158a54 6dcf84ed00
Author: Christopher L. Shannon
AuthorDate: Fri May 10 09:57:21 2024 -0400
Merge branch 'main' into elasticity
.../apache/accumulo/core/util/time/SteadyTime.java | 84 +
.../accumulo/core/util/time/SteadyTimeTest.java| 56 +
server/manager/pom.xml | 5 +
.../java/org/apache/accumulo/manager/Manager.java | 9 +-
.../org/apache/accumulo/manager/ManagerTime.java | 120 ---
.../accumulo/manager/TabletGroupWatcher.java | 6 +-
.../apache/accumulo/manager/ManagerTimeTest.java | 130 +
7 files changed, 389 insertions(+), 21 deletions(-)
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index a7bf212243,ded6d62f83..631daaddb1
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -120,10 -114,9 +120,11 @@@ import org.apache.accumulo.core.util.Re
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.core.util.time.NanoTime;
+ import org.apache.accumulo.core.util.time.SteadyTime;
+import
org.apache.accumulo.manager.compaction.coordinator.CompactionCoordinator;
import org.apache.accumulo.manager.metrics.ManagerMetrics;
import org.apache.accumulo.manager.recovery.RecoveryManager;
+import org.apache.accumulo.manager.split.Splitter;
import org.apache.accumulo.manager.state.TableCounts;
import org.apache.accumulo.manager.tableOps.TraceRepo;
import org.apache.accumulo.manager.upgrade.PreUpgradeValidation;
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 060411fcbf,530bd950b1..ca69231394
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -214,484 -177,203 +214,484 @@@ abstract class TabletGroupWatcher exten
}
}
- @Override
- public void run() {
-int[] oldCounts = new int[TabletState.values().length];
-EventCoordinator.Listener eventListener =
this.manager.nextEvent.getListener();
+ class EventHandler implements EventCoordinator.Listener {
-WalStateManager wals = new WalStateManager(manager.getContext());
+// Setting this to true to start with because its not know what happended
before this object was
+// created, so just start off with full scan.
+private boolean needsFullScan = true;
-while (manager.stillManager()) {
- // slow things down a little, otherwise we spam the logs when there are
many wake-up events
- sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+private final BlockingQueue rangesToProcess;
- final long waitTimeBetweenScans = manager.getConfiguration()
- .getTimeInMillis(Property.MANAGER_TABLET_GROUP_WATCHER_INTERVAL);
+class RangeProccessor implements Runnable {
+ @Override
+ public void run() {
+try {
+ while (manager.stillManager()) {
+var range = rangesToProcess.poll(100, TimeUnit.MILLISECONDS);
+if (range == null) {
+ // check to see if still the manager
+ continue;
+}
- int totalUnloaded = 0;
- int unloaded = 0;
- ClosableIterator iter = null;
- try {
-Map mergeStatsCache = new HashMap<>();
-Map currentMerges = new HashMap<>();
-for (MergeInfo merge : manager.merges()) {
- if (merge.getExtent() != null) {
-currentMerges.put(merge.getExtent().tableId(), new
MergeStats(merge));
+ArrayList ranges = new ArrayList<>();
+ranges.add(range);
+
+rangesToProcess.drainTo(ranges);
+
+if (manager.getManagerGoalState() == ManagerGoalState.CLEAN_STOP)
{
+ // only do full scans when trying to shutdown
+ setNeedsFullScan();
+ continue;
+}
+
+TabletManagementParameters tabletMgmtParams =
createTabletManagementParameters(false);
+
+var currentTservers =
getCurrentTservers(tabletMgmtParams.getOnlineTsevers());
+if (currentTservers.isEmpty()) {
+ setNeedsFullScan();
+ continue;
+}
+
+try (var iter = store.iterator(ranges, tabletMgmtParams)) {
+ long t1 = System.currentTimeMillis();
+ manageTablets(iter, tabletMgmtParams, currentTservers, false);
+ long t2 = System.currentTimeMillis();
+ Manager.log.d
(accumulo) branch main updated: Create a SteadyTime type in the Manager (#4494)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6dcf84ed00 Create a SteadyTime type in the Manager (#4494)
6dcf84ed00 is described below
commit 6dcf84ed00eca6129eee6a559beebd625dadf966
Author: Christopher L. Shannon
AuthorDate: Fri May 10 09:55:34 2024 -0400
Create a SteadyTime type in the Manager (#4494)
* Create a SteadyTime type in the Manager
This replaces Manager.getSteadyTime() long value with a concrete type to
make it more apparent what time is being used. The serialization
and deserialization logic have been encapsulated as methods to make the
conversion consistent.
This closes #4482
---
.../apache/accumulo/core/util/time/SteadyTime.java | 84 +
.../accumulo/core/util/time/SteadyTimeTest.java| 56 +
server/manager/pom.xml | 5 +
.../java/org/apache/accumulo/manager/Manager.java | 9 +-
.../org/apache/accumulo/manager/ManagerTime.java | 120 ---
.../accumulo/manager/TabletGroupWatcher.java | 6 +-
.../apache/accumulo/manager/ManagerTimeTest.java | 130 +
7 files changed, 389 insertions(+), 21 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
b/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
new file mode 100644
index 00..a94ae0ce55
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/util/time/SteadyTime.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.util.time;
+
+import java.time.Duration;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * SteadyTime represents an approximation of the total duration of time this
cluster has had a
+ * Manager. Because this represents an elapsed time it is guaranteed to not be
negative. SteadyTime
+ * is not expected to represent real world date times, its main use is for
computing deltas similar
+ * System.nanoTime but across JVM processes.
+ */
+public class SteadyTime implements Comparable {
+
+ private final Duration time;
+
+ private SteadyTime(Duration time) {
+Preconditions.checkArgument(!time.isNegative(), "SteadyTime '%s' should
not be negative.",
+time.toNanos());
+this.time = time;
+ }
+
+ public long getMillis() {
+return time.toMillis();
+ }
+
+ public long getNanos() {
+return time.toNanos();
+ }
+
+ public Duration getDuration() {
+return time;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+if (this == o) {
+ return true;
+}
+if (o == null || getClass() != o.getClass()) {
+ return false;
+}
+SteadyTime that = (SteadyTime) o;
+return Objects.equals(time, that.time);
+ }
+
+ @Override
+ public int hashCode() {
+return Objects.hashCode(time);
+ }
+
+ @Override
+ public int compareTo(SteadyTime other) {
+return time.compareTo(other.time);
+ }
+
+ public static SteadyTime from(long time, TimeUnit unit) {
+return new SteadyTime(Duration.of(time, unit.toChronoUnit()));
+ }
+
+ public static SteadyTime from(Duration time) {
+return new SteadyTime(time);
+ }
+}
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/time/SteadyTimeTest.java
b/core/src/test/java/org/apache/accumulo/core/util/time/SteadyTimeTest.java
new file mode 100644
index 00..016c771a8d
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/util/time/SteadyTimeTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the Licens
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 861d0fdfc2b7bdd6da72e0e5652c027643ec5a49
Merge: 1c706c541e 1ae2b17e2b
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 16:11:28 2024 -0400
Merge branch 'main' into elasticity
.../core/metadata/schema/TabletMetadata.java | 389 ++---
.../core/metadata/schema/TabletMetadataTest.java | 105 +-
2 files changed, 362 insertions(+), 132 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index be7d2fb056,f2ad719fd8..6f2ffd6237
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -92,44 -82,63 +92,80 @@@ import com.google.common.collect.Immuta
import com.google.common.net.HostAndPort;
public class TabletMetadata {
+
private static final Logger log =
LoggerFactory.getLogger(TabletMetadata.class);
- private TableId tableId;
- private Text prevEndRow;
- private boolean sawPrevEndRow = false;
- private Text endRow;
- private Location location;
- private Map files;
- private List scans;
- private Map loadedFiles;
- private SelectedFiles selectedFiles;
- private EnumSet fetchedCols;
- private KeyExtent extent;
- private Location last;
- private SuspendingTServer suspend;
- private String dirName;
- private MetadataTime time;
- private String cloned;
- private SortedMap keyValues;
- private OptionalLong flush = OptionalLong.empty();
- private OptionalLong flushNonce = OptionalLong.empty();
- private List logs;
- private Map extCompactions;
- private boolean merged;
- private TabletAvailability availability = TabletAvailability.ONDEMAND;
- private boolean onDemandHostingRequested = false;
- private TabletOperationId operationId;
- private boolean futureAndCurrentLocationSet = false;
- private Set compacted;
- private Set userCompactionsRequested;
- private UnSplittableMetadata unSplittableMetadata;
- private Supplier fileSize;
+ private final TableId tableId;
+ private final Text prevEndRow;
+ private final boolean sawPrevEndRow;
- private final Text oldPrevEndRow;
- private final boolean sawOldPrevEndRow;
+ private final Text endRow;
+ private final Location location;
+ private final Map files;
+ private final List scans;
- private final Map loadedFiles;
++ private final Map loadedFiles;
++ private final SelectedFiles selectedFiles;
+ private final EnumSet fetchedCols;
+ private final Supplier extent;
+ private final Location last;
+ private final SuspendingTServer suspend;
+ private final String dirName;
+ private final MetadataTime time;
+ private final String cloned;
+ private final SortedMap keyValues;
+ private final OptionalLong flush;
++ private final OptionalLong flushNonce;
+ private final List logs;
- private final OptionalLong compact;
- private final Double splitRatio;
- private final Map
extCompactions;
++ private final Map extCompactions;
+ private final boolean merged;
++ private final TabletAvailability availability;
++ private final boolean onDemandHostingRequested;
++ private final TabletOperationId operationId;
++ private final boolean futureAndCurrentLocationSet;
++ private final Set compacted;
++ private final Set userCompactionsRequested;
++ private final UnSplittableMetadata unSplittableMetadata;
++ private final Supplier fileSize;
+
+ private TabletMetadata(Builder tmBuilder) {
+ this.tableId = tmBuilder.tableId;
+ this.prevEndRow = tmBuilder.prevEndRow;
+ this.sawPrevEndRow = tmBuilder.sawPrevEndRow;
-this.oldPrevEndRow = tmBuilder.oldPrevEndRow;
-this.sawOldPrevEndRow = tmBuilder.sawOldPrevEndRow;
+ this.endRow = tmBuilder.endRow;
+ this.location = tmBuilder.location;
+ this.files = Objects.requireNonNull(tmBuilder.files.build());
+ this.scans = Objects.requireNonNull(tmBuilder.scans.build());
+ this.loadedFiles = tmBuilder.loadedFiles.build();
++this.selectedFiles = tmBuilder.selectedFiles;
+ this.fetchedCols = Objects.requireNonNull(tmBuilder.fetchedCols);
+ this.last = tmBuilder.last;
+ this.suspend = tmBuilder.suspend;
+ this.dirName = tmBuilder.dirName;
+ this.time = tmBuilder.time;
+ this.cloned = tmBuilder.cloned;
+ this.keyValues =
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableSortedMap.Builder::build)
+ .orElse(null);
+ this.flush = tmBuilder.flush;
++this.flushNonce = tmBuilder.flushNonce;
+ this.logs = Objects.requireNonNull(tmBuilder.logs.build());
-this.compact = Objects.requireNonNull(tmBuilder.compact);
-this.splitRatio = tmBuilder.splitRatio;
+ this.extCompactions =
Objects.requireNonNull(tmBuilder.extCompactions.build
(accumulo) branch elasticity updated (1c706c541e -> 861d0fdfc2)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1c706c541e Merge branch 'main' into elasticity add 1ae2b17e2b Update TabletMetadata to be immutable (#4437) new 861d0fdfc2 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/metadata/schema/TabletMetadata.java | 389 ++--- .../core/metadata/schema/TabletMetadataTest.java | 105 +- 2 files changed, 362 insertions(+), 132 deletions(-)
(accumulo) branch main updated: Update TabletMetadata to be immutable (#4437)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1ae2b17e2b Update TabletMetadata to be immutable (#4437)
1ae2b17e2b is described below
commit 1ae2b17e2bb94c922b5a3a7482cf92fb5ae23883
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:52:31 2024 -0400
Update TabletMetadata to be immutable (#4437)
This commit refactors all of the fields to be final for TabletMetadata.
The object is already treated as immutable in practice, but
previously didn't declare fields final so this could lead to potential
future bugs and also means not being able to guarantee the state of a
field which is now possible.
---
.../core/metadata/schema/TabletMetadata.java | 302 +++--
.../core/metadata/schema/TabletMetadataTest.java | 101 ++-
2 files changed, 313 insertions(+), 90 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index b98d9e8540..f2ad719fd8 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -74,38 +74,70 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.ImmutableSortedMap.Builder;
import com.google.common.net.HostAndPort;
public class TabletMetadata {
private static final Logger log =
LoggerFactory.getLogger(TabletMetadata.class);
- private TableId tableId;
- private Text prevEndRow;
- private boolean sawPrevEndRow = false;
- private Text oldPrevEndRow;
- private boolean sawOldPrevEndRow = false;
- private Text endRow;
- private Location location;
- private Map files;
- private List scans;
- private Map loadedFiles;
- private EnumSet fetchedCols;
- private KeyExtent extent;
- private Location last;
- private SuspendingTServer suspend;
- private String dirName;
- private MetadataTime time;
- private String cloned;
- private SortedMap keyValues;
- private OptionalLong flush = OptionalLong.empty();
- private List logs;
- private OptionalLong compact = OptionalLong.empty();
- private Double splitRatio = null;
- private Map extCompactions;
- private boolean merged;
+ private final TableId tableId;
+ private final Text prevEndRow;
+ private final boolean sawPrevEndRow;
+ private final Text oldPrevEndRow;
+ private final boolean sawOldPrevEndRow;
+ private final Text endRow;
+ private final Location location;
+ private final Map files;
+ private final List scans;
+ private final Map loadedFiles;
+ private final EnumSet fetchedCols;
+ private final Supplier extent;
+ private final Location last;
+ private final SuspendingTServer suspend;
+ private final String dirName;
+ private final MetadataTime time;
+ private final String cloned;
+ private final SortedMap keyValues;
+ private final OptionalLong flush;
+ private final List logs;
+ private final OptionalLong compact;
+ private final Double splitRatio;
+ private final Map
extCompactions;
+ private final boolean merged;
+
+ private TabletMetadata(Builder tmBuilder) {
+this.tableId = tmBuilder.tableId;
+this.prevEndRow = tmBuilder.prevEndRow;
+this.sawPrevEndRow = tmBuilder.sawPrevEndRow;
+this.oldPrevEndRow = tmBuilder.oldPrevEndRow;
+this.sawOldPrevEndRow = tmBuilder.sawOldPrevEndRow;
+this.endRow = tmBuilder.endRow;
+this.location = tmBuilder.location;
+this.files = Objects.requireNonNull(tmBuilder.files.build());
+this.scans = Objects.requireNonNull(tmBuilder.scans.build());
+this.loadedFiles = tmBuilder.loadedFiles.build();
+this.fetchedCols = Objects.requireNonNull(tmBuilder.fetchedCols);
+this.last = tmBuilder.last;
+this.suspend = tmBuilder.suspend;
+this.dirName = tmBuilder.dirName;
+this.time = tmBuilder.time;
+this.cloned = tmBuilder.cloned;
+this.keyValues =
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableSortedMap.Builder::build)
+.orElse(null);
+this.flush = tmBuilder.flush;
+this.logs = Objects.requireNonNull(tmBuilder.logs.build());
+this.compact = Objects.requireNonNull(tmBuilder.compact);
+this.splitRatio = tmBuilder.splitRatio;
+this.extCompactions =
Objects.requireNonNull(tmBuilder.extCompactions.build());
+this.merged = tmBuilder.merged;
+this.extent =
+Suppliers.memoize(() -> new KeyExt
(accumulo) branch elasticity updated (77e95f95b9 -> 1c706c541e)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 77e95f95b9 Existing logging in ManagerClientServiceHandler is sufficient, removed todo (#4433) add 90bcb465fe Ensure getTabletState() in TabletMetadata does not allow a null extent (#4438) add 84e8557ab8 Merge branch '2.1' new 1c706c541e Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1c706c541e1e81fa2a8c87b72e3c1a59d52a282a Merge: 77e95f95b9 84e8557ab8 Author: Christopher L. Shannon AuthorDate: Sat Apr 6 15:34:09 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch main updated (b912506d43 -> 84e8557ab8)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b912506d43 avoids acquiring recovery lock when tablet has no wals (#4429) add 90bcb465fe Ensure getTabletState() in TabletMetadata does not allow a null extent (#4438) new 84e8557ab8 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++- .../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++- .../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8 3 files changed, 13 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 84e8557ab8ddabd23c8813d9dbed2e1223d99646
Merge: b912506d43 90bcb465fe
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:23:23 2024 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++-
.../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++-
.../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8
3 files changed, 13 insertions(+), 2 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
index eeaa5cde95,0a70594027..4c2421b4e8
---
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
@@@ -57,8 -57,9 +58,8 @@@ public class TabletLocationState
}
public TabletLocationState(KeyExtent extent, Location future, Location
current, Location last,
- SuspendingTServer suspend, Collection> walogs,
boolean chopped)
- throws BadLocationStateException {
+ SuspendingTServer suspend, Collection walogs) throws
BadLocationStateException {
- this.extent = extent;
+ this.extent = Objects.requireNonNull(extent);
this.future = validateLocation(future,
TabletMetadata.LocationType.FUTURE);
this.current = validateLocation(current,
TabletMetadata.LocationType.CURRENT);
this.last = validateLocation(last, TabletMetadata.LocationType.LAST);
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index af31605d51,c1e816109a..b98d9e8540
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -371,7 -373,9 +371,9 @@@ public class TabletMetadata
future = location;
}
// only care about the state so don't need walogs and chopped params
- var tls = new TabletLocationState(extent, future, current, last,
suspend, null);
+ // Use getExtent() when passing the extent as the private reference may
not have been
+ // initialized yet. This will also ensure PREV_ROW was fetched
- var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null, false);
++ var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null);
return tls.getState(liveTServers);
} catch (TabletLocationState.BadLocationStateException blse) {
throw new IllegalArgumentException("Error creating
TabletLocationState", blse);
(accumulo) branch 2.1 updated: Ensure getTabletState() in TabletMetadata does not allow a null extent (#4438)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 90bcb465fe Ensure getTabletState() in TabletMetadata does not allow a
null extent (#4438)
90bcb465fe is described below
commit 90bcb465fe672ec0181bbcd8acc1d6f4f22e66be
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:20:07 2024 -0400
Ensure getTabletState() in TabletMetadata does not allow a null extent
(#4438)
Previously the private reference for the extent inside TabletMetadata
was passed to the TabletLocationState constructor inside of
getTabletState(). The extent may not have been loaded at this point as
it is lazy loaded by the getExtent() method and requires PREV_ROW to
have been fetched. This change now uses the getter to make sure we do
not inadvertently pass a null extent which is invalid.
---
.../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++-
.../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++-
.../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8
3 files changed, 13 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
index d0cd66300b..0a70594027 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
@@ -22,6 +22,7 @@ import static java.util.Objects.requireNonNull;
import java.util.Collection;
import java.util.Collections;
+import java.util.Objects;
import java.util.Set;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -58,7 +59,7 @@ public class TabletLocationState {
public TabletLocationState(KeyExtent extent, Location future, Location
current, Location last,
SuspendingTServer suspend, Collection> walogs,
boolean chopped)
throws BadLocationStateException {
-this.extent = extent;
+this.extent = Objects.requireNonNull(extent);
this.future = validateLocation(future, TabletMetadata.LocationType.FUTURE);
this.current = validateLocation(current,
TabletMetadata.LocationType.CURRENT);
this.last = validateLocation(last, TabletMetadata.LocationType.LAST);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 323f0cb142..c1e816109a 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -373,7 +373,9 @@ public class TabletMetadata {
future = location;
}
// only care about the state so don't need walogs and chopped params
- var tls = new TabletLocationState(extent, future, current, last,
suspend, null, false);
+ // Use getExtent() when passing the extent as the private reference may
not have been
+ // initialized yet. This will also ensure PREV_ROW was fetched
+ var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null, false);
return tls.getState(liveTServers);
} catch (TabletLocationState.BadLocationStateException blse) {
throw new IllegalArgumentException("Error creating TabletLocationState",
blse);
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 67a22ccd3f..7d9284ba00 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@ -26,6 +26,7 @@ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -196,6 +197,13 @@ public class TabletMetadataTest {
.put(ser1.getHostPort());
SortedMap rowMap = toRowMap(mutation);
+// PREV_ROW was not fetched
+final var missingPrevRow =
+TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsTo
(accumulo) branch elasticity updated: Update offline operation to wait for no opid or ecomp (#4414)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 7aba4f984a Update offline operation to wait for no opid or ecomp
(#4414)
7aba4f984a is described below
commit 7aba4f984a35a4ae5c351b0b4716d4ed22151b51
Author: Christopher L. Shannon
AuthorDate: Tue Mar 26 06:44:01 2024 -0400
Update offline operation to wait for no opid or ecomp (#4414)
Prior to elasticity a compaction or split could never run unless a
tablet was hosted and how these operations can run on unhosted tablets.
Taking a table offline should now wait for no OPID and ECOMP columns to
exist in addition to the existing check for no location.
This addresses a TODO in #3412
---
.../core/clientImpl/TableOperationsImpl.java | 12 ++-
.../org/apache/accumulo/test/OfflineTableIT.java | 119 +
2 files changed, 128 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 2ea43612f1..293177a557 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -27,11 +27,13 @@ import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toSet;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
@@ -1305,19 +1307,23 @@ public class TableOperationsImpl extends
TableOperationsHelper {
Text continueRow = null;
MapCounter serverCounts = new MapCounter<>();
- try (TabletsMetadata tablets =
TabletsMetadata.builder(context).scanMetadataTable()
- .overRange(range).fetch(AVAILABILITY, HOSTING_REQUESTED, LOCATION,
PREV_ROW).build()) {
+ try (TabletsMetadata tablets =
+ TabletsMetadata.builder(context).scanMetadataTable().overRange(range)
+ .fetch(AVAILABILITY, HOSTING_REQUESTED, LOCATION, PREV_ROW,
OPID, ECOMP).build()) {
for (TabletMetadata tablet : tablets) {
total++;
Location loc = tablet.getLocation();
TabletAvailability availability = tablet.getTabletAvailability();
+ var opid = tablet.getOperationId();
+ var externalCompactions = tablet.getExternalCompactions();
if ((expectedState == TableState.ONLINE
&& (availability == TabletAvailability.HOSTED
|| (availability == TabletAvailability.ONDEMAND) &&
tablet.getHostingRequested())
&& (loc == null || loc.getType() == LocationType.FUTURE))
- || (expectedState == TableState.OFFLINE && loc != null)) {
+ || (expectedState == TableState.OFFLINE
+ && (loc != null || opid != null ||
!externalCompactions.isEmpty( {
if (continueRow == null) {
continueRow = tablet.getExtent().toMetaRow();
}
diff --git a/test/src/main/java/org/apache/accumulo/test/OfflineTableIT.java
b/test/src/main/java/org/apache/accumulo/test/OfflineTableIT.java
index 1ddab10e52..9cdc2e3777 100644
--- a/test/src/main/java/org/apache/accumulo/test/OfflineTableIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/OfflineTableIT.java
@@ -18,19 +18,42 @@
*/
package org.apache.accumulo.test;
+import static
org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.GROUP1;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit eae367264b0e224c59de90ba72fcae463f348357 Merge: eaa1f678c9 6784e59da3 Author: Christopher L. Shannon AuthorDate: Fri Mar 22 07:47:34 2024 -0400 Merge branch '2.1' .../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +- pom.xml| 10 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --cc pom.xml index ff26ae9641,fb747c50a8..275fc88b18 --- a/pom.xml +++ b/pom.xml @@@ -429,8 -441,19 +429,8 @@@ org.apache.commons commons-text - 1.10.0 + 1.11.0 - -org.apache.commons -commons-vfs2 -2.9.0 - - -org.apache.hadoop -hadoop-hdfs-client - - - org.apache.curator curator-framework
(accumulo) branch main updated (eaa1f678c9 -> eae367264b)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from eaa1f678c9 Merge branch '2.1' add 6784e59da3 Bump commons-configuration2 from 2.9.0 to 2.10.1 (#4412) new eae367264b Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +- pom.xml| 10 +- 2 files changed, 6 insertions(+), 6 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1d6fd65dede4c41cb2f5c64267e6f4f52e4f266d Merge: 972ab250d9 eae367264b Author: Christopher L. Shannon AuthorDate: Fri Mar 22 07:47:51 2024 -0400 Merge branch 'main' into elasticity .../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +- pom.xml| 10 +- 2 files changed, 6 insertions(+), 6 deletions(-)
(accumulo) branch elasticity updated (972ab250d9 -> 1d6fd65ded)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 972ab250d9 removes 'tx_' prefix from UserFateStore (#4408) add 6784e59da3 Bump commons-configuration2 from 2.9.0 to 2.10.1 (#4412) add eae367264b Merge branch '2.1' new 1d6fd65ded Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +- pom.xml| 10 +- 2 files changed, 6 insertions(+), 6 deletions(-)
(accumulo) branch 2.1 updated (7947c2c2df -> 6784e59da3)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7947c2c2df adds PluginEnv support to client side iterators (#4283) add 6784e59da3 Bump commons-configuration2 from 2.9.0 to 2.10.1 (#4412) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +- pom.xml| 10 +- 2 files changed, 6 insertions(+), 6 deletions(-)
(accumulo) branch elasticity updated: removes 'tx_' prefix from UserFateStore (#4408)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 972ab250d9 removes 'tx_' prefix from UserFateStore (#4408)
972ab250d9 is described below
commit 972ab250d9ee34337d66f03003eb56c603a080e4
Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com>
AuthorDate: Fri Mar 22 07:33:26 2024 -0400
removes 'tx_' prefix from UserFateStore (#4408)
---
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java | 4 ++--
.../main/java/org/apache/accumulo/core/fate/user/UserFateStore.java | 6 +++---
.../main/java/org/apache/accumulo/test/fate/user/UserFateIT.java| 2 +-
.../java/org/apache/accumulo/test/fate/user/UserFateStoreIT.java| 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
index 70f93211a4..d8cecd6a47 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
@@ -359,8 +359,8 @@ public class MetaFateStore extends AbstractFateStore {
protected Stream getTransactions() {
try {
return zk.getChildren(path).stream().map(strTxid -> {
-String hexTid = strTxid.split("_")[1];
-FateId fateId = FateId.from(fateInstanceType, hexTid);
+String txUUIDStr = strTxid.split("_")[1];
+FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
// Memoizing for two reasons. First the status may never be requested,
so in that case avoid
// the lookup. Second, if its requested multiple times the result will
always be consistent.
Supplier statusSupplier = Suppliers.memoize(() ->
_getStatus(fateId));
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java
index 88de9e8fcf..c0e6623a93 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java
@@ -157,8 +157,8 @@ public class UserFateStore extends AbstractFateStore {
scanner.setRange(new Range());
TxColumnFamily.STATUS_COLUMN.fetch(scanner);
return scanner.stream().onClose(scanner::close).map(e -> {
-String hexTid = e.getKey().getRow().toString().split("_")[1];
-FateId fateId = FateId.from(fateInstanceType, hexTid);
+String txUUIDStr = e.getKey().getRow().toString();
+FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
return new FateIdStatusBase(fateId) {
@Override
public TStatus getStatus() {
@@ -248,7 +248,7 @@ public class UserFateStore extends AbstractFateStore {
}
public static String getRowId(FateId fateId) {
-return "tx_" + fateId.getTxUUIDStr();
+return fateId.getTxUUIDStr();
}
private FateMutatorImpl newMutator(FateId fateId) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateIT.java
index 6ecbc0b5a5..4d6a8da118 100644
--- a/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateIT.java
@@ -78,6 +78,6 @@ public class UserFateIT extends FateIT {
}
private static Range getRow(FateId fateId) {
-return new Range("tx_" + fateId.getTxUUIDStr());
+return new Range(fateId.getTxUUIDStr());
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateStoreIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateStoreIT.java
index 8d286e9d2b..337f9e4bdc 100644
--- a/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateStoreIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/fate/user/UserFateStoreIT.java
@@ -258,7 +258,7 @@ public class UserFateStoreIT extends SharedMiniClusterBase {
private void injectStatus(ClientContext client, String table, FateId fateId,
TStatus status)
throws TableNotFoundException {
try (BatchWriter writer = client.createBatchWriter(table)) {
- Mutation mutation = new Mutation(new Text("tx_" +
fateId.getTxUUIDStr()));
+ Mutation mutation = new Mutation(new Text(fateId.getTxUUIDStr()));
FateSchema.TxColumnFamily.STATUS_COLUMN.put(mutation, new
Value(status.name()));
writer.addMutation(mutation);
} catch (MutationsRejectedException e) {
(accumulo) branch elasticity updated (36df8cee36 -> cca87e4e6a)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 36df8cee36 removed queued jobs for a tablet if no new jobs are seen (#4394) add cca87e4e6a Fix MetaFateStoreFateIT (#4379) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/accumulo/test/fate/FateStoreIT.java | 11 --- .../apache/accumulo/test/fate/meta/MetaFateStoreFateIT.java | 9 +++-- 2 files changed, 11 insertions(+), 9 deletions(-)
(accumulo) branch elasticity updated: Rename a couple more Fate variables (#4253)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new ac6ee4f564 Rename a couple more Fate variables (#4253)
ac6ee4f564 is described below
commit ac6ee4f56415a551d682a6b00e33f89103659856
Author: Christopher L. Shannon
AuthorDate: Sat Mar 16 15:29:27 2024 -0400
Rename a couple more Fate variables (#4253)
---
.../src/main/java/org/apache/accumulo/core/fate/AdminUtil.java | 10 +-
.../apache/accumulo/manager/metrics/fate/FateMetricValues.java | 8
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
index c18defb1ac..bea7d3518c 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/AdminUtil.java
@@ -224,7 +224,7 @@ public class AdminUtil {
* Get the FATE transaction status and lock information stored in zookeeper,
optionally filtered
* by fate id, status, and fate instance type
*
- * @param zs read-only zoostore
+ * @param mfs read-only MetaFateStore
* @param zk zookeeper reader.
* @param lockPath the zookeeper path for locks
* @param fateIdFilter filter results to include only provided fate
transaction ids
@@ -234,7 +234,7 @@ public class AdminUtil {
* @throws KeeperException if zookeeper exception occurs
* @throws InterruptedException if process is interrupted.
*/
- public FateStatus getStatus(ReadOnlyFateStore zs, ZooReader zk,
+ public FateStatus getStatus(ReadOnlyFateStore mfs, ZooReader zk,
ServiceLock.ServiceLockPath lockPath, Set fateIdFilter,
EnumSet statusFilter,
EnumSet typesFilter) throws KeeperException,
InterruptedException {
Map> heldLocks = new HashMap<>();
@@ -242,15 +242,15 @@ public class AdminUtil {
findLocks(zk, lockPath, heldLocks, waitingLocks);
-return getTransactionStatus(Map.of(FateInstanceType.META, zs),
fateIdFilter, statusFilter,
+return getTransactionStatus(Map.of(FateInstanceType.META, mfs),
fateIdFilter, statusFilter,
typesFilter, heldLocks, waitingLocks);
}
- public FateStatus getStatus(ReadOnlyFateStore as, Set
fateIdFilter,
+ public FateStatus getStatus(ReadOnlyFateStore ufs, Set
fateIdFilter,
EnumSet statusFilter, EnumSet typesFilter)
throws KeeperException, InterruptedException {
-return getTransactionStatus(Map.of(FateInstanceType.USER, as),
fateIdFilter, statusFilter,
+return getTransactionStatus(Map.of(FateInstanceType.USER, ufs),
fateIdFilter, statusFilter,
typesFilter, new HashMap<>(), new HashMap<>());
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetricValues.java
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetricValues.java
index 841cf61dd1..9850e2c7b9 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetricValues.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetricValues.java
@@ -98,11 +98,11 @@ class FateMetricValues {
*
* @param context Accumulo context
* @param fateRootPath the zookeeper path to fate info
- * @param zooStore a readonly zoostore
+ * @param metaFateStore a readonly MetaFateStore
* @return the current FATE metric values.
*/
public static FateMetricValues getFromZooKeeper(final ServerContext context,
- final String fateRootPath, final ReadOnlyFateStore
zooStore) {
+ final String fateRootPath, final ReadOnlyFateStore
metaFateStore) {
FateMetricValues.Builder builder = FateMetricValues.builder();
@@ -110,8 +110,8 @@ class FateMetricValues {
try {
- List currFates =
- admin.getTransactionStatus(Map.of(FateInstanceType.META, zooStore),
null, null, null);
+ List currFates = admin
+ .getTransactionStatus(Map.of(FateInstanceType.META, metaFateStore),
null, null, null);
builder.withCurrentFateOps(currFates.size());
(accumulo) branch elasticity updated: Rename Fate store implementations (#4393)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c766525a6a Rename Fate store implementations (#4393)
c766525a6a is described below
commit c766525a6a4dba5d1be1d76f56cdecdb225cebd9
Author: Christopher L. Shannon
AuthorDate: Sat Mar 16 15:20:04 2024 -0400
Rename Fate store implementations (#4393)
This renames ZooStore to MetaFateStore and AccumuloStore to
UserFateStore to be more descriptive and better align with what the
different Fate stores are used for. This also aligns the names with the
FateInstanceType enum.
This closes #4253
---
.../java/org/apache/accumulo/core/fate/Fate.java | 2 +-
.../fate/{ZooStore.java => MetaFateStore.java} | 15 +
.../core/fate/{accumulo => user}/FateMutator.java | 2 +-
.../fate/{accumulo => user}/FateMutatorImpl.java | 14
.../{accumulo => user}/StatusMappingIterator.java | 4 +--
.../AccumuloStore.java => user/UserFateStore.java} | 18 +-
.../fate/{accumulo => user}/schema/FateSchema.java | 2 +-
.../org/apache/accumulo/server/util/Admin.java | 12 +++
.../java/org/apache/accumulo/manager/Manager.java | 8 ++---
.../accumulo/manager/metrics/fate/FateMetrics.java | 8 ++---
.../manager/upgrade/UpgradeCoordinator.java| 4 +--
.../test/compaction/ExternalCompaction_1_IT.java | 38 +++---
.../test/fate/{accumulo => }/FateStoreIT.java | 3 +-
.../ZookeeperFateIT.java => meta/MetaFateIT.java} | 14
.../MetaFateOpsCommandsIT.java}| 8 ++---
.../MetaFateStoreFateIT.java} | 14
.../fate/{zookeeper => meta}/ZooMutatorIT.java | 2 +-
.../fate/{accumulo => user}/FateMutatorImplIT.java | 8 ++---
.../AccumuloFateIT.java => user/UserFateIT.java} | 12 +++
.../UserFateOpsCommandsIT.java}| 8 ++---
.../UserFateStoreFateIT.java} | 15 +
.../UserFateStoreIT.java} | 21 ++--
.../test/functional/FateConcurrencyIT.java | 16 +
.../test/functional/FunctionalTestUtils.java | 5 +--
24 files changed, 129 insertions(+), 124 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
b/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
index e5be68dbb2..eed785b39b 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/Fate.java
@@ -164,7 +164,7 @@ public class Fate {
// Here, deferTime is only used to determine success (zero) or
failure (non-zero),
// proceeding on success and returning to the while loop on
failure.
- // The value of deferTime is only used as a wait time in
ZooStore.unreserve
+ // The value of deferTime is only used as a wait time in
FateStore.unreserve
if (deferTime == 0) {
prevOp = op;
if (status == SUBMITTED) {
diff --git a/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
similarity index 96%
rename from core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
rename to core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
index 77f7a50a6e..70f93211a4 100644
--- a/core/src/main/java/org/apache/accumulo/core/fate/ZooStore.java
+++ b/core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java
@@ -54,9 +54,9 @@ import com.google.common.base.Suppliers;
//TODO use zoocache? - ACCUMULO-1297
//TODO handle zookeeper being down gracefully - ACCUMULO-1297
-public class ZooStore extends AbstractFateStore {
+public class MetaFateStore extends AbstractFateStore {
- private static final Logger log = LoggerFactory.getLogger(ZooStore.class);
+ private static final Logger log =
LoggerFactory.getLogger(MetaFateStore.class);
private static final FateInstanceType fateInstanceType =
FateInstanceType.META;
private String path;
private ZooReaderWriter zk;
@@ -65,13 +65,14 @@ public class ZooStore extends AbstractFateStore {
return path + "/tx_" + fateId.getTxUUIDStr();
}
- public ZooStore(String path, ZooReaderWriter zk) throws KeeperException,
InterruptedException {
+ public MetaFateStore(String path, ZooReaderWriter zk)
+ throws KeeperException, InterruptedException {
this(path, zk, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
@VisibleForTesting
- public ZooStore(String path, ZooReaderWriter zk, int maxDeferred,
FateIdGenerator fateIdGenerator)
- throws KeeperException, InterruptedException {
+ public MetaFateStore(String path, ZooReaderWriter zk, int maxDefer
(accumulo) branch elasticity updated: Update dead compaction detector to handle metadata/root (#4354)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 22621ea501 Update dead compaction detector to handle metadata/root
(#4354)
22621ea501 is described below
commit 22621ea50190389131146b92546bf5b9c2ab47cf
Author: Christopher L. Shannon
AuthorDate: Wed Mar 13 11:43:02 2024 -0400
Update dead compaction detector to handle metadata/root (#4354)
This updates DeadCompactionDetector to also look at the root and
metadata table metadata for dead compactions on those tables
This closes #4340
Co-authored-by: Keith Turner
---
.../coordinator/CompactionCoordinator.java | 19 +-
.../coordinator/DeadCompactionDetector.java| 20 +-
.../test/compaction/ExternalCompaction_1_IT.java | 231 +++--
3 files changed, 197 insertions(+), 73 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 2a12f08708..97397c019b 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -20,6 +20,9 @@ package org.apache.accumulo.manager.compaction.coordinator;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
@@ -81,6 +84,7 @@ import org.apache.accumulo.core.metadata.CompactableFileImpl;
import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.RejectionHandler;
import org.apache.accumulo.core.metadata.schema.CompactionMetadata;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
@@ -577,7 +581,7 @@ public class CompactionCoordinator
var dfv =
metaJob.getTabletMetadata().getFilesMap().get(storedTabletFile);
return new InputFile(storedTabletFile.getMetadata(), dfv.getSize(),
dfv.getNumEntries(),
dfv.getTime());
-}).collect(Collectors.toList());
+}).collect(toList());
FateInstanceType type =
FateInstanceType.fromTableId(metaJob.getTabletMetadata().getTableId());
FateId fateId = FateId.from(type, 0);
@@ -707,10 +711,19 @@ public class CompactionCoordinator
KeyExtent fromThriftExtent = KeyExtent.fromThrift(extent);
LOG.info("Compaction failed, id: {}, extent: {}", externalCompactionId,
fromThriftExtent);
final var ecid = ExternalCompactionId.of(externalCompactionId);
-compactionFailed(Map.of(ecid, KeyExtent.fromThrift(extent)));
+compactionsFailed(Map.of(ecid, KeyExtent.fromThrift(extent)));
}
- void compactionFailed(Map compactions) {
+ void compactionsFailed(Map compactions) {
+// Need to process each level by itself because the conditional tablet
mutator does not support
+// mutating multiple data levels at the same time
+compactions.entrySet().stream()
+.collect(groupingBy(entry -> DataLevel.of(entry.getValue().tableId()),
+Collectors.toMap(Entry::getKey, Entry::getValue)))
+.forEach((level, compactionsByLevel) ->
compactionFailedForLevel(compactionsByLevel));
+ }
+
+ void compactionFailedForLevel(Map
compactions) {
try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) {
compactions.forEach((ecid, extent) -> {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
index 0857385239..87a4cef78b 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/DeadCompactionDetector.java
@@ -40,6 +40,7 @@ import org.apache.accumulo.core.fate.FateKey;
import org.apache.accumulo.core.iterators.user.HasExternalCompact
(accumulo) branch elasticity updated (b095fbc87d -> d4722432a8)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from b095fbc87d Fix missing import add d4722432a8 Update TabletsMetadata stream() to autoclose TabletsMetadata - elasticity (#4357) No new revisions were added by this update. Summary of changes: .../accumulo/core/clientImpl/TableOperationsImpl.java| 5 ++--- .../apache/accumulo/gc/GarbageCollectWriteAheadLogs.java | 16 2 files changed, 6 insertions(+), 15 deletions(-)
(accumulo) branch elasticity updated: Fix missing import
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new b095fbc87d Fix missing import b095fbc87d is described below commit b095fbc87dcafce729d5e75afca46372e0aa236e Author: Christopher L. Shannon AuthorDate: Mon Mar 11 11:32:02 2024 -0400 Fix missing import --- .../org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index edd91c705a..3e1f45786e 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -49,6 +49,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import org.apache.accumulo.core.client.admin.TabletAvailability;
(accumulo) branch elasticity updated (649bf132e0 -> 83d84e501e)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 649bf132e0 Merge branch 'main' into elasticity add 5515656132 Update TabletsMetadata stream() to autoclose TabletsMetadata (#4356) add 83d84e501e Merge branch 'main' into elasticity No new revisions were added by this update. Summary of changes: .../accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java | 3 +-- server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java| 5 ++--- .../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-)
(accumulo) branch main updated: Update TabletsMetadata stream() to autoclose TabletsMetadata (#4356)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5515656132 Update TabletsMetadata stream() to autoclose
TabletsMetadata (#4356)
5515656132 is described below
commit 551565613275e202a8f6612b956789488bc6e3b1
Author: Christopher L. Shannon
AuthorDate: Mon Mar 11 11:26:07 2024 -0400
Update TabletsMetadata stream() to autoclose TabletsMetadata (#4356)
Follow on changes to #4355
Before this change if stream() was used the the TabletsMetadata close
method still had to be called manually as the close method on the stream
did not delegate. With this change if the stream is used in a
try-with-resources to autoClsoe the stream or if the stream is closed
manually, TabletsMetadata will also be closed.
---
.../accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java | 3 +--
server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java| 5 ++---
.../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 4 ++--
3 files changed, 5 insertions(+), 7 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
index acd5924ff9..e338fe99fa 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
@@ -89,8 +89,7 @@ class ConcurrentKeyExtentCache implements KeyExtentCache {
protected Stream lookupExtents(Text row) {
TabletsMetadata tabletsMetadata =
TabletsMetadata.builder(ctx).forTable(tableId)
.overlapping(row, true,
null).checkConsistency().fetch(PREV_ROW).build();
-return tabletsMetadata.stream().onClose(tabletsMetadata::close).limit(100)
-.map(TabletMetadata::getExtent);
+return tabletsMetadata.stream().limit(100).map(TabletMetadata::getExtent);
}
@Override
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
index f02b85634e..1939d6e51a 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@ -175,9 +175,8 @@ public class GCRun implements GarbageCollectionEnvironment {
if (level == Ample.DataLevel.ROOT) {
tabletStream = Stream.of(context.getAmple().readTablet(RootTable.EXTENT,
DIR, FILES, SCANS));
} else {
- TabletsMetadata tm =
TabletsMetadata.builder(context).scanTable(level.metaTable())
- .checkConsistency().fetch(DIR, FILES, SCANS).build();
- tabletStream = tm.stream().onClose(tm::close);
+ tabletStream =
TabletsMetadata.builder(context).scanTable(level.metaTable())
+ .checkConsistency().fetch(DIR, FILES, SCANS).build().stream();
}
// there is a lot going on in this "one line" so see below for more info
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index fa838231af..8fcc742ba2 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@ -159,8 +159,8 @@ public class ScanServer extends AbstractServer
Collection extents = (Collection) keys;
try (TabletsMetadata tabletsMetadata =
ample.readTablets().forTablets(extents, Optional.empty()).build()) {
-tms = tabletsMetadata.stream().onClose(tabletsMetadata::close)
-.collect(Collectors.toMap(TabletMetadata::getExtent, tm -> tm));
+tms =
+
tabletsMetadata.stream().collect(Collectors.toMap(TabletMetadata::getExtent, tm
-> tm));
}
long t2 = System.currentTimeMillis();
LOG.trace("Read metadata for {} tablets in {} ms", keys.size(), t2 - t1);
(accumulo) branch elasticity updated (8b0d235581 -> 649bf132e0)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8b0d235581 Merge remote-tracking branch 'upstream/main' into elasticity add 802b6e948d Update TabletsMetadata stream() to autoclose TabletsMetadata (#4355) add e3039f402b Merge branch '2.1' add 649bf132e0 Merge branch 'main' into elasticity No new revisions were added by this update. Summary of changes: .../core/metadata/schema/TabletsMetadata.java | 2 +- .../core/metadata/schema/TabletMetadataTest.java | 23 ++ 2 files changed, 24 insertions(+), 1 deletion(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit e3039f402bcc725dbf48e18a22f69267323935f1
Merge: 9d9bcc1d48 802b6e948d
Author: Christopher L. Shannon
AuthorDate: Mon Mar 11 11:16:11 2024 -0400
Merge branch '2.1'
.../core/metadata/schema/TabletsMetadata.java | 2 +-
.../core/metadata/schema/TabletMetadataTest.java | 25 ++
2 files changed, 26 insertions(+), 1 deletion(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 5287d689ba,67a22ccd3f..9a352b82d5
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -41,7 -41,8 +43,9 @@@ import java.util.Map
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.UUID;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.stream.Stream;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@@ -259,30 -259,27 +263,51 @@@ public class TabletMetadataTest
assertFalse(tm.hasCurrent());
}
+ @Test
+ public void testMergedColumn() {
+KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+// Test merged column set
+Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+MERGED_COLUMN.put(mutation, MERGED_VALUE);
+TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.MERGED), true);
+assertTrue(tm.hasMerged());
+
+// Column not set
+mutation = TabletColumnFamily.createPrevRowMutation(extent);
+tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.MERGED), true);
+assertFalse(tm.hasMerged());
+
+// MERGED Column not fetched
+mutation = TabletColumnFamily.createPrevRowMutation(extent);
+tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.PREV_ROW), true);
+assertThrows(IllegalStateException.class, tm::hasMerged);
+ }
+
+ @Test
+ public void testTabletsMetadataAutoClose() throws Exception {
+ AtomicBoolean closeCalled = new AtomicBoolean();
+ AutoCloseable autoCloseable = () -> closeCalled.set(true);
+ Constructor tmConstructor =
+ TabletsMetadata.class.getDeclaredConstructor(AutoCloseable.class,
Iterable.class);
+ tmConstructor.setAccessible(true);
+
+ try (TabletsMetadata ignored = tmConstructor.newInstance(autoCloseable,
List.of())) {
+ // test autoCloseable used directly on TabletsMetadata
+ }
+ assertTrue(closeCalled.get());
+
+ closeCalled.set(false);
+ try (Stream ignored =
+ tmConstructor.newInstance(autoCloseable, List.of()).stream()) {
+ // test stream delegates to close on TabletsMetadata
+ }
+ assertTrue(closeCalled.get());
+ }
+
private SortedMap toRowMap(Mutation mutation) {
SortedMap rowMap = new TreeMap<>();
mutation.getUpdates().forEach(cu -> {
(accumulo) branch main updated (9d9bcc1d48 -> e3039f402b)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9d9bcc1d48 Merge remote-tracking branch 'upstream/2.1' add 802b6e948d Update TabletsMetadata stream() to autoclose TabletsMetadata (#4355) new e3039f402b Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/metadata/schema/TabletsMetadata.java | 2 +- .../core/metadata/schema/TabletMetadataTest.java | 25 ++ 2 files changed, 26 insertions(+), 1 deletion(-)
(accumulo) branch 2.1 updated: Update TabletsMetadata stream() to autoclose TabletsMetadata (#4355)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 802b6e948d Update TabletsMetadata stream() to autoclose
TabletsMetadata (#4355)
802b6e948d is described below
commit 802b6e948d818fb5a32a3a0cc1db9757200cdf34
Author: Christopher L. Shannon
AuthorDate: Mon Mar 11 11:13:18 2024 -0400
Update TabletsMetadata stream() to autoclose TabletsMetadata (#4355)
Before this change if stream() was used the the TabletsMetadata close
method still had to be called manually as the close method on the stream
did not delegate. With this change if the stream is used in a
try-with-resources to autoClsoe the stream or if the stream is closed
manually, TabletsMetadata will also be closed.
---
.../core/metadata/schema/TabletsMetadata.java | 2 +-
.../core/metadata/schema/TabletMetadataTest.java | 25 ++
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 5f8ce56a94..6ce050c222 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -603,6 +603,6 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
}
public Stream stream() {
-return StreamSupport.stream(tablets.spliterator(), false);
+return StreamSupport.stream(tablets.spliterator(),
false).onClose(this::close);
}
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index f30ff5bd4d..67a22ccd3f 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@ -33,12 +33,16 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.lang.reflect.Constructor;
import java.util.EnumSet;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Stream;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@ -255,6 +259,27 @@ public class TabletMetadataTest {
assertFalse(tm.hasCurrent());
}
+ @Test
+ public void testTabletsMetadataAutoClose() throws Exception {
+AtomicBoolean closeCalled = new AtomicBoolean();
+AutoCloseable autoCloseable = () -> closeCalled.set(true);
+Constructor tmConstructor =
+TabletsMetadata.class.getDeclaredConstructor(AutoCloseable.class,
Iterable.class);
+tmConstructor.setAccessible(true);
+
+try (TabletsMetadata ignored = tmConstructor.newInstance(autoCloseable,
List.of())) {
+ // test autoCloseable used directly on TabletsMetadata
+}
+assertTrue(closeCalled.get());
+
+closeCalled.set(false);
+try (Stream ignored =
+tmConstructor.newInstance(autoCloseable, List.of()).stream()) {
+ // test stream delegates to close on TabletsMetadata
+}
+assertTrue(closeCalled.get());
+ }
+
private SortedMap toRowMap(Mutation mutation) {
SortedMap rowMap = new TreeMap<>();
mutation.getUpdates().forEach(cu -> {
(accumulo) branch elasticity updated: Move the tracking of unsplittable tablets to metadata table (#4317)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 490a4634f6 Move the tracking of unsplittable tablets to metadata table
(#4317)
490a4634f6 is described below
commit 490a4634f6a299b26287bda7ae596572b2ee0808
Author: Christopher L. Shannon
AuthorDate: Fri Mar 8 17:04:12 2024 -0500
Move the tracking of unsplittable tablets to metadata table (#4317)
This adds a new column to store information for tracking unsplittable
tablets in the metadata table instead of in memory. This information
can be used by the tablet management iterator to know if a tablet needs
to split and avoid unnecessarily trying to split a tablet that can't be
split. The data stored includes a hash of the file set and the relevant
config related to splits and if this changes then the iterator will try
and split again and retest.
Co-authored-by: Keith Turner
---
.../accumulo/core/metadata/schema/Ample.java | 4 +
.../core/metadata/schema/MetadataSchema.java | 13 ++
.../core/metadata/schema/TabletMetadata.java | 36 -
.../metadata/schema/TabletMetadataBuilder.java | 13 ++
.../core/metadata/schema/TabletMutatorBase.java| 13 ++
.../core/metadata/schema/TabletsMetadata.java | 4 +
.../core/metadata/schema/UnSplittableMetadata.java | 116 ++
.../java/org/apache/accumulo/core/util/Merge.java | 7 +-
.../core/metadata/schema/TabletMetadataTest.java | 110 -
.../server/constraints/MetadataConstraints.java| 20 ++-
.../manager/state/TabletManagementIterator.java| 37 +++--
.../apache/accumulo/server}/split/SplitUtils.java | 46 +-
.../constraints/MetadataConstraintsTest.java | 60 ++-
.../accumulo/server}/split/SplitUtilsTest.java | 2 +-
.../manager/tableOps/split/FindSplits.java | 82 +-
.../manager/tableOps/split/UpdateTablets.java | 6 +
.../manager/tableOps/split/UpdateTabletsTest.java | 7 +-
.../org/apache/accumulo/test/LargeSplitRowIT.java | 177 -
18 files changed, 708 insertions(+), 45 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index d3f27a7f36..d85553f75e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -419,6 +419,10 @@ public interface Ample {
T putUserCompactionRequested(FateId fateId);
T deleteUserCompactionRequested(FateId fateId);
+
+T setUnSplittable(UnSplittableMetadata unSplittableMeta);
+
+T deleteUnSplittable();
}
interface TabletMutator extends TabletUpdates {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 46cbfb7b06..b093a4bccf 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -430,6 +430,19 @@ public class MetadataSchema {
public static final Text NAME = new Text(STR_NAME);
}
+/**
+ * This family is used to track information needed for splits. Currently,
the only thing stored
+ * is if the tablets are un-splittable based on the files the tablet and
configuration related
+ * to splits.
+ */
+public static class SplitColumnFamily {
+ public static final String STR_NAME = "split";
+ public static final Text NAME = new Text(STR_NAME);
+ public static final String UNSPLITTABLE_QUAL = "unsplittable";
+ public static final ColumnFQ UNSPLITTABLE_COLUMN =
+ new ColumnFQ(NAME, new Text(UNSPLITTABLE_QUAL));
+}
+
// TODO when removing the Upgrader12to13 class in the upgrade package,
also remove this class.
public static class Upgrade12to13 {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index ccdb2acaec..be7d2fb056 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -70,6 +70,7 @@ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Lo
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileC
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit afe2857935b42067cbce86ad93e85c567df70f52
Merge: 8798fccd5c c976af383f
Author: Christopher L. Shannon
AuthorDate: Fri Mar 1 17:26:38 2024 -0500
Merge branch 'main' into elasticity
.../accumulo/test/functional/MetadataIT.java | 69 ++
1 file changed, 69 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
index 63f1b30a7a,60ca6cb562..5a4dd306cc
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
@@@ -40,7 -41,8 +41,9 @@@ import org.apache.accumulo.core.client.
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Scanner;
++import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.clientImpl.ClientContext;
+ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
@@@ -221,4 -227,69 +226,68 @@@ public class MetadataIT extends Accumul
}
}
+
+ // Test that configs related to the correctness of the Root/Metadata tables
+ // are initialized correctly
+ @Test
+ public void testSystemTablesInitialConfigCorrectness() throws Exception {
+ try (ClientContext client =
+ (ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
+
+ // It is important here to use getTableProperties() and not
getConfiguration()
+ // because we want only the table properties and not a merged view
+ var rootTableProps =
+
client.tableOperations().getTableProperties(AccumuloTable.ROOT.tableName());
+ var metadataTableProps =
+
client.tableOperations().getTableProperties(AccumuloTable.METADATA.tableName());
+
+ // Verify root table config
- testCommonSystemTableConfig(rootTableProps);
- assertEquals("root",
-
rootTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
++ testCommonSystemTableConfig(client, AccumuloTable.ROOT.tableId(),
rootTableProps);
+
+ // Verify metadata table config
- testCommonSystemTableConfig(metadataTableProps);
- assertEquals("meta",
-
metadataTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
++ testCommonSystemTableConfig(client, AccumuloTable.METADATA.tableId(),
metadataTableProps);
+ }
+ }
+
- private void testCommonSystemTableConfig(Map tableProps) {
++ private void testCommonSystemTableConfig(ClientContext client, TableId
tableId,
++ Map tableProps) {
+ // Verify properties all have a table. prefix
+ assertTrue(tableProps.keySet().stream().allMatch(key ->
key.startsWith("table.")));
+
+ // Verify properties are correctly set
+ assertEquals("5",
tableProps.get(Property.TABLE_FILE_REPLICATION.getKey()));
+ assertEquals("sync", tableProps.get(Property.TABLE_DURABILITY.getKey()));
+ assertEquals("false",
tableProps.get(Property.TABLE_FAILURES_IGNORE.getKey()));
+ assertEquals("",
tableProps.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey()));
+ assertEquals("tablet,server",
tableProps.get(Property.TABLE_LOCALITY_GROUPS.getKey()));
+ assertEquals(
+ String.format("%s,%s",
MetadataSchema.TabletsSection.TabletColumnFamily.NAME,
+ MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME),
+ tableProps.get(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"tablet"));
+ assertEquals(
+ String.format("%s,%s,%s,%s",
MetadataSchema.TabletsSection.DataFileColumnFamily.NAME,
+ MetadataSchema.TabletsSection.LogColumnFamily.NAME,
+ MetadataSchema.TabletsSection.ServerColumnFamily.NAME,
+ MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME),
+ tableProps.get(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"server"));
-assertEquals("20," + MetadataBulkLoadFilter.class.getName(),
-tableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.bulkLoadFilter"));
-assertEquals(SimpleCompactionDispatcher.class.getName(),
-tableProps.get(Property.TABLE_COMPACTION_DISPATCHER.getKey()));
+
+ // Verify VersioningIterator related properties are correct
+ var iterClass = "10," + VersioningIterator.class.getName();
+ var maxVersions = "1";
+ assertEquals(iterClass,
tableProps.get(Property.TABLE_ITERATOR
(accumulo) branch elasticity updated (8798fccd5c -> afe2857935)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 8798fccd5c updates IdleProcessMetricsIT to verify config instead of set it. (#4331) add 507a96380b Add test to verify Root and Metadata tables are correctly initalized (#4332) add c976af383f Merge branch '2.1' new afe2857935 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/test/functional/MetadataIT.java | 69 ++ 1 file changed, 69 insertions(+)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c976af383f992bdefcdd579220c704235960a6c7
Merge: 9817474333 507a96380b
Author: Christopher L. Shannon
AuthorDate: Fri Mar 1 17:21:36 2024 -0500
Merge branch '2.1'
.../accumulo/test/functional/MetadataIT.java | 71 ++
1 file changed, 71 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
index 0b3116c395,3c6514624e..60ca6cb562
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
@@@ -45,7 -48,10 +47,9 @@@ import org.apache.accumulo.core.data.Ke
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.iterators.user.VersioningIterator;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.AccumuloTable;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
@@@ -221,4 -224,67 +227,69 @@@ public class MetadataIT extends Accumul
}
}
+
+ // Test that configs related to the correctness of the Root/Metadata tables
+ // are initialized correctly
+ @Test
+ public void testSystemTablesInitialConfigCorrectness() throws Exception {
+ try (ClientContext client =
+ (ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
+
+ // It is important here to use getTableProperties() and not
getConfiguration()
+ // because we want only the table properties and not a merged view
- var rootTableProps =
client.tableOperations().getTableProperties(RootTable.NAME);
- var metadataTableProps =
client.tableOperations().getTableProperties(MetadataTable.NAME);
++ var rootTableProps =
++
client.tableOperations().getTableProperties(AccumuloTable.ROOT.tableName());
++ var metadataTableProps =
++
client.tableOperations().getTableProperties(AccumuloTable.METADATA.tableName());
+
+ // Verify root table config
+ testCommonSystemTableConfig(rootTableProps);
+ assertEquals("root",
+
rootTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
+
+ // Verify metadata table config
+ testCommonSystemTableConfig(metadataTableProps);
+ assertEquals("meta",
+
metadataTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
+ }
+ }
+
+ private void testCommonSystemTableConfig(Map tableProps) {
+ // Verify properties all have a table. prefix
+ assertTrue(tableProps.keySet().stream().allMatch(key ->
key.startsWith("table.")));
+
+ // Verify properties are correctly set
+ assertEquals("5",
tableProps.get(Property.TABLE_FILE_REPLICATION.getKey()));
+ assertEquals("sync", tableProps.get(Property.TABLE_DURABILITY.getKey()));
+ assertEquals("false",
tableProps.get(Property.TABLE_FAILURES_IGNORE.getKey()));
+ assertEquals("",
tableProps.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey()));
+ assertEquals("tablet,server",
tableProps.get(Property.TABLE_LOCALITY_GROUPS.getKey()));
+ assertEquals(
+ String.format("%s,%s",
MetadataSchema.TabletsSection.TabletColumnFamily.NAME,
+ MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME),
+ tableProps.get(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"tablet"));
+ assertEquals(
+ String.format("%s,%s,%s,%s",
MetadataSchema.TabletsSection.DataFileColumnFamily.NAME,
+ MetadataSchema.TabletsSection.LogColumnFamily.NAME,
+ MetadataSchema.TabletsSection.ServerColumnFamily.NAME,
+ MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME),
+ tableProps.get(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"server"));
+ assertEquals("20," + MetadataBulkLoadFilter.class.getName(),
+ tableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.bulkLoadFilter"));
+ assertEquals(SimpleCompactionDispatcher.class.getName(),
+ tableProps.get(Property.TABLE_COMPACTION_DISPATCHER.getKey()));
+
+ // Verify VersioningIterator related properties are correct
+ var iterClass = "10," + VersioningIterator.class.getName();
+ var maxVe
(accumulo) branch main updated (9817474333 -> c976af383f)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9817474333 Merge branch '2.1' add 507a96380b Add test to verify Root and Metadata tables are correctly initalized (#4332) new c976af383f Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/test/functional/MetadataIT.java | 71 ++ 1 file changed, 71 insertions(+)
(accumulo) branch 2.1 updated: Add test to verify Root and Metadata tables are correctly initalized (#4332)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 507a96380b Add test to verify Root and Metadata tables are correctly
initalized (#4332)
507a96380b is described below
commit 507a96380beac6ce0da95cc3250bc5d1964a05f5
Author: Christopher L. Shannon
AuthorDate: Fri Mar 1 17:18:45 2024 -0500
Add test to verify Root and Metadata tables are correctly initalized (#4332)
This adds tests to verify that the table properties that are related
to correctness of the system tables (root and metadata) are set properly
when the configuration is initialized
---
.../accumulo/test/functional/MetadataIT.java | 69 ++
1 file changed, 69 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
index 119ede2111..3c6514624e 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
@@ -30,6 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
@@ -42,12 +43,15 @@ import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
@@ -55,8 +59,10 @@ import
org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
@@ -218,4 +224,67 @@ public class MetadataIT extends AccumuloClusterHarness {
}
}
+
+ // Test that configs related to the correctness of the Root/Metadata tables
+ // are initialized correctly
+ @Test
+ public void testSystemTablesInitialConfigCorrectness() throws Exception {
+try (ClientContext client =
+(ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
+
+ // It is important here to use getTableProperties() and not
getConfiguration()
+ // because we want only the table properties and not a merged view
+ var rootTableProps =
client.tableOperations().getTableProperties(RootTable.NAME);
+ var metadataTableProps =
client.tableOperations().getTableProperties(MetadataTable.NAME);
+
+ // Verify root table config
+ testCommonSystemTableConfig(rootTableProps);
+ assertEquals("root",
+
rootTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
+
+ // Verify metadata table config
+ testCommonSystemTableConfig(metadataTableProps);
+ assertEquals("meta",
+
metadataTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
+}
+ }
+
+ private void testCommonSystemTableConfig(Map tableProps) {
+// Verify properties all have a table. prefix
+assertTrue(tableProps.keySet().stream().allMatch(key ->
key.startsWith("table.")));
+
+// Verify properties are correctly set
+assertEquals("5",
tableProps.get(Property.TABLE_FILE_REPLICATION.getKey()));
+assertEquals("sync", tableProps.get(Property.TABLE_DURABILITY.getKey()));
+assertEquals("false",
tableProps.get(Property.TABLE_FAILURES_IGNORE.getKey()));
+assertEquals("",
tableProps.get(Property.TAB
(accumulo) branch elasticity updated: Update FATE tests to create fate table with system config (#4327)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 01a8a2759d Update FATE tests to create fate table with system config
(#4327)
01a8a2759d is described below
commit 01a8a2759d065b9f22e1a572c2fd3c76feea5cd2
Author: Christopher L. Shannon
AuthorDate: Fri Mar 1 11:19:03 2024 -0500
Update FATE tests to create fate table with system config (#4327)
This commit updates the different FATE ITs to create the test table that
is used to match the same configuration that is used when creating the
system table in InitialConfiguration.
This closes #4321
Co-authored-by: Keith Turner
---
.../test/fate/accumulo/AccumuloFateIT.java | 4 ++-
.../test/fate/accumulo/AccumuloStoreFateIT.java| 3 ++-
.../test/fate/accumulo/AccumuloStoreIT.java| 30 --
3 files changed, 33 insertions(+), 4 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloFateIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloFateIT.java
index 3091f8c88f..eb3743326e 100644
---
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloFateIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloFateIT.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.test.fate.accumulo;
+import static
org.apache.accumulo.test.fate.accumulo.AccumuloStoreIT.createFateTable;
+
import java.util.stream.StreamSupport;
import org.apache.accumulo.core.client.Accumulo;
@@ -57,7 +59,7 @@ public class AccumuloFateIT extends FateIT {
table = getUniqueNames(1)[0];
try (ClientContext client =
(ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
- client.tableOperations().create(table);
+ createFateTable(client, table);
testMethod.execute(new AccumuloStore<>(client, table, maxDeferred,
fateIdGenerator),
getCluster().getServerContext());
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreFateIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreFateIT.java
index da0b62e14d..11bdf48e15 100644
---
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreFateIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreFateIT.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.test.fate.accumulo;
import static org.apache.accumulo.core.fate.accumulo.AccumuloStore.getRowId;
+import static
org.apache.accumulo.test.fate.accumulo.AccumuloStoreIT.createFateTable;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.BatchWriter;
@@ -53,7 +54,7 @@ public class AccumuloStoreFateIT extends FateStoreIT {
String table = getUniqueNames(1)[0] + "fatestore";
try (ClientContext client =
(ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
- client.tableOperations().create(table);
+ createFateTable(client, table);
testMethod.execute(new AccumuloStore<>(client, table, maxDeferred,
fateIdGenerator),
getCluster().getServerContext());
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
index 4a5e0c18c7..cb38235242 100644
---
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
@@ -33,10 +33,13 @@ import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.client.admin.TabletInformation;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.fate.FateInstanceType;
@@ -59,6 +62,8 @@ import org.junit.jupiter.api.function.Executable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.MoreCollectors;
+
public class AccumuloStoreIT extends SharedMiniClusterBase {
private static final Logger log =
LoggerFactory.getLogger(AccumuloStore.class);
@@ -148,7 +153,7 @@ public class AccumuloStoreIT extends SharedMiniClusterBa
(accumulo) branch elasticity updated: Add test to verify correctness of Accumulo fate table (#4328)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5ccb9195ac Add test to verify correctness of Accumulo fate table
(#4328)
5ccb9195ac is described below
commit 5ccb9195ac39297bce23a1a8e941cf11ce0477d0
Author: Christopher L. Shannon
AuthorDate: Fri Mar 1 11:15:28 2024 -0500
Add test to verify correctness of Accumulo fate table (#4328)
This test makes sure configuration that is relevant to the correctness
of the Fate table, such as the versioning iterator, are set correctly.
This closes #4322
---
.../test/fate/accumulo/AccumuloStoreIT.java| 51 ++
1 file changed, 51 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
index 785b52b290..4a5e0c18c7 100644
---
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.test.fate.accumulo;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Iterator;
import java.util.List;
@@ -32,7 +33,9 @@ import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.fate.FateId;
@@ -41,6 +44,8 @@ import org.apache.accumulo.core.fate.FateStore;
import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus;
import org.apache.accumulo.core.fate.accumulo.AccumuloStore;
import org.apache.accumulo.core.fate.accumulo.schema.FateSchema;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.test.fate.FateIT;
import org.apache.hadoop.io.Text;
@@ -92,6 +97,52 @@ public class AccumuloStoreIT extends SharedMiniClusterBase {
}
}
+ // Test that configs related to the correctness of the FATE instance user
table
+ // are initialized correctly
+ @Test
+ public void testFateInitialConfigCorrectness() throws Exception {
+try (ClientContext client =
+(ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
+
+ // It is important here to use getTableProperties() and not
getConfiguration()
+ // because we want only the table properties and not a merged view
+ var fateTableProps =
+
client.tableOperations().getTableProperties(AccumuloTable.FATE.tableName());
+
+ // Verify properties all have a table. prefix
+ assertTrue(fateTableProps.keySet().stream().allMatch(key ->
key.startsWith("table.")));
+
+ // Verify properties are correctly set
+ assertEquals("5",
fateTableProps.get(Property.TABLE_FILE_REPLICATION.getKey()));
+ assertEquals("sync",
fateTableProps.get(Property.TABLE_DURABILITY.getKey()));
+ assertEquals("false",
fateTableProps.get(Property.TABLE_FAILURES_IGNORE.getKey()));
+ assertEquals("",
fateTableProps.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey()));
+
+ // Verify VersioningIterator related properties are correct
+ var iterClass = "10," + VersioningIterator.class.getName();
+ var maxVersions = "1";
+ assertEquals(iterClass,
+ fateTableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"scan.vers"));
+ assertEquals(maxVersions, fateTableProps
+ .get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"scan.vers.opt.maxVersions"));
+ assertEquals(iterClass,
+ fateTableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"minc.vers"));
+ assertEquals(maxVersions, fateTableProps
+ .get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"minc.vers.opt.maxVersions"));
+ assertEquals(iterClass,
+ fateTableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.vers"));
+ assertEquals(maxVersions, fateTableProps
+ .get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.vers.o
(accumulo) branch elasticity updated: Add new metadata column to prevent user compaction starvation (#4254)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 3310612bc8 Add new metadata column to prevent user compaction
starvation (#4254)
3310612bc8 is described below
commit 3310612bc8216b2fca311a189b9e68dce8f1e193
Author: Christopher L. Shannon
AuthorDate: Mon Feb 26 13:33:14 2024 -0500
Add new metadata column to prevent user compaction starvation (#4254)
This commit adds a new column that will be set by user compactions
during the fate operation if it is detected that it can't run due to a
system compaction. This will prevent future system compactions from
running until the user compaction is finished.
Co-authored-by: Keith Turner
---
.../core/manager/state/TabletManagement.java | 8 +-
.../accumulo/core/metadata/schema/Ample.java | 4 +
.../core/metadata/schema/MetadataSchema.java | 9 ++
.../core/metadata/schema/TabletMetadata.java | 18 +++-
.../metadata/schema/TabletMetadataBuilder.java | 13 +++
.../core/metadata/schema/TabletMutatorBase.java| 13 +++
.../core/metadata/schema/TabletsMetadata.java | 4 +
.../core/metadata/schema/TabletMetadataTest.java | 43 +++-
.../server/constraints/MetadataConstraints.java| 10 +-
.../manager/state/TabletManagementIterator.java| 2 +
.../metadata/ConditionalTabletMutatorImpl.java | 8 ++
.../constraints/MetadataConstraintsTest.java | 16 ++-
.../coordinator/CompactionCoordinator.java | 8 +-
.../accumulo/manager/tableOps/compact/CleanUp.java | 19 +++-
.../manager/tableOps/compact/CompactionDriver.java | 41 +--
.../compaction/ExternalCompactionTestUtils.java| 1 +
.../test/functional/AmpleConditionalWriterIT.java | 89 +++
.../accumulo/test/functional/CompactionIT.java | 120 +
18 files changed, 398 insertions(+), 28 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
index 04d956dd8c..911dfc8deb 100644
---
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
+++
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
@@ -42,10 +42,10 @@ import com.google.common.base.Splitter;
*/
public class TabletManagement {
- public static final EnumSet CONFIGURED_COLUMNS =
- EnumSet.of(ColumnType.PREV_ROW, ColumnType.LOCATION, ColumnType.SUSPEND,
ColumnType.LOGS,
- ColumnType.AVAILABILITY, ColumnType.HOSTING_REQUESTED,
ColumnType.FILES, ColumnType.LAST,
- ColumnType.OPID, ColumnType.ECOMP, ColumnType.DIR,
ColumnType.SELECTED);
+ public static final EnumSet CONFIGURED_COLUMNS =
EnumSet.of(ColumnType.PREV_ROW,
+ ColumnType.LOCATION, ColumnType.SUSPEND, ColumnType.LOGS,
ColumnType.AVAILABILITY,
+ ColumnType.HOSTING_REQUESTED, ColumnType.FILES, ColumnType.LAST,
ColumnType.OPID,
+ ColumnType.ECOMP, ColumnType.DIR, ColumnType.SELECTED,
ColumnType.USER_COMPACTION_REQUESTED);
private static final Text ERROR_COLUMN_NAME = new Text("ERROR");
private static final Text REASONS_COLUMN_NAME = new Text("REASONS");
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 18d54d1f25..d3f27a7f36 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -415,6 +415,10 @@ public interface Ample {
T setMerged();
T deleteMerged();
+
+T putUserCompactionRequested(FateId fateId);
+
+T deleteUserCompactionRequested(FateId fateId);
}
interface TabletMutator extends TabletUpdates {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 852eb257c8..46cbfb7b06 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -421,6 +421,15 @@ public class MetadataSchema {
public static final Text NAME = new Text(STR_NAME);
}
+/**
+ * Column family for indicating that a user has requested to compact a
tablet. The column
+ * qualifier is expected to contain the fate transaction id that is
executing the request.
+ */
+public static class UserCompactionRequestedColumnFamily {
+ public static final String STR_NAME = "userRequestToCompact";
+ public static final Text NAME = new Text(STR_NAME);
+}
+
// TODO wh
(accumulo) branch elasticity updated: Replace boolean with enum for table locks in Elasticity (#4298)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c720dd4e6d Replace boolean with enum for table locks in Elasticity
(#4298)
c720dd4e6d is described below
commit c720dd4e6d865ea8b7d5d8624c29a488a3abcfa2
Author: Christopher L. Shannon
AuthorDate: Mon Feb 26 13:28:19 2024 -0500
Replace boolean with enum for table locks in Elasticity (#4298)
Instead of using a boolean value for obtaining table locks, use an enum
which makes the code more readable and makes it easier to search and
catch errors with the lock type.
This closes #4276
---
.../fate/zookeeper/DistributedReadWriteLock.java | 2 +-
.../manager/tableOps/ChangeTableState.java | 13 +++---
.../apache/accumulo/manager/tableOps/Utils.java| 53 +++---
.../availability/SetTabletAvailability.java| 13 +++---
.../manager/tableOps/clone/CloneTable.java | 11 +++--
.../manager/tableOps/clone/CloneZookeeper.java | 9 ++--
.../manager/tableOps/clone/FinishCloneTable.java | 9 ++--
.../manager/tableOps/compact/CompactRange.java | 10 ++--
.../tableOps/compact/cancel/CancelCompactions.java | 10 ++--
.../compact/cancel/FinishCancelCompaction.java | 5 +-
.../manager/tableOps/create/CreateTable.java | 7 +--
.../manager/tableOps/create/FinishCreateTable.java | 5 +-
.../manager/tableOps/create/PopulateZookeeper.java | 5 +-
.../accumulo/manager/tableOps/delete/CleanUp.java | 5 +-
.../manager/tableOps/delete/DeleteTable.java | 10 ++--
.../manager/tableOps/delete/PreDeleteTable.java| 14 +++---
.../manager/tableOps/merge/FinishTableRangeOp.java | 5 +-
.../manager/tableOps/merge/TableRangeOp.java | 10 ++--
.../namespace/create/FinishCreateNamespace.java| 3 +-
.../create/PopulateZookeeperWithNamespace.java | 7 +--
.../tableOps/namespace/delete/DeleteNamespace.java | 5 +-
.../namespace/delete/NamespaceCleanUp.java | 3 +-
.../tableOps/namespace/rename/RenameNamespace.java | 7 +--
.../manager/tableOps/rename/RenameTable.java | 14 +++---
.../tableOps/tableExport/WriteExportFiles.java | 15 +++---
.../tableOps/tableImport/FinishImportTable.java| 5 +-
.../tableImport/ImportPopulateZookeeper.java | 5 +-
.../manager/tableOps/tableImport/ImportTable.java | 7 +--
28 files changed, 150 insertions(+), 117 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
b/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
index 84b3c1148c..0bf4af19c7 100644
---
a/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
+++
b/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
*/
public class DistributedReadWriteLock implements
java.util.concurrent.locks.ReadWriteLock {
- public static enum LockType {
+ public enum LockType {
READ, WRITE,
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
index 17fb9b40c0..378f72460d 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
@@ -25,6 +25,7 @@ import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.fate.Repo;
+import
org.apache.accumulo.core.fate.zookeeper.DistributedReadWriteLock.LockType;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.manager.Manager;
import org.slf4j.LoggerFactory;
@@ -53,8 +54,8 @@ public class ChangeTableState extends ManagerRepo {
public long isReady(FateId fateId, Manager env) throws Exception {
// reserve the table so that this op does not run concurrently with
create, clone, or delete
// table
-return Utils.reserveNamespace(env, namespaceId, fateId, false, true, top)
-+ Utils.reserveTable(env, tableId, fateId, true, true, top);
+return Utils.reserveNamespace(env, namespaceId, fateId, LockType.READ,
true, top)
++ Utils.reserveTable(env, tableId, fateId, LockType.WRITE, true, top);
}
@Override
@@ -65,8 +66,8 @@ public class ChangeTableState extends ManagerRepo {
}
env.getTableManager().transitionTableState(tableId, ts,
expectedCurrStates);
-Utils.unreserveNamespace(env, namespaceId, fateId, false);
-Utils.unreserveTable(env, tableId, fateId, true);
+Utils.unreserveNamespace(env
(accumulo) branch elasticity updated (d39d304c5d -> ce36ac583e)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from d39d304c5d Allow a status of Successful when popping FATE repos (#4302) add 3b1566fd99 Replace boolean with enum for table locks in 3.1 (#4297) new ce36ac583e Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ce36ac583ec524f830a0184cb7d917c71f4c4846 Merge: d39d304c5d 3b1566fd99 Author: Christopher L. Shannon AuthorDate: Mon Feb 26 13:20:51 2024 -0500 Merge branch 'main' into elasticity
(accumulo) branch main updated: Replace boolean with enum for table locks in 3.1 (#4297)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3b1566fd99 Replace boolean with enum for table locks in 3.1 (#4297)
3b1566fd99 is described below
commit 3b1566fd992de76eb9251a37ceb5a5b18bb4a468
Author: Christopher L. Shannon
AuthorDate: Mon Feb 26 13:20:22 2024 -0500
Replace boolean with enum for table locks in 3.1 (#4297)
Instead of using a boolean value for obtaining table locks, use an enum
which makes the code more readable and makes it easier to search and
catch errors with the lock type.
This closes #4276
---
.../fate/zookeeper/DistributedReadWriteLock.java | 2 +-
.../manager/tableOps/ChangeTableState.java | 13 +++---
.../apache/accumulo/manager/tableOps/Utils.java| 49 +++---
.../manager/tableOps/clone/CloneTable.java | 11 ++---
.../manager/tableOps/clone/CloneZookeeper.java | 11 ++---
.../manager/tableOps/clone/FinishCloneTable.java | 9 ++--
.../manager/tableOps/compact/CompactRange.java | 10 +++--
.../tableOps/compact/cancel/CancelCompactions.java | 10 +++--
.../compact/cancel/FinishCancelCompaction.java | 5 ++-
.../manager/tableOps/create/CreateTable.java | 5 ++-
.../manager/tableOps/create/FinishCreateTable.java | 5 ++-
.../manager/tableOps/create/PopulateZookeeper.java | 5 ++-
.../accumulo/manager/tableOps/delete/CleanUp.java | 5 ++-
.../manager/tableOps/delete/DeleteTable.java | 10 +++--
.../manager/tableOps/delete/PreDeleteTable.java| 13 +++---
.../manager/tableOps/merge/TableRangeOp.java | 9 ++--
.../manager/tableOps/merge/TableRangeOpWait.java | 5 ++-
.../namespace/create/FinishCreateNamespace.java| 3 +-
.../create/PopulateZookeeperWithNamespace.java | 5 ++-
.../tableOps/namespace/delete/DeleteNamespace.java | 6 ++-
.../namespace/delete/NamespaceCleanUp.java | 3 +-
.../tableOps/namespace/rename/RenameNamespace.java | 8 ++--
.../manager/tableOps/rename/RenameTable.java | 13 +++---
.../tableOps/tableExport/WriteExportFiles.java | 14 ---
.../tableOps/tableImport/FinishImportTable.java| 5 ++-
.../tableImport/ImportPopulateZookeeper.java | 5 ++-
.../manager/tableOps/tableImport/ImportTable.java | 5 ++-
.../accumulo/test/fate/zookeeper/FateIT.java | 14 ---
28 files changed, 146 insertions(+), 112 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
b/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
index 84b3c1148c..0bf4af19c7 100644
---
a/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
+++
b/core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
*/
public class DistributedReadWriteLock implements
java.util.concurrent.locks.ReadWriteLock {
- public static enum LockType {
+ public enum LockType {
READ, WRITE,
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
index 146ea96c84..496565c0cc 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/ChangeTableState.java
@@ -24,6 +24,7 @@ import
org.apache.accumulo.core.clientImpl.thrift.TableOperation;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.fate.Repo;
+import
org.apache.accumulo.core.fate.zookeeper.DistributedReadWriteLock.LockType;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.manager.Manager;
import org.slf4j.LoggerFactory;
@@ -52,8 +53,8 @@ public class ChangeTableState extends ManagerRepo {
public long isReady(long tid, Manager env) throws Exception {
// reserve the table so that this op does not run concurrently with
create, clone, or delete
// table
-return Utils.reserveNamespace(env, namespaceId, tid, false, true, top)
-+ Utils.reserveTable(env, tableId, tid, true, true, top);
+return Utils.reserveNamespace(env, namespaceId, tid, LockType.READ, true,
top)
++ Utils.reserveTable(env, tableId, tid, LockType.WRITE, true, top);
}
@Override
@@ -64,8 +65,8 @@ public class ChangeTableState extends ManagerRepo {
}
env.getTableManager().transitionTableState(tableId, ts,
expectedCurrStates);
-Utils.unreserveNamespace(env, namespaceId, tid, false);
-Utils.unreserveTable(env, tableId, tid, true);
+Utils.unreserveNamespace(env, namespaceId
(accumulo) branch elasticity updated: Allow a status of Successful when popping FATE repos (#4302)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new d39d304c5d Allow a status of Successful when popping FATE repos (#4302)
d39d304c5d is described below
commit d39d304c5dea88e6316854880e6812532d781792
Author: Christopher L. Shannon
AuthorDate: Mon Feb 26 11:54:28 2024 -0500
Allow a status of Successful when popping FATE repos (#4302)
When FATE cleans up transactions that are successful, it will check the
auto_clean flag and if set will delete the transaction. But if it is
false or not set then it will iterate over all the repos and call pop to
remove them to save space so we need to add SUCCESSFUL to the allowed
statuses for pop, along with FAILED_IN_PROGRESS.
This closes #4300
---
.../accumulo/core/fate/accumulo/AccumuloStore.java| 4 ++--
.../accumulo/test/fate/accumulo/AccumuloStoreIT.java | 19 +--
2 files changed, 11 insertions(+), 12 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/fate/accumulo/AccumuloStore.java
b/core/src/main/java/org/apache/accumulo/core/fate/accumulo/AccumuloStore.java
index 8ae222c61c..0d65f3e5d1 100644
---
a/core/src/main/java/org/apache/accumulo/core/fate/accumulo/AccumuloStore.java
+++
b/core/src/main/java/org/apache/accumulo/core/fate/accumulo/AccumuloStore.java
@@ -361,8 +361,8 @@ public class AccumuloStore extends AbstractFateStore {
verifyReserved(true);
Optional top = findTop();
- top.ifPresent(
- t ->
newMutator(fateId).requireStatus(TStatus.FAILED_IN_PROGRESS).deleteRepo(t).mutate());
+ top.ifPresent(t -> newMutator(fateId)
+ .requireStatus(TStatus.FAILED_IN_PROGRESS,
TStatus.SUCCESSFUL).deleteRepo(t).mutate());
}
@Override
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
index a501526cba..785b52b290 100644
---
a/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/fate/accumulo/AccumuloStoreIT.java
@@ -38,7 +38,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.fate.FateInstanceType;
import org.apache.accumulo.core.fate.FateStore;
-import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore.TStatus;
import org.apache.accumulo.core.fate.accumulo.AccumuloStore;
import org.apache.accumulo.core.fate.accumulo.schema.FateSchema;
import org.apache.accumulo.harness.SharedMiniClusterBase;
@@ -143,8 +143,8 @@ public class AccumuloStoreIT extends SharedMiniClusterBase {
}
private void testOperationWithStatuses(Runnable beforeOperation,
Executable operation,
-Set acceptableStatuses) throws Exception {
- for (ReadOnlyFateStore.TStatus status :
ReadOnlyFateStore.TStatus.values()) {
+Set acceptableStatuses) throws Exception {
+ for (TStatus status : TStatus.values()) {
// Run any needed setup for the operation before each iteration
beforeOperation.run();
@@ -164,7 +164,7 @@ public class AccumuloStoreIT extends SharedMiniClusterBase {
public void push() throws Exception {
testOperationWithStatuses(() -> {}, // No special setup needed for push
() -> txStore.push(new FateIT.TestRepo("testOp")),
- Set.of(ReadOnlyFateStore.TStatus.IN_PROGRESS,
ReadOnlyFateStore.TStatus.NEW));
+ Set.of(TStatus.IN_PROGRESS, TStatus.NEW));
}
@Test
@@ -172,28 +172,27 @@ public class AccumuloStoreIT extends
SharedMiniClusterBase {
testOperationWithStatuses(() -> {
// Setup for pop: Ensure there something to pop by first pushing
try {
- injectStatus(client, tableName, fateId,
ReadOnlyFateStore.TStatus.NEW);
+ injectStatus(client, tableName, fateId, TStatus.NEW);
txStore.push(new FateIT.TestRepo("testOp"));
} catch (Exception e) {
throw new RuntimeException("Failed to setup for pop", e);
}
- }, txStore::pop, Set.of(ReadOnlyFateStore.TStatus.FAILED_IN_PROGRESS));
+ }, txStore::pop, Set.of(TStatus.FAILED_IN_PROGRESS, TStatus.SUCCESSFUL));
}
@Test
public void delete() throws Exception {
testOperationWithStatuses(() -> {}, // No special setup needed for delete
txStore::delete,
- Set.of(ReadOnlyFateStore.TStatus.NEW,
ReadOnlyFateStore.TStatus.SUBMITTED,
- ReadOnlyFateStore.TStatus.SUCCESSFUL,
ReadOnlyFateStore.TStatus.FAILED));
+ Set.of(TStatus.NEW, TStatus.SUBMITTED, TStatus.SUCCESSFUL,
TStatus.FAIL
(accumulo) branch elasticity updated (47b54a6bcf -> 8417794980)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 47b54a6bcf Merge branch 'main' into elasticity add 8417794980 Add support for creating a FATE transaction by key (#4204) No new revisions were added by this update. Summary of changes: .../accumulo/core/fate/AbstractFateStore.java | 172 ++--- .../org/apache/accumulo/core/fate/FateKey.java | 162 .../org/apache/accumulo/core/fate/FateStore.java | 15 ++ .../accumulo/core/fate/ReadOnlyFateStore.java | 7 + .../accumulo/core/fate/WrappedFateTxStore.java | 14 ++ .../org/apache/accumulo/core/fate/ZooStore.java| 124 ++-- .../accumulo/core/fate/accumulo/AccumuloStore.java | 95 - .../accumulo/core/fate/accumulo/FateMutator.java | 3 + .../core/fate/accumulo/FateMutatorImpl.java| 10 +- .../core/fate/accumulo/schema/FateSchema.java | 3 + .../apache/accumulo/core/logging/FateLogger.java | 17 ++ .../org/apache/accumulo/core/fate/TestStore.java | 42 +++- .../java/org/apache/accumulo/test/fate/FateIT.java | 10 +- .../apache/accumulo/test/fate/FateTestRunner.java | 21 +- .../test/fate/accumulo/AccumuloFateIT.java | 6 +- .../test/fate/accumulo/AccumuloStoreFateIT.java| 29 ++- .../accumulo/test/fate/accumulo/FateStoreIT.java | 215 - .../test/fate/zookeeper/ZooStoreFateIT.java| 47 - .../test/fate/zookeeper/ZookeeperFateIT.java | 21 +- 19 files changed, 930 insertions(+), 83 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/fate/FateKey.java
(accumulo) branch elasticity updated (c8a5b11c4f -> 64b93290c9)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from c8a5b11c4f Refactor AccumuloStoreReadWriteIT to also run against ZooStore (#4202) add 1531fe5d7e Fix intermittent failure in AuthenticationTokenSecretManagerTest (#4203) add bce2ef2bfc Merge branch '2.1' new 64b93290c9 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../security/delegation/AuthenticationTokenSecretManagerTest.java | 6 ++ 1 file changed, 6 insertions(+)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 64b93290c977103b8e9962cf154f103c393f8a61 Merge: c8a5b11c4f bce2ef2bfc Author: Christopher L. Shannon AuthorDate: Sat Jan 27 10:38:53 2024 -0500 Merge branch 'main' into elasticity .../security/delegation/AuthenticationTokenSecretManagerTest.java | 6 ++ 1 file changed, 6 insertions(+)
(accumulo) branch main updated (b85cafb61c -> bce2ef2bfc)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b85cafb61c Merge branch '2.1' add 1531fe5d7e Fix intermittent failure in AuthenticationTokenSecretManagerTest (#4203) new bce2ef2bfc Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../security/delegation/AuthenticationTokenSecretManagerTest.java | 6 ++ 1 file changed, 6 insertions(+)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bce2ef2bfc8b3a80dee5cf3e7313fb99b23bb7d1 Merge: b85cafb61c 1531fe5d7e Author: Christopher L. Shannon AuthorDate: Sat Jan 27 10:38:04 2024 -0500 Merge branch '2.1' .../security/delegation/AuthenticationTokenSecretManagerTest.java | 6 ++ 1 file changed, 6 insertions(+) diff --cc server/base/src/test/java/org/apache/accumulo/server/security/delegation/AuthenticationTokenSecretManagerTest.java index 8a7b6ea7bf,00d9e6cf03..7198e16883 --- a/server/base/src/test/java/org/apache/accumulo/server/security/delegation/AuthenticationTokenSecretManagerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/security/delegation/AuthenticationTokenSecretManagerTest.java @@@ -207,9 -207,13 +207,15 @@@ public class AuthenticationTokenSecretM // The passwords line up against multiple calls with the same ID assertArrayEquals(password, secretManager.retrievePassword(id)); + // Sleep 50 ms to make sure we generate another token for the test + // System.currentTimeMillis() is used as part of the token generation and if + // the test runs fast enough it can return the same value that was used + // when generating the first token and the test will fail + Thread.sleep(50); + // Make a second token for the same user +// Briefly sleep to guarantee token is unique, since the token is based on the time +Thread.sleep(100); Entry,AuthenticationTokenIdentifier> pair2 = secretManager.generateToken(principal, cfg); Token token2 = pair2.getKey();
