(accumulo) branch elasticity updated (46d8c0b71d -> d8cb61a73f)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 46d8c0b71d makes behavior of reading absent fate consistent (#4688) add c6e31e2cc2 Add prettyPrint method to ConditionalMutatoin (#4694) new d8cb61a73f Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/data/ConditionalMutation.java| 46 +++ .../core/data/ConditionalMutationTest.java | 92 ++ 2 files changed, 138 insertions(+)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d8cb61a73f5767ba5fd0a22f3c9eb55cf4a73d6d Merge: 46d8c0b71d c6e31e2cc2 Author: Mark Owens AuthorDate: Mon Jun 24 17:33:44 2024 + Merge remote-tracking branch 'upstream/main' into elasticity .../accumulo/core/data/ConditionalMutation.java| 46 +++ .../core/data/ConditionalMutationTest.java | 92 ++ 2 files changed, 138 insertions(+)
(accumulo) branch main updated: Add prettyPrint method to ConditionalMutatoin (#4694)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new c6e31e2cc2 Add prettyPrint method to ConditionalMutatoin (#4694)
c6e31e2cc2 is described below
commit c6e31e2cc252e324c5bd3635a9f7edbbce79
Author: Mark Owens
AuthorDate: Mon Jun 24 13:29:38 2024 -0400
Add prettyPrint method to ConditionalMutatoin (#4694)
Added prettyPrint method to ConditionalMutatiol. The method prints the
usual information that is currently printed when prettyPrint is called on a
mutation, but adds additional condition information for the provided
conditionalMutation.
Closes #4675
---
.../accumulo/core/data/ConditionalMutation.java| 46 +++
.../core/data/ConditionalMutationTest.java | 92 ++
2 files changed, 138 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/data/ConditionalMutation.java
b/core/src/main/java/org/apache/accumulo/core/data/ConditionalMutation.java
index 1fe82e4717..4db8ca8e00 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/ConditionalMutation.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/ConditionalMutation.java
@@ -19,12 +19,14 @@
package org.apache.accumulo.core.data;
import static com.google.common.base.Preconditions.checkArgument;
+import static java.nio.charset.StandardCharsets.UTF_8;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.hadoop.io.Text;
/**
@@ -82,6 +84,50 @@ public class ConditionalMutation extends Mutation {
return Collections.unmodifiableList(conditions);
}
+ private String toString(ByteSequence bs) {
+if (bs == null) {
+ return null;
+}
+return new String(bs.toArray(), UTF_8);
+ }
+
+ @Override
+ public String prettyPrint() {
+StringBuilder sb = new StringBuilder(super.prettyPrint());
+for (Condition c : conditions) {
+ sb.append(" condition: ");
+ sb.append(toString(c.getFamily()));
+ sb.append(":");
+ sb.append(toString(c.getQualifier()));
+ if (c.getValue() != null && !toString(c.getValue()).isBlank()) {
+sb.append(" value: ");
+sb.append(toString(c.getValue()));
+ }
+ if (c.getVisibility() != null && !toString(c.getVisibility()).isBlank())
{
+sb.append(" visibility: '");
+sb.append(toString(c.getVisibility()));
+sb.append("'");
+ }
+ if (c.getTimestamp() != null) {
+sb.append(" timestamp: ");
+sb.append("'");
+sb.append(c.getTimestamp());
+sb.append("'");
+ }
+ if (c.getIterators().length != 0) {
+sb.append(" iterator: ");
+IteratorSetting[] iterators = c.getIterators();
+for (IteratorSetting its : iterators) {
+ sb.append("'");
+ sb.append(its.toString());
+ sb.append("' ");
+}
+ }
+ sb.append("\n");
+}
+return sb.toString();
+ }
+
@Override
public boolean equals(Object o) {
if (o == this) {
diff --git
a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
index 161dfe5c32..a563c6101b 100644
---
a/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/data/ConditionalMutationTest.java
@@ -25,18 +25,29 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.util.HashMap;
import java.util.List;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
public class ConditionalMutationTest {
private static final byte[] ROW = "row".getBytes(UTF_8);
+ private static final String VALUE = "val01";
private static final String FAMILY = "family";
+ private static final String FAMILY2 = "family2";
+ private static final String FAMILY3 = "family3";
private static final String QUALIFIER = "qualifier";
private static final String QUALIFIER2 = "qualifier2";
private static final String QUALIFIER3 = "qualifie
(accumulo) branch elasticity updated (01fa10be63 -> 93aac1a800)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a change to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 01fa10be63 Merge branch 'main' into elasticity
add e00c3c6630 Update PrintInfo to not print unneeded exception messages
(#4634)
add bfae1b7cd9 Merge remote-tracking branch 'upstream/2.1'
add 9ec0087069 Correcting error introduced when merging 2.1 branch
new 93aac1a800 Merge remote-tracking branch 'upstream/main' into elasticity
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../accumulo/core/summary/SummaryReader.java | 11 +-
.../java/org/apache/accumulo/test/PrintInfoIT.java | 202 +
.../resources/org/apache/accumulo/test}/ver_7.rf | Bin
4 files changed, 213 insertions(+), 2 deletions(-)
create mode 100644 test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
copy {core/src/test/resources/org/apache/accumulo/core/file/rfile =>
test/src/main/resources/org/apache/accumulo/test}/ver_7.rf (100%)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 93aac1a800c889a060bc5ce4dad41aecc16145c4 Merge: 01fa10be63 9ec0087069 Author: Mark Owens AuthorDate: Wed Jun 5 15:08:41 2024 + Merge remote-tracking branch 'upstream/main' into elasticity .../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +- .../accumulo/core/summary/SummaryReader.java | 11 +- .../java/org/apache/accumulo/test/PrintInfoIT.java | 202 + .../resources/org/apache/accumulo/test/ver_7.rf| Bin 0 -> 14557 bytes 4 files changed, 213 insertions(+), 2 deletions(-)
(accumulo) branch main updated: Correcting error introduced when merging 2.1 branch
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9ec0087069 Correcting error introduced when merging 2.1 branch
9ec0087069 is described below
commit 9ec008706927533adcd95a8e1ded43735836cdab
Author: Mark Owens
AuthorDate: Wed Jun 5 14:41:56 2024 +
Correcting error introduced when merging 2.1 branch
Correct reference to MetaDataTable name when merging 2.1 branch into
main. Error was introduced in the new PrintInfoIT tests.
---
test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
b/test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
index 459beb9ea1..6544135053 100644
--- a/test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
@@ -43,7 +43,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.rfile.PrintInfo;
-import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
@@ -170,7 +170,7 @@ public class PrintInfoIT extends SharedMiniClusterBase {
boolean foundFile = false;
String rfileName = null;
try (BatchScanner bscanner =
-client.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY,
1)) {
+client.createBatchScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY, 1)) {
String tableId = client.tableOperations().tableIdMap().get(tableName);
bscanner.setRanges(
Collections.singletonList(new Range(new Text(tableId + ";"), new
Text(tableId + "<";
(accumulo) branch main updated (ee197905e9 -> bfae1b7cd9)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from ee197905e9 Merge branch '2.1'
add e00c3c6630 Update PrintInfo to not print unneeded exception messages
(#4634)
new bfae1b7cd9 Merge remote-tracking branch 'upstream/2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../accumulo/core/summary/SummaryReader.java | 11 +-
.../java/org/apache/accumulo/test/PrintInfoIT.java | 202 +
.../resources/org/apache/accumulo/test}/ver_7.rf | Bin
4 files changed, 213 insertions(+), 2 deletions(-)
create mode 100644 test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
copy {core/src/test/resources/org/apache/accumulo/core/file/rfile =>
test/src/main/resources/org/apache/accumulo/test}/ver_7.rf (100%)
(accumulo) 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bfae1b7cd98ab617d8699868ee9e19cbe00b7f95 Merge: ee197905e9 e00c3c6630 Author: Mark Owens AuthorDate: Wed Jun 5 14:12:11 2024 + Merge remote-tracking branch 'upstream/2.1' .../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +- .../accumulo/core/summary/SummaryReader.java | 11 +- .../java/org/apache/accumulo/test/PrintInfoIT.java | 202 + .../resources/org/apache/accumulo/test/ver_7.rf| Bin 0 -> 14557 bytes 4 files changed, 213 insertions(+), 2 deletions(-)
(accumulo) branch 2.1 updated (b0643beb5f -> e00c3c6630)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a change to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from b0643beb5f Modified FileCompactor to set interruptFlag on RFile (#4628)
add e00c3c6630 Update PrintInfo to not print unneeded exception messages
(#4634)
No new revisions were added by this update.
Summary of changes:
.../apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../accumulo/core/summary/SummaryReader.java | 11 +-
.../java/org/apache/accumulo/test/PrintInfoIT.java | 202 +
.../resources/org/apache/accumulo/test}/ver_7.rf | Bin
4 files changed, 213 insertions(+), 2 deletions(-)
create mode 100644 test/src/main/java/org/apache/accumulo/test/PrintInfoIT.java
copy {core/src/test/resources/org/apache/accumulo/core/file/rfile =>
test/src/main/resources/org/apache/accumulo/test}/ver_7.rf (100%)
(accumulo-website) branch main updated: Fix broken link (#418)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new 610539c7 Fix broken link (#418) 610539c7 is described below commit 610539c7f08d7e281207b279283ca120d6e51542 Author: Mark Owens AuthorDate: Mon Feb 26 09:49:02 2024 -0500 Fix broken link (#418) When clicking on the `Contribute` link for the Core Project under the Accumulo Repositories list, the link presents a 404 error. This PR updates the link to point to the proper page. --- pages/how-to-contribute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/how-to-contribute.md b/pages/how-to-contribute.md index 0dbe60b0..547d2c1d 100644 --- a/pages/how-to-contribute.md +++ b/pages/how-to-contribute.md @@ -72,7 +72,7 @@ For more information, see the [contributor guide](/contributors-guide/). [confluence]: https://cwiki.apache.org/confluence/display/ACCUMULO/Apache+Accumulo+Home [contact]: /contact-us/ [a]: https://github.com/apache/accumulo -[ac]: https://github.com/apache/accumulo/blob/main/CONTRIBUTING.md +[ac]: https://github.com/apache/accumulo/blob/main/.github/CONTRIBUTING.md [ai]: https://github.com/apache/accumulo/issues [w]: https://github.com/apache/accumulo-website [wc]: https://github.com/apache/accumulo-website/blob/main/CONTRIBUTING.md
(accumulo) branch elasticity updated: Remove unused argument to setOperationIdOnce (#4294)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f75dbf8c5b Remove unused argument to setOperationIdOnce (#4294)
f75dbf8c5b is described below
commit f75dbf8c5baa21d2d52ce055f3b627b72a0dc96e
Author: Mark Owens
AuthorDate: Mon Feb 26 08:28:50 2024 -0500
Remove unused argument to setOperationIdOnce (#4294)
Remove unused supressError boolean argument from setOperationIdOnce method
in TabletMetadata.
Adjust javadoc accordingly.
---
.../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 693ae04b62..5ad11b5945 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -488,7 +488,7 @@ public class TabletMetadata {
te.flushNonce = OptionalLong.of(Long.parseUnsignedLong(val, 16));
break;
case OPID_QUAL:
- te.setOperationIdOnce(val, suppressLocationError);
+ te.setOperationIdOnce(val);
break;
case SELECTED_QUAL:
te.selectedFiles = SelectedFiles.from(val);
@@ -581,10 +581,9 @@ public class TabletMetadata {
* Sets an operation ID only once.
*
* @param val operation id to set
- * @param suppressError set to true to suppress an exception being thrown,
else false
* @throws IllegalStateException if an operation id or location is already
set
*/
- private void setOperationIdOnce(String val, boolean suppressError) {
+ private void setOperationIdOnce(String val) {
Preconditions.checkState(operationId == null);
operationId = TabletOperationId.from(val);
}
(accumulo) branch elasticity updated: Add IllegalStateException to TabletMetadata class (#4213)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 0e82f76e90 Add IllegalStateException to TabletMetadata class (#4213)
0e82f76e90 is described below
commit 0e82f76e900246eff889c22268945ed961164094
Author: Mark Owens
AuthorDate: Fri Feb 2 05:46:31 2024 -0500
Add IllegalStateException to TabletMetadata class (#4213)
Add an IllegalStateException to the TabletMetadata class in instances where
an unexpected TabletColumnFamily qualifier is used.
---
.../java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 22066b0cff..460a52e22b 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -459,6 +459,8 @@ public class TabletMetadata {
case REQUESTED_QUAL:
te.onDemandHostingRequested = true;
break;
+default:
+ throw new IllegalStateException("Unexpected TabletColumnFamily
qualifier: " + qual);
}
break;
case ServerColumnFamily.STR_NAME:
(accumulo) branch elasticity updated: Remove property tserver.compaction.major.delay (#3950)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 67f3a7cdfa Remove property tserver.compaction.major.delay (#3950)
67f3a7cdfa is described below
commit 67f3a7cdfafad8745d5c808259f5a02aa568e116
Author: Mark Owens
AuthorDate: Thu Nov 16 11:10:20 2023 -0500
Remove property tserver.compaction.major.delay (#3950)
The tserver no longer does compactions, therefore the
tserver.compation.major.delay property is no longer needed. This PR removes the
property and associated references within the code.
Updated Tablet.java and TabletServer.java with changes provided by
@keith-turner.
Refactored `configure` method in LargeSplitRowIT to remove two unnecessary
lines.
---
.../org/apache/accumulo/core/conf/Property.java| 3 --
.../accumulo/minicluster/MiniAccumuloRunner.java | 4 +-
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 3 --
.../org/apache/accumulo/tserver/TabletServer.java | 50 +++---
.../org/apache/accumulo/tserver/tablet/Tablet.java | 5 +--
.../org/apache/accumulo/test/LargeSplitRowIT.java | 3 --
.../BalanceInPresenceOfOfflineTableIT.java | 1 -
.../accumulo/test/functional/BatchScanSplitIT.java | 7 ---
.../accumulo/test/functional/BigRootTabletIT.java | 1 -
.../accumulo/test/functional/BinaryStressIT.java | 17
.../test/functional/BulkSplitOptimizationIT.java | 27 +++-
.../test/functional/ChaoticBalancerIT.java | 1 -
.../accumulo/test/functional/CompactionIT.java | 1 -
.../accumulo/test/functional/ConcurrencyIT.java| 1 -
.../test/functional/DeleteEverythingIT.java| 26 ---
.../test/functional/GarbageCollectorIT.java| 1 -
.../functional/GarbageCollectorTrashDefaultIT.java | 1 -
.../functional/GarbageCollectorTrashEnabledIT.java | 1 -
...ageCollectorTrashEnabledWithCustomPolicyIT.java | 1 -
.../accumulo/test/functional/LargeRowIT.java | 17
.../test/functional/ManyWriteAheadLogsIT.java | 19 +---
.../test/functional/MetadataMaxFilesIT.java| 1 -
.../accumulo/test/functional/MetadataSplitIT.java | 8
.../accumulo/test/functional/RestartStressIT.java | 1 -
.../accumulo/test/functional/RowDeleteIT.java | 1 -
.../test/functional/SimpleBalancerFairnessIT.java | 1 -
.../apache/accumulo/test/functional/SplitIT.java | 20 -
.../accumulo/test/functional/WriteAheadLogIT.java | 1 -
28 files changed, 25 insertions(+), 198 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index a305a6846f..29710d76a8 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -581,9 +581,6 @@ public enum Property {
"2.1.0"),
TSERV_MIGRATE_MAXCONCURRENT("tserver.migrations.concurrent.max", "1",
PropertyType.COUNT,
"The maximum number of concurrent tablet migrations for a tablet
server.", "1.3.5"),
- // ELASTICITY_TODO look into removing this prop, may need to deprecate in 3.0
- TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s",
PropertyType.TIMEDURATION,
- "Time a tablet server will sleep between checking which tablets need
compaction.", "1.3.5"),
TSERV_COMPACTION_SERVICE_PREFIX("tserver.compaction.major.service.", null,
PropertyType.PREFIX,
"Prefix for compaction services.", "2.1.0"),
TSERV_COMPACTION_SERVICE_ROOT_PLANNER("tserver.compaction.major.service.root.planner",
diff --git
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
index 31cd0898f9..f4a7bf24c0 100644
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
@@ -105,9 +105,9 @@ public class MiniAccumuloRunner {
System.out.println();
System.out.println("# Configuration normally placed in accumulo.properties
can be added using"
+ " a site.* prefix.");
-System.out.println("# For example the following line will set
tserver.compaction.major.delay");
+System.out.println("# For example the following line will set
tserver.compaction.warn.time");
System.out.println();
-System.out.println("#site.tserver.compaction.major.delay=60s");
+System.out.println("#site.tserver.compaction.warn.time=10m");
}
diff --git
a/minicluster/src/main/j
[accumulo] branch elasticity updated: Fix bug in testGetHostingGoalCommand test (#3848)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 82bcc5d4b3 Fix bug in testGetHostingGoalCommand test (#3848)
82bcc5d4b3 is described below
commit 82bcc5d4b37dea332cafd203f956e3f6b59bb397
Author: Mark Owens
AuthorDate: Mon Oct 16 13:25:58 2023 -0400
Fix bug in testGetHostingGoalCommand test (#3848)
A previous commit had corrected several tests in ShellServerIT where the
tests would fail if the tableId was a value other than 1 or 2. The
testGetHostingGoalCommand test failed to get that fix. This commit updates that
IT tests to handle tableId's of any value.
---
.../apache/accumulo/test/shell/ShellServerIT.java | 32 +++---
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index ae640bd7cc..e318fbf9b7 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@ -1281,29 +1281,29 @@ public class ShellServerIT extends
SharedMiniClusterBase {
result = ts.exec("gethostinggoal");
assertTrue(result.contains("TABLE: " + tableName));
assertTrue(result.contains("TABLET IDHOSTING GOAL"));
- assertTrue(result.contains(tableId + ";d< ONDEMAND"));
- assertTrue(result.contains(tableId + ";m;dONDEMAND"));
- assertTrue(result.contains(tableId + ";s;mONDEMAND"));
- assertTrue(result.contains(tableId + "<;s ONDEMAND"));
+ assertTrue(result.matches("(?s).*" + tableId + ";d<\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";m;d\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";s;m\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + "<;s\\s+ONDEMAND.*"));
ts.exec("sethostinggoal -g ALWAYS -r p");
result = ts.exec("gethostinggoal -r p");
- assertFalse(result.contains(tableId + ";d< ONDEMAND"));
- assertFalse(result.contains(tableId + ";m;dONDEMAND"));
- assertTrue(result.contains(tableId + ";s;mALWAYS"));
- assertFalse(result.contains(tableId + "<;s ONDEMAND"));
+ assertFalse(result.matches("(?s).*" + tableId + ";d<\\s+ONDEMAND.*"));
+ assertFalse(result.matches("(?s).*" + tableId + ";m;d\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";s;m\\s+ALWAYS.*"));
+ assertFalse(result.matches("(?s).*" + tableId + "<;s\\s+ONDEMAND.*"));
result = ts.exec("gethostinggoal");
- assertTrue(result.contains(tableId + ";d< ONDEMAND"));
- assertTrue(result.contains(tableId + ";m;dONDEMAND"));
- assertTrue(result.contains(tableId + ";s;mALWAYS"));
- assertTrue(result.contains(tableId + "<;s ONDEMAND"));
+ assertTrue(result.matches("(?s).*" + tableId + ";d<\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";m;d\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";s;m\\s+ALWAYS.*"));
+ assertTrue(result.matches("(?s).*" + tableId + "<;s\\s+ONDEMAND.*"));
result = ts.exec("gethostinggoal -b f -e p");
- assertFalse(result.contains(tableId + ";d< ONDEMAND"));
- assertTrue(result.contains(tableId + ";m;dONDEMAND"));
- assertTrue(result.contains(tableId + ";s;mALWAYS"));
- assertFalse(result.contains(tableId + "<;s ONDEMAND"));
+ assertFalse(result.matches("(?s).*" + tableId + ";d<\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";m;d\\s+ONDEMAND.*"));
+ assertTrue(result.matches("(?s).*" + tableId + ";s;m\\s+ALWAYS.*"));
+ assertFalse(result.matches("(?s).*" + tableId + "<;s\\s+ONDEMAND.*"));
} finally {
if (splitsFilePath != null) {
[accumulo] branch elasticity updated: Rename setgoal/getgoal to sethostinggoal/gethostinggoal (#3842)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c3358c0d83 Rename setgoal/getgoal to sethostinggoal/gethostinggoal
(#3842)
c3358c0d83 is described below
commit c3358c0d8398319f29ff2cbd7b643e324f26e1b7
Author: Mark Owens
AuthorDate: Fri Oct 13 07:54:33 2023 -0400
Rename setgoal/getgoal to sethostinggoal/gethostinggoal (#3842)
Rename the setgoal and getgoal shell commands to sethostinggoal and
gethostinggoal, respectively. This change is being made to provide more clarity
as to what the commands are doing and make them less generic.
Several other instances if 'goal' were changed to 'hostinggoal' in order to
add more clarity.
Closes #3800
---
.../commands/GetTabletHostingGoalCommand.java | 2 +-
.../commands/SetTabletHostingGoalCommand.java | 12 +++---
.../apache/accumulo/test/shell/ShellServerIT.java | 46 +++---
3 files changed, 31 insertions(+), 29 deletions(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
index 39033a7c12..1588e2924c 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetTabletHostingGoalCommand.java
@@ -34,7 +34,7 @@ public class GetTabletHostingGoalCommand extends
TableOperation {
@Override
public String getName() {
-return "getgoal";
+return "gethostinggoal";
}
@Override
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/SetTabletHostingGoalCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/SetTabletHostingGoalCommand.java
index 414d3f60a7..e68d2ab248 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/SetTabletHostingGoalCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/SetTabletHostingGoalCommand.java
@@ -39,11 +39,11 @@ public class SetTabletHostingGoalCommand extends
TableOperation {
private Option goalOpt;
private Range range;
- private TabletHostingGoal goal;
+ private TabletHostingGoal hostingGoal;
@Override
public String getName() {
-return "setgoal";
+return "sethostinggoal";
}
@Override
@@ -54,8 +54,10 @@ public class SetTabletHostingGoalCommand extends
TableOperation {
@Override
protected void doTableOp(final Shell shellState, final String tableName)
throws AccumuloException, AccumuloSecurityException,
TableNotFoundException, IOException {
-
shellState.getAccumuloClient().tableOperations().setTabletHostingGoal(tableName,
range, goal);
-Shell.log.debug("Set goal state: {} on table: {}, range: {}", goal,
tableName, range);
+
shellState.getAccumuloClient().tableOperations().setTabletHostingGoal(tableName,
range,
+hostingGoal);
+Shell.log.debug("Set hosting goal state: {} on table: {}, range: {}",
hostingGoal, tableName,
+range);
}
@Override
@@ -80,7 +82,7 @@ public class SetTabletHostingGoalCommand extends
TableOperation {
this.range = new Range(startRow, startInclusive, endRow, endInclusive);
}
-this.goal =
TabletHostingGoal.valueOf(cl.getOptionValue(goalOpt).toUpperCase());
+this.hostingGoal =
TabletHostingGoal.valueOf(cl.getOptionValue(goalOpt).toUpperCase());
return super.execute(fullCommand, cl, shellState);
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index 93710c8023..3efa258654 100644
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@ -1233,14 +1233,14 @@ public class ShellServerIT extends
SharedMiniClusterBase {
}
@Test
- public void testSetGoalCommand() throws Exception {
+ public void testSetHostingGoalCommand() throws Exception {
final String table = getUniqueNames(1)[0];
ts.exec("createtable " + table);
ts.exec("addsplits -t " + table + " a c e g");
-String result = ts.exec("setgoal -?");
+String result = ts.exec("sethostinggoal -?");
assertTrue(result.contains("Sets the hosting goal"));
-ts.exec("setgoal -t " + table + " -b a -e a -g never");
-ts.exec("setgoal -t " + table + " -b c -e e -ee -g always");
+ts.exec("sethostinggoal -t " + table + " -b a -e a -g never");
+ts.exec("sethostinggoal -t " + table + " -b c -e e
[accumulo] branch elasticity updated: Adjust shell to indicate how re-authentication occurs
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 064948ef2f Adjust shell to indicate how re-authentication occurs
064948ef2f is described below
commit 064948ef2f9fd92419150516372e0ea2ca427766
Author: Mark Owens
AuthorDate: Mon Aug 28 12:51:49 2023 -0400
Adjust shell to indicate how re-authentication occurs
This PR adds presents additional information to the shell when the
re-authentication is automatically performed using the value from
accumulo-client.properties.
It additionally corrects a situation where an endless loop can be
triggered when attempting re-authentication if the
accumulo-client.properties file password is altered after initial setup.
---
shell/src/main/java/org/apache/accumulo/shell/Shell.java | 16 +---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
index 44197ad18e..0a9fac2f64 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@ -234,6 +234,8 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
private long authTimeout;
private long lastUserActivity = System.nanoTime();
private boolean logErrorsToConsole = false;
+ private boolean askAgain = false;
+ private boolean usedClientProps = false;
static {
// set the JLine output encoding to some reasonable default if it isn't
already set
@@ -251,7 +253,7 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
}
}
- // no arg constructor should do minimal work since its used in Main
ServiceLoader
+ // no arg constructor should do minimal work since it's used in Main
ServiceLoader
public Shell() {}
public Shell(LineReader reader) {
@@ -272,8 +274,10 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
&& clientProperties.containsKey(ClientProperty.AUTH_TOKEN.getKey())
&&
principal.equals(ClientProperty.AUTH_PRINCIPAL.getValue(clientProperties))) {
token = ClientProperty.getAuthenticationToken(clientProperties);
+ usedClientProps = true;
}
-if (token == null) {
+if (token == null || askAgain) {
+ usedClientProps = false;
// Read password if the user explicitly asked for it, or didn't specify
anything at all
if (PasswordConverter.STDIN.equals(authenticationString) ||
authenticationString == null) {
authenticationString = reader.readLine(passwordPrompt, '*');
@@ -584,7 +588,7 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
writer.println();
String partialLine = uie.getPartialLine();
-if (partialLine == null || "".equals(uie.getPartialLine().trim())) {
+if (partialLine == null || partialLine.trim().isEmpty()) {
// No content, actually exit
return exitCode;
}
@@ -726,6 +730,12 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
if (authFailed) {
writer.print("Invalid password. ");
+ askAgain = true;
+} else {
+ if (usedClientProps) {
+writer.println(
+"User re-authenticated using value from
accumulo-client.properties file");
+ }
}
} while (authFailed);
lastUserActivity = System.nanoTime();
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 59897142d4ba39e563f1a9ad14ef512efb842abe Merge: f88500dfd0 fc793ec9a1 Author: Mark Owens AuthorDate: Mon Aug 28 10:04:22 2023 -0400 Merge branch '2.1' shell/src/main/java/org/apache/accumulo/shell/Shell.java | 16 +--- 1 file changed, 13 insertions(+), 3 deletions(-)
[accumulo] branch main updated (f88500dfd0 -> 59897142d4)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from f88500dfd0 Merge remote-tracking branch 'upstream/2.1' add fc793ec9a1 Adjust shell to indicate how re-authtication occurs (#3726) new 59897142d4 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: shell/src/main/java/org/apache/accumulo/shell/Shell.java | 16 +--- 1 file changed, 13 insertions(+), 3 deletions(-)
[accumulo] branch 2.1 updated: Adjust shell to indicate how re-authtication occurs (#3726)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new fc793ec9a1 Adjust shell to indicate how re-authtication occurs (#3726)
fc793ec9a1 is described below
commit fc793ec9a141a43efe1dead2fcdbb620efc552ae
Author: Mark Owens
AuthorDate: Mon Aug 28 09:41:30 2023 -0400
Adjust shell to indicate how re-authtication occurs (#3726)
This PR adds presents additional information to the shell when the
re-authentication is automatically performed using the value from
accumulo-client.properties.
Once merged, the update will be applied to 3.x where it corrects a
situation where an endless loop can be triggered when attempting
re-authentication if the accumulo-client.properties file password is altered
after initial setup.
See #3485 for more detail.
Closes #
---
shell/src/main/java/org/apache/accumulo/shell/Shell.java | 16 +---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
index 3923346d40..615b2cc4c8 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/Shell.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/Shell.java
@@ -233,6 +233,8 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
private long authTimeout;
private long lastUserActivity = System.nanoTime();
private boolean logErrorsToConsole = false;
+ private boolean askAgain = false;
+ private boolean usedClientProps = false;
static {
// set the JLine output encoding to some reasonable default if it isn't
already set
@@ -250,7 +252,7 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
}
}
- // no arg constructor should do minimal work since its used in Main
ServiceLoader
+ // no arg constructor should do minimal work since it's used in Main
ServiceLoader
public Shell() {}
public Shell(LineReader reader) {
@@ -272,8 +274,10 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
&& clientProperties.containsKey(ClientProperty.AUTH_TOKEN.getKey())
&&
principal.equals(ClientProperty.AUTH_PRINCIPAL.getValue(clientProperties))) {
token = ClientProperty.getAuthenticationToken(clientProperties);
+ usedClientProps = true;
}
-if (token == null) {
+if (token == null || askAgain) {
+ usedClientProps = false;
// Read password if the user explicitly asked for it, or didn't specify
anything at all
if (PasswordConverter.STDIN.equals(authenticationString) ||
authenticationString == null) {
authenticationString = reader.readLine(passwordPrompt, '*');
@@ -618,7 +622,7 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
writer.println();
String partialLine = uie.getPartialLine();
-if (partialLine == null || "".equals(uie.getPartialLine().trim())) {
+if (partialLine == null || partialLine.trim().isEmpty()) {
// No content, actually exit
return exitCode;
}
@@ -760,6 +764,12 @@ public class Shell extends ShellOptions implements
KeywordExecutable {
if (authFailed) {
writer.print("Invalid password. ");
+ askAgain = true;
+} else {
+ if (usedClientProps) {
+writer.println(
+"User re-authenticated using value from
accumulo-client.properties file");
+ }
}
} while (authFailed);
lastUserActivity = System.nanoTime();
[accumulo] branch elasticity updated: Remove getTabletHostingGoal method from TableOperations (#3681)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new b1aa1f58ba Remove getTabletHostingGoal method from TableOperations
(#3681)
b1aa1f58ba is described below
commit b1aa1f58ba40403ce516f50eadcbe94f16d25306
Author: Mark Owens
AuthorDate: Fri Aug 11 10:18:06 2023 -0400
Remove getTabletHostingGoal method from TableOperations (#3681)
Remove the `getTabletHostingGoal` method from `TableOperations` as the
information can now be retrieved with the more robust `getTabletInformation`
method.
The shell command, `getgoals`, still works as before, but the
`GetTabletHostingGoalCommand` was updated to use the new
`getTabletInformation` method to retrieve the necessary information rather
than the removed `getTabletHostingGoal` method.
---
.../core/client/admin/TableOperations.java | 16 ++
.../core/clientImpl/TableOperationsImpl.java | 25 --
.../commands/GetTabletHostingGoalCommand.java | 11 +++---
3 files changed, 5 insertions(+), 47 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index b30aace300..a5fd10a9df 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -1023,20 +1023,8 @@ public interface TableOperations {
}
/**
- * Retrieve the hosting goal for a range of tablets in the specified table.
- *
- * @param tableName table name
- * @param range tablet range
- * @since 4.0.0
- */
- default Stream getTabletHostingGoal(final String
tableName,
- final Range range) throws TableNotFoundException {
-throw new UnsupportedOperationException();
- }
-
- /**
- * @return a stream of tablets information for tablets that fall in the
specified range. The
- * stream may be backed by a scanner, so its best to close the
stream.
+ * @return a stream of tablet information for tablets that fall in the
specified range. The stream
+ * may be backed by a scanner, so it's best to close the stream.
* @since 4.0.0
*/
default Stream getTabletInformation(final String
tableName, final Range range)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 2fa8433d1f..6c88dba09e 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -94,7 +94,6 @@ import
org.apache.accumulo.core.client.admin.CloneConfiguration;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.DiskUsage;
import org.apache.accumulo.core.client.admin.FindMax;
-import org.apache.accumulo.core.client.admin.HostingGoalForTablet;
import org.apache.accumulo.core.client.admin.ImportConfiguration;
import org.apache.accumulo.core.client.admin.Locations;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
@@ -2157,30 +2156,6 @@ public class TableOperationsImpl extends
TableOperationsHelper {
}
}
- @Override
- public Stream getTabletHostingGoal(final String
tableName,
- final Range range) throws TableNotFoundException {
-EXISTING_TABLE_NAME.validate(tableName);
-
-final Text scanRangeStart = (range.getStartKey() == null) ? null :
range.getStartKey().getRow();
-TableId tableId = context.getTableId(tableName);
-
-TabletsMetadata tabletsMetadata =
-
context.getAmple().readTablets().forTable(tableId).overlapping(scanRangeStart,
true, null)
-.fetch(HOSTING_GOAL, PREV_ROW).checkConsistency().build();
-
-return tabletsMetadata.stream().peek(tm -> {
- if (scanRangeStart != null && tm.getEndRow() != null
- && tm.getEndRow().compareTo(scanRangeStart) < 0) {
-log.debug("tablet {} is before scan start range: {}", tm.getExtent(),
scanRangeStart);
-throw new RuntimeException("Bug in ample or this code.");
- }
-}).takeWhile(tm -> tm.getPrevEndRow() == null
-|| !range.afterEndKey(new
Key(tm.getPrevEndRow()).followingKey(PartialKey.ROW)))
-.map(tm -> new HostingGoalForTablet(new TabletIdImpl(tm.getExtent()),
tm.getHostingGoal()))
-.onClose(tabletsMetadata::close);
- }
-
@Override
public Stream getTabletInformation(final String
tableName, final Range range)
throws TableNotFoundException {
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/Get
[accumulo] branch elasticity updated: Create getTabletInformation API method (#3645)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 09828fe04a Create getTabletInformation API method (#3645)
09828fe04a is described below
commit 09828fe04ad0ceffc5280a2e67bcdd4a56f08383
Author: Mark Owens
AuthorDate: Thu Aug 10 11:59:59 2023 -0400
Create getTabletInformation API method (#3645)
This PR creates a new public API method, getTabletInformation, which
returns a stream of TabletInformation objects containing information for each
tablet within a provided table. The method can be accessed from the code or
JShell via the TableOperations object.
A new TabletInformation class is used to hold information for an individual
tablet.
The 'listtablets' command has been modified to use this new method rather
than using non-public API. Additionally, the listtablets command now includes
the TabletHostingGoal of each tablet in its output.
The ListTabletsCommandTest was updated to reflect the new method. A new IT
test has been added to ShellServerIT which tests 'listtablets' and effectively
tests the new API since the method is used to obtain the information for
'listtablets'.
Closes #3503
Co-authored-by: Keith Turner
---
.../core/client/admin/TableOperations.java | 10 +
.../core/client/admin/TabletInformation.java | 76 +++
.../core/clientImpl/TableOperationsImpl.java | 53 -
.../core/clientImpl/TabletInformationImpl.java | 102 +
.../shell/commands/ListTabletsCommand.java | 253 -
.../shell/commands/ListTabletsCommandTest.java | 211 -
.../apache/accumulo/test/TableOperationsIT.java| 100 +++-
.../apache/accumulo/test/shell/ShellServerIT.java | 79 +++
8 files changed, 494 insertions(+), 390 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 6b05f527e2..b30aace300 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -1034,4 +1034,14 @@ public interface TableOperations {
throw new UnsupportedOperationException();
}
+ /**
+ * @return a stream of tablets information for tablets that fall in the
specified range. The
+ * stream may be backed by a scanner, so its best to close the
stream.
+ * @since 4.0.0
+ */
+ default Stream getTabletInformation(final String
tableName, final Range range)
+ throws TableNotFoundException {
+throw new UnsupportedOperationException();
+ }
+
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/TabletInformation.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TabletInformation.java
new file mode 100644
index 00..b303dadfcf
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/TabletInformation.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.Optional;
+
+import org.apache.accumulo.core.data.TabletId;
+
+/**
+ * @since 4.0.0
+ */
+public interface TabletInformation {
+
+ /**
+ * @return the TabletId for this tablet.
+ */
+ TabletId getTabletId();
+
+ /**
+ * @return the number of files in the tablet directory.
+ */
+ int getNumFiles();
+
+ /**
+ * @return the number of write-ahead logs associated with the tablet.
+ */
+ int getNumWalLogs();
+
+ /**
+ * @return an estimated number of entries in the tablet.
+ */
+ long getEstimatedEntries();
+
+ /**
+ * @return an estimated size of the tablet data on disk, which is likely the
compressed size of
+ * the data.
+ */
+ long getEstimatedSize();
+
+ /**
+ * @return the tablet hosting state.
+ */
+ String getTabletState();
+
+ /**
+
[accumulo] branch elasticity updated: Verify hostingGoals in table with mixed goals (#3532)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new de48b78544 Verify hostingGoals in table with mixed goals (#3532)
de48b78544 is described below
commit de48b785447be37622d1c9bd4cd32e89d6177c5f
Author: Mark Owens
AuthorDate: Fri Jun 23 13:48:48 2023 -0400
Verify hostingGoals in table with mixed goals (#3532)
* Verify hostingGoals in table with mixed goals
This PR adds a suggested additional test for the setting of
`TabletHostingGoals` in `TableOperationsImpl.java`.
Specifically, the new tests verifies that when a table that already
contains mixed `TabletHostingGoals` splits further, the new splits retain the
hostingGoal of the tablet from which they were split. The new test,
`testGetHostingGoals_StaggeredSplits`, does the following:
- creates table
- add two splits; leave first tablet to default ONDEMAND. Set second to
NEVER and third to ALWAYS
- add an additional split point within each of the existing three tablets
- verify newly created tablets are set with the hostingGoal of the tablet
from which they were split
---
.../apache/accumulo/test/TableOperationsIT.java| 63 ++
1 file changed, 63 insertions(+)
diff --git a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
index 94d41f82b4..caef8145e7 100644
--- a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
@@ -617,6 +617,69 @@ public class TableOperationsIT extends
AccumuloClusterHarness {
}
}
+ // This test checks that tablets are correct when staggered splits are added
to a table, i.e.,
+ // a table is split and assigned differing goals. Later when the two
existing tablets are
+ // split, verify that the new splits contain the appropriate hosting goal of
the tablet from
+ // which they wre split. Steps are as follows:
+ // - create table
+ // - add two splits; leave first tablet to default ONDEMAND, seconds to
NEVER, third to ALWAYS
+ // - add an additional split within each of the three existing tablets
+ // - verify the newly created tablets are set with the hostingGoals of the
tablet from which they
+ // are split.
+ @Test
+ public void testGetHostingGoals_StaggeredSplits() throws AccumuloException,
TableExistsException,
+ AccumuloSecurityException, TableNotFoundException {
+
+String tableName = getUniqueNames(1)[0];
+
+try {
+ accumuloClient.tableOperations().create(tableName);
+ String tableId =
accumuloClient.tableOperations().tableIdMap().get(tableName);
+
+ // add split 'h' and 'q'. Leave first as ONDEMAND, set second to NEVER,
and third to ALWAYS
+ SortedSet splits = Sets.newTreeSet(Arrays.asList(new Text("h"),
new Text("q")));
+ accumuloClient.tableOperations().addSplits(tableName, splits);
+ Range range = new Range(new Text("h"), false, new Text("q"), true);
+ accumuloClient.tableOperations().setTabletHostingGoal(tableName, range,
+ TabletHostingGoal.NEVER);
+ range = new Range(new Text("q"), false, null, true);
+ accumuloClient.tableOperations().setTabletHostingGoal(tableName, range,
+ TabletHostingGoal.ALWAYS);
+
+ // verify
+ List expectedGoals = new ArrayList<>();
+ setExpectedGoal(expectedGoals, tableId, "h", null,
TabletHostingGoal.ONDEMAND);
+ setExpectedGoal(expectedGoals, tableId, "q", "h",
TabletHostingGoal.NEVER);
+ setExpectedGoal(expectedGoals, tableId, null, "q",
TabletHostingGoal.ALWAYS);
+
+ List hostingInfo = accumuloClient.tableOperations()
+ .getTabletHostingGoal(tableName, new
Range()).collect(Collectors.toList());
+
+ assertEquals(expectedGoals, hostingInfo);
+
+ // Add a split within each of the existing tablets. Adding 'd', 'm', and
'v'
+ splits = Sets.newTreeSet(Arrays.asList(new Text("d"), new Text("m"), new
Text("v")));
+ accumuloClient.tableOperations().addSplits(tableName, splits);
+
+ // verify results
+ expectedGoals.clear();
+ hostingInfo.clear();
+ setExpectedGoal(expectedGoals, tableId, "d", null,
TabletHostingGoal.ONDEMAND);
+ setExpectedGoal(expectedGoals, tableId, "h", "d",
TabletHostingGoal.ONDEMAND);
+ setExpectedGoal(expectedGoals, tableId, "m", "h",
TabletHostingGoal.NEVER);
+ setExpectedGoal(expectedGoals, tableId, "q", "m",
TabletHostingGoal.NEVER);
+ setExpectedGoal(
[accumulo] branch elasticity updated: Add API method for getting tablet hosting goals (#3451)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 18e9a3453b Add API method for getting tablet hosting goals (#3451)
18e9a3453b is described below
commit 18e9a3453bf3a16bc81f68edb1839206045ddf71
Author: Mark Owens
AuthorDate: Mon Jun 12 13:06:42 2023 -0400
Add API method for getting tablet hosting goals (#3451)
* Add API method for getting tablet hosting goals
Add an API method for obtaining hosting goals for a tablet. This PR creates
the new API method and also the corresponding shell command.
In order to simplify setting and retrieving hosting goals the previously
created shell command, 'goal', which sets hosting goals was renamed to
'setgoal'. The shell command to retrieve hosting goals is named 'getgoal'.
Change summary:
* A class, HostingGoalForTablet, was created to hold the hosting goal
informtion from a tablet query.
* TableOperations class was updated to contain the 'getTabletHostingGoal'
API method.
* TabletOperationsImpl class was updated to contain the API
implementation details.
* A simple typo was corrected in the KeyExtent class.
* Shell class modifications include updating the TableCommands section
with the new GetTabletHostingGoalsCommand and renaming the
TabletHostingGoalCommand to SetTabletHostingGoalCommand.
* Corrected a minor issue in the renamed SetTabletHostingGoalCommand usage
statement. The statement was incorrectly using 'range' rather than 'row' in
its output.
* Created a new class, GetTabletHostingGoalCommand.
* Wrote integration tests for both the API and Shell commands.
* Added @since tag to class
Co-authored-by: Keith Turner
Closes #3302
---
.../core/client/admin/HostingGoalForTablet.java| 65 +
.../core/client/admin/TableOperations.java | 14 +
.../core/clientImpl/TableOperationsImpl.java | 28 ++
.../apache/accumulo/core/dataImpl/KeyExtent.java | 4 +-
.../main/java/org/apache/accumulo/shell/Shell.java | 10 +-
...mmand.java => GetTabletHostingGoalCommand.java} | 48 ++--
...mmand.java => SetTabletHostingGoalCommand.java} | 14 +-
.../apache/accumulo/test/TableOperationsIT.java| 308 +
.../apache/accumulo/test/shell/ShellServerIT.java | 131 -
9 files changed, 576 insertions(+), 46 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/HostingGoalForTablet.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/HostingGoalForTablet.java
new file mode 100644
index 00..10718ff01f
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/HostingGoalForTablet.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import java.util.Objects;
+
+import org.apache.accumulo.core.data.TabletId;
+
+/**
+ * This class contains information that defines the tablet hosting data for a
table. The class
+ * contains the TabletId and associated goal for each tablet in a table or a
subset of tablets if a
+ * range is provided.
+ *
+ * @since 4.0.0
+ */
+public class HostingGoalForTablet {
+ private final TabletId tabletId;
+ private final TabletHostingGoal hostingGoal;
+
+ public HostingGoalForTablet(TabletId tabletId, TabletHostingGoal
hostingGoal) {
+this.tabletId = tabletId;
+this.hostingGoal = hostingGoal;
+ }
+
+ public TabletHostingGoal getHostingGoal() {
+return hostingGoal;
+ }
+
+ public TabletId getTabletId() {
+return tabletId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+if (this == o) {
+ return true;
+}
+if (o == null || getClass() != o.getClass()) {
+ return false;
+}
+HostingGoalForTablet that = (HostingGoalForTablet) o;
+return Objects.equals(tabletId, that.tabletId) && hostingGoal ==
that.hostingGoal;
+ }
+
+ @Ove
[accumulo] branch elasticity updated: Allow setting of initial hosting goal with NewTableConfiguration (#3379)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 6559f8c694 Allow setting of initial hosting goal with
NewTableConfiguration (#3379)
6559f8c694 is described below
commit 6559f8c6940904907bdb60758ac54b4faa4b4bdf
Author: Mark Owens
AuthorDate: Fri May 5 11:35:18 2023 -0400
Allow setting of initial hosting goal with NewTableConfiguration (#3379)
Modified NewTableConfiguration to support users setting initial hosting
goals for tables at time of table creation.
Updates allow for setting goals via API and via the Accumulo shell.
CreateTable fate operation was updated to set initial goal upon creation of
tables. If splits are set at table creation, each tablet will have the
appropriate hosting goal set for the tablet.
Unit and IT tests were created to verify the changes.
---
.../core/client/admin/NewTableConfiguration.java | 10 +++
.../core/clientImpl/TableOperationsImpl.java | 2 +
.../client/admin/NewTableConfigurationTest.java| 19 +
.../accumulo/manager/FateServiceHandler.java | 11 ++-
.../accumulo/manager/tableOps/TableInfo.java | 11 +++
.../manager/tableOps/create/CreateTable.java | 6 +-
.../manager/tableOps/create/PopulateMetadata.java | 21 ++
.../manager/tableOps/goal/SetHostingGoal.java | 5 +-
.../shell/commands/CreateTableCommand.java | 24 ++
.../accumulo/test/NewTableConfigurationIT.java | 87 ++
.../accumulo/test/shell/ShellCreateTableIT.java| 60 +++
11 files changed, 248 insertions(+), 8 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
index fc6ba06c3f..f4412d42f1 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/NewTableConfiguration.java
@@ -72,6 +72,7 @@ public class NewTableConfiguration {
private Map localityProps = Collections.emptyMap();
private final Map iteratorProps = new HashMap<>();
private SortedSet splitProps = Collections.emptySortedSet();
+ private TabletHostingGoal initialHostingGoal = TabletHostingGoal.ONDEMAND;
private void checkDisjoint(Map props, Map
derivedProps,
String kind) {
@@ -309,6 +310,15 @@ public class NewTableConfiguration {
return this;
}
+ public NewTableConfiguration withInitialHostingGoal(final TabletHostingGoal
goal) {
+this.initialHostingGoal = goal;
+return this;
+ }
+
+ public TabletHostingGoal getInitialHostingGoal() {
+return this.initialHostingGoal;
+ }
+
/**
* Verify the provided properties are valid table properties.
*/
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index a813325020..1338510053 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -246,6 +246,8 @@ public class TableOperationsImpl extends
TableOperationsHelper {
args.add(ByteBuffer.wrap(ntc.getTimeType().name().getBytes(UTF_8)));
// Send info relating to initial table creation i.e, create online or
offline
args.add(ByteBuffer.wrap(ntc.getInitialTableState().name().getBytes(UTF_8)));
+// send initialHostingGoal information
+
args.add(ByteBuffer.wrap(ntc.getInitialHostingGoal().name().getBytes(UTF_8)));
// Check for possible initial splits to be added at table creation
// Always send number of initial splits to be created, even if zero. If
greater than zero,
// add the splits to the argument List which will be used by the FATE
operations.
diff --git
a/core/src/test/java/org/apache/accumulo/core/client/admin/NewTableConfigurationTest.java
b/core/src/test/java/org/apache/accumulo/core/client/admin/NewTableConfigurationTest.java
index 89930fb37a..15695f75c2 100644
---
a/core/src/test/java/org/apache/accumulo/core/client/admin/NewTableConfigurationTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/client/admin/NewTableConfigurationTest.java
@@ -103,6 +103,25 @@ public class NewTableConfigurationTest {
}
+ @Test
+ public void testWithAndGetInitialHostingGoals() {
+NewTableConfiguration ntc = new NewTableConfiguration();
+TabletHostingGoal initialHostingGoal = ntc.getInitialHostingGoal();
+assertEquals(TabletHostingGoal.ONDEMAND, initialHostingGoal);
+
+ntc = new
NewTableConfiguration().withInitialHostingGoal(TabletHostingGoal.ONDEMAND);
+initialHosti
[accumulo-examples] branch main updated: Trivial documentation cleanup
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new b69906c Trivial documentation cleanup b69906c is described below commit b69906c9950d9f81f23399dddc9d5c95ad856f9b Author: Mark Owens AuthorDate: Fri Mar 10 10:32:14 2023 -0500 Trivial documentation cleanup Removing extraneous '>' characters from example output. --- docs/uniquecols.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/uniquecols.md b/docs/uniquecols.md index 99ad701..7441fa2 100644 --- a/docs/uniquecols.md +++ b/docs/uniquecols.md @@ -27,9 +27,9 @@ qualifiers. $ /path/to/accumulo shell -u username -p secret username@instance> createnamespace examples username@instance> createtable examples.unique -username@instance> examples.unique> insert row1 fam1 qual1 v1 -username@instance> examples.unique> insert row2 fam1 qual1 v2 -username@instance> examples.unique> insert row3 fam1 qual1 v3 +username@instance examples.unique> insert row1 fam1 qual1 v1 +username@instance examples.unique> insert row2 fam1 qual1 v2 +username@instance examples.unique> insert row3 fam1 qual1 v3 ``` Exit the Accumulo shell and run the uniqueColumns mapReduce job against
[accumulo-examples] branch 2.1 updated: Trivial documentation cleanup
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new 9d400cd Trivial documentation cleanup 9d400cd is described below commit 9d400cd71a6e3d81896c942469694bcb2ab954e0 Author: Mark Owens AuthorDate: Fri Mar 10 10:28:24 2023 -0500 Trivial documentation cleanup Removing extraneous '>' characters from example output. --- docs/uniquecols.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/uniquecols.md b/docs/uniquecols.md index 714fcb2..3be2ab6 100644 --- a/docs/uniquecols.md +++ b/docs/uniquecols.md @@ -27,9 +27,9 @@ qualifiers. $ /path/to/accumulo shell -u username -p secret username@instance> createnamespace examples username@instance> createtable examples.unique -username@instance> examples.unique> insert row1 fam1 qual1 v1 -username@instance> examples.unique> insert row2 fam1 qual1 v2 -username@instance> examples.unique> insert row3 fam1 qual1 v3 +username@instance examples.unique> insert row1 fam1 qual1 v1 +username@instance examples.unique> insert row2 fam1 qual1 v2 +username@instance examples.unique> insert row3 fam1 qual1 v3 ``` Exit the Accumulo shell and run the uniqueColumns mapReduce job against
[accumulo-examples] branch main updated (97b9df3 -> 04317dc)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git from 97b9df3 Create user documentation for the UniqueColumns example. (#114) add 7c0576b Update POM to target 2.1.1-SNAPSHOT (#105) add 9fda851 Update POM version info for 2.1 examples branch add ae45b3d Add documentation for uniquecols example (#115) add 3b8088a Update SampleExample and documentation (#116) add 1a3c6cc Update sample.md docs add bc2d9aa Update Compaction Strategy example (#117) new 04317dc Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: docs/compactionStrategy.md | 161 +++-- docs/sample.md | 8 +- docs/shard.md | 42 +++--- .../accumulo/examples/sample/SampleExample.java| 1 + 4 files changed, 140 insertions(+), 72 deletions(-)
[accumulo-examples] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git commit 04317dcca06c50c3da7b8232d73206a95d81e674 Merge: 97b9df3 bc2d9aa Author: Mark Owens AuthorDate: Mon Feb 27 14:32:21 2023 -0500 Merge branch '2.1' docs/compactionStrategy.md | 161 +++-- docs/sample.md | 8 +- docs/shard.md | 42 +++--- .../accumulo/examples/sample/SampleExample.java| 1 + 4 files changed, 140 insertions(+), 72 deletions(-)
[accumulo-examples] branch sampexpfix created (now b5ebce4)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch sampexpfix in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git at b5ebce4 Add flush command following use of batchwriter This branch includes the following new commits: new b5ebce4 Add flush command following use of batchwriter The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[accumulo-examples] 01/01: Add flush command following use of batchwriter
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch sampexpfix
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
commit b5ebce40f7ec7ec157f91b3d758db1d2024a1c28
Author: Mark Owens
AuthorDate: Mon Feb 27 12:55:42 2023 -0500
Add flush command following use of batchwriter
Add a flush command following use of batchWriter addMutaion command in
SampleExample.java.
---
src/main/java/org/apache/accumulo/examples/sample/SampleExample.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/src/main/java/org/apache/accumulo/examples/sample/SampleExample.java
b/src/main/java/org/apache/accumulo/examples/sample/SampleExample.java
index 2e27811..6b5e3c8 100644
--- a/src/main/java/org/apache/accumulo/examples/sample/SampleExample.java
+++ b/src/main/java/org/apache/accumulo/examples/sample/SampleExample.java
@@ -111,6 +111,7 @@ public class SampleExample {
// update a document in the sample data
bw.addMutation(createMutation("2317", "milk, eggs, bread,
parmigiano-reggiano, butter",
"file://groceries/9/txt"));
+bw.flush();
System.out.println(
"Scanning sample after updating content for docId 2317 (should see
content change in sample data) : ");
[accumulo-examples] branch 2.1 updated: Update Compaction Strategy example (#117)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new bc2d9aa Update Compaction Strategy example (#117) bc2d9aa is described below commit bc2d9aa0a199aecd887f9e8df92c573253c63664 Author: Mark Owens AuthorDate: Mon Feb 27 11:53:43 2023 -0500 Update Compaction Strategy example (#117) * Update Compaction Strategy example Update the Compaction Strategy example to make use of the new Compaction Configuration API. --- docs/compactionStrategy.md | 161 - 1 file changed, 117 insertions(+), 44 deletions(-) diff --git a/docs/compactionStrategy.md b/docs/compactionStrategy.md index b0be2fa..e003ed7 100644 --- a/docs/compactionStrategy.md +++ b/docs/compactionStrategy.md @@ -16,64 +16,137 @@ limitations under the License. --> # Apache Accumulo Customizing the Compaction Strategy -This tutorial uses the following Java classes, which can be found in org.apache.accumulo.tserver.compaction: - - * DefaultCompactionStrategy.java - determines which files to compact based on table.compaction.major.ratio and table.file.max - * EverythingCompactionStrategy.java - compacts all files - * SizeLimitCompactionStrategy.java - compacts files no bigger than table.majc.compaction.strategy.opts.sizeLimit - * BasicCompactionStrategy.java - uses default compression table.majc.compaction.strategy.opts.filter.size to filter input - files based on size set and table.majc.compaction.strategy.opts.large.compress.threshold - and table.majc.compaction.strategy.opts.file.large.compress.type for larger files. - - This is an example of how to configure a compaction strategy. By default, Accumulo will always use the DefaultCompactionStrategy, unless these steps are taken to change the configuration. Use the strategy and settings that best fits your Accumulo setup. This example shows -how to configure and test one of the more complicated strategies, the BasicCompactionStrategy. Note that this example requires hadoop -native libraries built with snappy in order to use snappy compression. +how to configure a non-default strategy. Note that this example requires hadoop native libraries built with snappy in order to +use snappy compression. Within this example, commands starting with `user@uno>` are run from within the Accumulo shell, whereas +commands beginning with `$` are executed from the command line terminal. -To begin, run the command to create a table for testing. +Start by creating a table that will be used for the compactions. -```bash -$ accumulo shell -u -p -e "createnamespace examples" -$ accumulo shell -u -p -e "createtable examples.test1" -``` +user@uno> createnamespace examples +user@uno> createtable examples.test1 + +Take note of the TableID for examples.test1. This will be needed later. The TableID can be found by running: + + +user@uno> tables -l +accumulo.metadata=>!0 +accumulo.replication => +rep +accumulo.root=>+r +examples.test1 => 2 -The commands below will configure the BasicCompactionStrategy to: - - Avoid compacting files over 250M - - Compact files over 100M using gz +The commands below will configure the desired compaction strategy. The goals are: + + - Avoid compacting files over 250M. + - Compact files over 100M using gz. - Compact files less than 100M using snappy. - -```bash - $ accumulo shell -u -p -e "config -t examples.test1 -s table.file.compress.type=snappy" - $ accumulo shell -u -p -e "config -t examples.test1 -s table.majc.compaction.strategy=org.apache.accumulo.tserver.compaction.strategies.BasicCompactionStrategy" - $ accumulo shell -u -p -e "config -t examples.test1 -s table.majc.compaction.strategy.opts.filter.size=250M" - $ accumulo shell -u -p -e "config -t examples.test1 -s table.majc.compaction.strategy.opts.large.compress.threshold=100M" - $ accumulo shell -u -p -e "config -t examples.test1 -s table.majc.compaction.strategy.opts.large.compress.type=gz" -``` + - Limit the compaction throughput to 40MB/s. + +Create a compaction service named `cs1` that has three executors. The first executor named `small` has +8 threads and runs compactions less than 16M. The second executor, `medium`, runs compactions less than +128M with 4 threads. The last executor, `large`, runs all other compactions with 2 threads. + +user@uno> config -s tserver.compaction.major.service.cs1.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner +user@uno> config -s 't
[accumulo-website] branch asf-site updated (f4932958 -> 4df3a3e9)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git from f4932958 Automatic Site Publish by Buildbot add 4df3a3e9 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/docs/2.x/administration/compaction.html | 2 +- output/feed.xml| 4 ++-- output/search_data.json| 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
[accumulo-website] branch main updated: Correct compaction property name (#376)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 7752a1e2 Correct compaction property name (#376)
7752a1e2 is described below
commit 7752a1e2f421e2770ffcebb68de83976adfcf7ab
Author: Mark Owens
AuthorDate: Thu Feb 23 08:41:04 2023 -0500
Correct compaction property name (#376)
Replace use of 'throughput' with 'rate.limit' in the Configuration section
of the Compaction documentation.
Replaced:
config -s tserver.compaction.major.service.cs2.throughput=40M
with:
config -s tserver.compaction.major.service.cs2.rate.limit=40M
---
_docs-2/administration/compaction.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/_docs-2/administration/compaction.md
b/_docs-2/administration/compaction.md
index 6ef01668..d4eab1de 100644
--- a/_docs-2/administration/compaction.md
+++ b/_docs-2/administration/compaction.md
@@ -56,7 +56,7 @@ config -s
tserver.compaction.major.service.cs1.planner=org.apache.accumulo.core.
config -s
'tserver.compaction.major.service.cs1.planner.opts.executors=[{"name":"small","type":"internal","maxSize":"16M","numThreads":8},{"name":"medium","type":"internal","maxSize":"128M","numThreads":4},{"name":"large","type":"internal","numThreads":2}]'
config -s
tserver.compaction.major.service.cs2.planner=org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner
config -s
'tserver.compaction.major.service.cs2.planner.opts.executors=[{"name":"small","type":"internal","maxSize":"16M","numThreads":4},{"name":"medium","type":"internal","maxSize":"128M","numThreads":2},{"name":"large","type":"internal","numThreads":1}]'
-config -s tserver.compaction.major.service.cs2.throughput=40M
+config -s tserver.compaction.major.service.cs2.rate.limit=40M
config -t ci -s
table.compaction.dispatcher=org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher
config -t ci -s table.compaction.dispatcher.opts.service=cs1
config -t ci -s table.compaction.dispatcher.opts.service.user=cs2
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e6af9c041f1f184f1cff09f34f43e974b4cde418 Merge: 6d5e593373 55d972a4c7 Author: Mark Owens AuthorDate: Wed Feb 22 09:34:13 2023 -0500 Merge branch '2.1' .../spi/compaction/DefaultCompactionPlanner.java | 4 +- .../compaction/CompactionPlannerInitParams.java| 4 +- .../server/conf/CheckCompactionConfigTest.java | 51 ++ 3 files changed, 55 insertions(+), 4 deletions(-)
[accumulo] branch main updated (6d5e593373 -> e6af9c041f)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 6d5e593373 Rename GarbageCollectionLogger, optionally pause scans/compactions until memory usage is lower (#3161) add 55d972a4c7 Added error details for Compaction Configuration failures (#3199) new e6af9c041f Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../spi/compaction/DefaultCompactionPlanner.java | 4 +- .../compaction/CompactionPlannerInitParams.java| 4 +- .../server/conf/CheckCompactionConfigTest.java | 51 ++ 3 files changed, 55 insertions(+), 4 deletions(-)
[accumulo] branch 2.1 updated: Added error details for Compaction Configuration failures (#3199)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 55d972a4c7 Added error details for Compaction Configuration failures
(#3199)
55d972a4c7 is described below
commit 55d972a4c750c81aba8c59f5841d6369ad40d519
Author: Mark Owens
AuthorDate: Wed Feb 22 09:29:36 2023 -0500
Added error details for Compaction Configuration failures (#3199)
Added additional error details for a couple of Compaction configuration
failure conditions.
The use of duplicate Compaction Executor ID's in configuration will
return an explicit message detailing the issue in the error output.
Similarly, maxSize errors or null Compaction ID's will detail the issue in
the error output.
---
.../spi/compaction/DefaultCompactionPlanner.java | 4 +-
.../compaction/CompactionPlannerInitParams.java| 4 +-
.../server/conf/CheckCompactionConfigTest.java | 51 ++
3 files changed, 55 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index 4ffbbb100d..040296e140 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -128,8 +128,8 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
final Long maxSize;
public Executor(CompactionExecutorId ceid, Long maxSize) {
- Preconditions.checkArgument(maxSize == null || maxSize > 0);
- this.ceid = Objects.requireNonNull(ceid);
+ Preconditions.checkArgument(maxSize == null || maxSize > 0, "Invalid
value for maxSize");
+ this.ceid = Objects.requireNonNull(ceid, "Compaction ID is null");
this.maxSize = maxSize;
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
index bb1093d2b6..0f79ce4df0 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionPlannerInitParams.java
@@ -33,7 +33,6 @@ import
org.apache.accumulo.core.spi.compaction.ExecutorManager;
import com.google.common.base.Preconditions;
public class CompactionPlannerInitParams implements
CompactionPlanner.InitParameters {
-
private final Map plannerOpts;
private final Map requestedExecutors;
private final Set requestedExternalExecutors;
@@ -72,7 +71,8 @@ public class CompactionPlannerInitParams implements
CompactionPlanner.InitParame
Preconditions.checkArgument(threads > 0, "Positive number of threads
required : %s",
threads);
var ceid = CompactionExecutorIdImpl.internalId(serviceId,
executorName);
-Preconditions.checkState(!getRequestedExecutors().containsKey(ceid));
+Preconditions.checkState(!getRequestedExecutors().containsKey(ceid),
+"Duplicate Compaction Executor ID found");
getRequestedExecutors().put(ceid, threads);
return ceid;
}
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
index a1c2737d1a..c80ab28f8d 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/conf/CheckCompactionConfigTest.java
@@ -128,6 +128,57 @@ public class CheckCompactionConfigTest extends
WithTestNames {
assertTrue(e.getMessage().startsWith(expectedErrorMsg));
}
+ @Test
+ public void testRepeatedCompactionExecutorID() throws Exception {
+String inputString = ("tserver.compaction.major.service.cs1.planner="
++ "org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner \n"
++ "tserver.compaction.major.service.cs1.planner.opts.executors=\\\n"
++
"[{'name':'small','type':'internal','maxSize':'16M','numThreads':8},\\\n"
++
"{'name':'medium','type':'internal','maxSize':'128M','numThreads':4},\\\n"
++
"{'name':'small','type':'internal','numThreads':2}]").replaceAll("'", "\"");
+String expectedErrorMsg = "
[accumulo-website] branch asf-site updated (daa1a0ac -> f4932958)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git from daa1a0ac Automatic Site Publish by Buildbot add f8593fe8 Automatic Site Publish by Buildbot add 5b0b474c Automatic Site Publish by Buildbot add 010f8f92 Automatic Site Publish by Buildbot add 5cea24d0 Automatic Site Publish by Buildbot add 945824c0 Automatic Site Publish by Buildbot add f4932958 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/1.10/examples/batch.html| 2 +- output/1.10/examples/bloom.html| 2 +- output/1.10/examples/bulkIngest.html | 2 +- output/1.10/examples/classpath.html| 2 +- output/1.10/examples/client.html | 2 +- output/1.10/examples/combiner.html | 2 +- output/1.10/examples/constraints.html | 2 +- output/1.10/examples/dirlist.html | 2 +- output/1.10/examples/export.html | 2 +- output/1.10/examples/filedata.html | 2 +- output/1.10/examples/filter.html | 2 +- output/1.10/examples/helloworld.html | 2 +- output/1.10/examples/index.html| 2 +- output/1.10/examples/isolation.html| 2 +- output/1.10/examples/mapred.html | 2 +- output/1.10/examples/maxmutation.html | 2 +- output/1.10/examples/regex.html| 2 +- output/1.10/examples/reservations.html | 2 +- output/1.10/examples/rgbalancer.html | 2 +- output/1.10/examples/rowhash.html | 2 +- output/1.10/examples/sample.html | 2 +- output/1.10/examples/shard.html| 2 +- output/1.10/examples/tabletofile.html | 2 +- output/1.10/examples/terasort.html | 2 +- output/1.10/examples/visibility.html | 2 +- output/1.3/user_manual/Accumulo_Design.html| 2 +- output/1.3/user_manual/Accumulo_Shell.html | 2 +- output/1.3/user_manual/Administration.html | 2 +- output/1.3/user_manual/Analytics.html | 2 +- output/1.3/user_manual/Contents.html | 2 +- output/1.3/user_manual/High_Speed_Ingest.html | 2 +- output/1.3/user_manual/Introduction.html | 2 +- output/1.3/user_manual/Security.html | 2 +- output/1.3/user_manual/Shell_Commands.html | 2 +- output/1.3/user_manual/Table_Configuration.html| 2 +- output/1.3/user_manual/Table_Design.html | 2 +- .../1.3/user_manual/Writing_Accumulo_Clients.html | 2 +- output/1.3/user_manual/accumulo_user_manual.html | 2 +- output/1.3/user_manual/examples/aggregation.html | 2 +- output/1.3/user_manual/examples/batch.html | 2 +- output/1.3/user_manual/examples/bloom.html | 2 +- output/1.3/user_manual/examples/bulkIngest.html| 2 +- output/1.3/user_manual/examples/constraints.html | 2 +- output/1.3/user_manual/examples/dirlist.html | 2 +- output/1.3/user_manual/examples/filter.html| 2 +- output/1.3/user_manual/examples/helloworld.html| 2 +- output/1.3/user_manual/examples/index.html | 2 +- output/1.3/user_manual/examples/mapred.html| 2 +- output/1.3/user_manual/examples/shard.html | 2 +- output/1.3/user_manual/index.html | 2 +- output/1.4/examples/batch.html | 2 +- output/1.4/examples/bloom.html | 2 +- output/1.4/examples/bulkIngest.html| 2 +- output/1.4/examples/combiner.html | 2 +- output/1.4/examples/constraints.html | 2 +- output/1.4/examples/dirlist.html | 2 +- output/1.4/examples/filedata.html | 2 +- output/1.4/examples/filter.html| 2 +- output/1.4/examples/helloworld.html| 2 +- output/1.4/examples/index.html | 2 +- output/1.4/examples/isolation.html | 2 +- output/1.4/examples/mapred.html| 2 +- output/1.4/examples/shard.html | 2 +- output/1.4/examples/visibility.html| 2 +- output/1.4/user_manual/Accumulo_Design.html| 2 +- output/1.4/user_manual/Accumulo_Shell.html | 2 +- output/1.4/user_manual/Administration.html | 2 +- output/1.4/user_manual/Analytics.html | 2 +- output/1.4/user_manual/Contents.html | 2 +- output/1.4/user_manual/Development_Clients.html| 2 +- output/1.4/user_manual/High_Speed_Ingest.html | 2 +- output/1.4/user_manual/Introduction.html | 2 +- output/1.4/user_manual/Security.html | 2
[accumulo-examples] branch 2.1 updated: Update sample.md docs
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new 1a3c6cc Update sample.md docs 1a3c6cc is described below commit 1a3c6ccb04fb1719ac877f252c35a7357d3945d4 Author: Mark Owens AuthorDate: Tue Feb 14 10:49:31 2023 -0500 Update sample.md docs Updated some of the counts to more closely match output with recent updates to Accumulo. --- docs/sample.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sample.md b/docs/sample.md index 9990907..05ac19f 100644 --- a/docs/sample.md +++ b/docs/sample.md @@ -170,15 +170,15 @@ After enabling sampling, the command below counts the number of documents in the sample containing the words `import` and `int`. $ ./bin/runex shard.Query --sample -t examples.shard import int | fgrep '.java' | wc - 11 111246 + 4 4 395 The command below counts the total number of documents containing the words `import` and `int`. $ ./bin/runex shard.Query -t examples.shard import int | fgrep '.java' | wc - 10851085 118175 +382 382 40084 -The counts 11 out of 1085 total are around what would be expected for a modulus +The counts 4 out of 395 total are around what would be expected for a modulus of 101. Querying the sample first provides a quick way to estimate how much data the real query will bring back.
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a7a58d6dd7b918072712549c2b08913e58bfcece
Merge: 476ebf2001 d15493b275
Author: Mark Owens
AuthorDate: Tue Feb 14 09:40:05 2023 -0500
Merge branch '2.1'
.../core/client/sample/AbstractHashSampler.java| 37 +-
.../core/client/sample/RowColumnSampler.java | 5 +-
.../apache/accumulo/test/shell/ShellServerIT.java | 79 +-
3 files changed, 101 insertions(+), 20 deletions(-)
diff --cc test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
index 6c1cae6764,d24363a169..129eff219b
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java
@@@ -879,8 -889,20 +880,8 @@@ public class ShellServerIT extends Shar
assertEquals(3, countFiles(clone2Id));
}
- @Test
- public void testCompactionSelectionAndStrategy() throws Exception {
-
-final String table = getUniqueNames(1)[0];
-
-ts.exec("createtable " + table);
-
-// expect this to fail
-ts.exec("compact -t " + table + " -w --sf-ename F.* -s "
-+ TestCompactionStrategy.class.getName() + " -sc
inputPrefix=F,dropPrefix=A", false);
- }
-
@Test
- public void testScanScample() throws Exception {
+ public void testScanSample() throws Exception {
final String table = getUniqueNames(1)[0];
// compact
[accumulo] branch main updated (476ebf2001 -> a7a58d6dd7)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 476ebf2001 Use serialized object instead of byte[] for service locks (#3189) add d15493b275 Update validation of options in Sampler code (#3193) new a7a58d6dd7 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/sample/AbstractHashSampler.java| 37 +- .../core/client/sample/RowColumnSampler.java | 5 +- .../apache/accumulo/test/shell/ShellServerIT.java | 79 +- 3 files changed, 101 insertions(+), 20 deletions(-)
[accumulo] branch 2.1 updated: Update validation of options in Sampler code (#3193)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d15493b275 Update validation of options in Sampler code (#3193)
d15493b275 is described below
commit d15493b275cbd38aee1275957f30ef9c0724b927
Author: Mark Owens
AuthorDate: Tue Feb 14 08:14:10 2023 -0500
Update validation of options in Sampler code (#3193)
Discovered a bug in AbstractHashSampler and RowColumnSampler classes
while examining the Sample example. The final portion of the example was
failing to return any data. Found that the AbstractHashSampler would
produce and
error any time an any additional option were provided. It would only handle
the
required hasher and modulus options correctly.
Conversely, the RowColumnSampler would fail when presented with the
require hasher and modulus options.
Updated such that the RowColumnSampler will ignore the require options,
allowing the AbstractHashSampler to handle them, and only validate options
specific to the RowColumnSampler.
The AbstractHashSampler was updated to ignore any non-required option.
Comments were added to indicate that the AbstractHashSampler validates
required options while call sub-classes should handle any additional Sampler
specific options.
---
.../core/client/sample/AbstractHashSampler.java| 37 +-
.../core/client/sample/RowColumnSampler.java | 5 +-
.../apache/accumulo/test/shell/ShellServerIT.java | 79 +-
3 files changed, 101 insertions(+), 20 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
b/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
index f4a2abd1a8..072f6cae97 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/sample/AbstractHashSampler.java
@@ -57,8 +57,12 @@ public abstract class AbstractHashSampler implements Sampler
{
private HashFunction hashFunction;
private int modulus;
- private static final Set VALID_OPTIONS = Set.of("hasher", "modulus");
- private static final Set VALID_VALUES_HASHER = Set.of("murmur3_32",
"md5", "sha1");
+ private static final String HASHER_PROP_NAME = "hasher";
+ private static final String MODULUS_PROP_NAME = "modulus";
+
+ protected static final Set REQUIRED_SAMPLER_OPTIONS =
+ Set.of(HASHER_PROP_NAME, MODULUS_PROP_NAME);
+ private static final Set VALID_HASHERS = Set.of("murmur3_32", "md5",
"sha1");
/**
* Subclasses with options should override this method to validate subclass
options while also
@@ -66,17 +70,16 @@ public abstract class AbstractHashSampler implements
Sampler {
*/
@Override
public void validateOptions(Map config) {
+// Validate required properties, HASHER_PROP_NAME and MODULUS_PROP_NAME
+// Any additional options are validated in calling subclass
for (Map.Entry entry : config.entrySet()) {
- checkArgument(VALID_OPTIONS.contains(entry.getKey()), "Unknown option:
%s", entry.getKey());
-
- if (entry.getKey().equals("hasher")) {
-checkArgument(VALID_VALUES_HASHER.contains(entry.getValue()),
-"Unknown value for hasher: %s", entry.getValue());
+ if (entry.getKey().equals(HASHER_PROP_NAME)) {
+checkArgument(VALID_HASHERS.contains(entry.getValue()), "Unknown value
for %s: %s",
+HASHER_PROP_NAME, entry.getValue());
}
-
- if (entry.getKey().equals("modulus")) {
-checkArgument(Integer.parseInt(entry.getValue()) > 0,
-"Improper Integer value for modulus: %s", entry.getValue());
+ if (entry.getKey().equals(MODULUS_PROP_NAME)) {
+checkArgument(Integer.parseInt(entry.getValue()) > 0, "Improper
Integer value for %s: %s",
+MODULUS_PROP_NAME, entry.getValue());
}
}
}
@@ -89,7 +92,7 @@ public abstract class AbstractHashSampler implements Sampler {
*/
@Deprecated(since = "2.1.0")
protected boolean isValidOption(String option) {
-return VALID_OPTIONS.contains(option);
+return REQUIRED_SAMPLER_OPTIONS.contains(option);
}
/**
@@ -99,11 +102,11 @@ public abstract class AbstractHashSampler implements
Sampler {
justification = "these hashes don't protect any secrets, just used for
binning")
@Override
public void init(SamplerConfiguration config) {
-String hasherOpt = config.getOptions().get("hasher");
-String modulusOpt = config.getOptions().get("modulus");
+String hashe
[accumulo-examples] branch 2.1 updated: Update SampleExample and documentation (#116)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new 3b8088a Update SampleExample and documentation (#116) 3b8088a is described below commit 3b8088a64187d6f19a28bb357a7c105b1aaf754a Author: Mark Owens AuthorDate: Fri Jan 27 10:37:19 2023 -0500 Update SampleExample and documentation (#116) * Added BatchWriter flush command to SampleExample.java to ensure updated sample data is output when executing SampleExample via the runex command * Updated Sample.md documentation to correctly display changes with the update to Accumulo 2.1 * Updated Shard.md documentation to correctly display changes with the update to Accumulo 2.1 --- docs/sample.md | 2 +- docs/shard.md | 42 ++ .../accumulo/examples/sample/SampleExample.java| 1 + 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/docs/sample.md b/docs/sample.md index 33c9e81..9990907 100644 --- a/docs/sample.md +++ b/docs/sample.md @@ -79,7 +79,7 @@ failure and fixing the problem with a compaction. root@instance examples.sampex> compact -t examples.sampex --sf-no-sample 2015-09-09 12:23:07,242 [shell.Shell] INFO : Compaction of table sampex started for given range root@instance examples.sampex> scan --sample -2317 doc:content []milk, eggs, bread, parmigiano-reggiano +2317 doc:content []milk, eggs, bread, parmigiano-reggiano, butter 2317 doc:url []file://groceries/9.txt 3900 doc:content []EC2 ate my homework 3900 doc:uril []file://final_project.txt diff --git a/docs/shard.md b/docs/shard.md index d779c6f..05beea3 100644 --- a/docs/shard.md +++ b/docs/shard.md @@ -30,36 +30,29 @@ To run these example programs, create two tables like below. username@instance> createtable examples.shard username@instance examples.shard> createtable examples.doc2term -After creating the tables, index some files. The following command indexes all of the java files in the Accumulo source code. +After creating the tables, index some files. The following command indexes all the java files in the Accumulo source code. $ find /path/to/accumulo/core -name "*.java" | xargs ./bin/runex shard.Index -t examples.shard --partitions 30 The following command queries the index to find all files containing 'foo' and 'bar'. $ ./bin/runex shard.Query -t examples.shard foo bar - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/client/admin/NewTableConfigurationTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/BaseHostRegexTableLoadBalancerTest.java /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/iterators/user/WholeRowIteratorTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/iterators/user/WholeColumnFamilyIteratorTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/data/KeyBuilderTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerReconfigurationTest.java /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/conf/IterConfigUtilTest.java /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/clientImpl/TableOperationsHelperTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/clientImpl/mapreduce/BatchInputSplitTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerReconfigurationTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/client/IteratorSettingTest.java - /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/client/mapred/RangeInputSplitTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/
[accumulo-examples] branch 2.1 updated: Add documentation for uniquecols example (#115)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new ae45b3d Add documentation for uniquecols example (#115) ae45b3d is described below commit ae45b3db10ed71576910eedc1c169331f4fdefbe Author: Mark Owens AuthorDate: Thu Jan 26 16:00:46 2023 -0500 Add documentation for uniquecols example (#115) Add documentation for the uniquecols examples for the 2.1 branch. --- docs/uniquecols.md | 71 -- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/uniquecols.md b/docs/uniquecols.md index 46b6a30..714fcb2 100644 --- a/docs/uniquecols.md +++ b/docs/uniquecols.md @@ -17,7 +17,74 @@ limitations under the License. # Apache Accumulo Unique Columns example The UniqueColumns examples ([UniqueColumns.java]) computes the unique set -of columns in a table and shows how a map reduce job can directly read a -tables files from HDFS. +of column family and column qualifiers in a table. It also demonstrates +how a mapReduce job can directly read a tables files from HDFS. + +Create a table and add rows that all have identical column family and column +qualifiers. + +``` +$ /path/to/accumulo shell -u username -p secret +username@instance> createnamespace examples +username@instance> createtable examples.unique +username@instance> examples.unique> insert row1 fam1 qual1 v1 +username@instance> examples.unique> insert row2 fam1 qual1 v2 +username@instance> examples.unique> insert row3 fam1 qual1 v3 +``` + +Exit the Accumulo shell and run the uniqueColumns mapReduce job against +this table. Note that if the output file already exists in HDFS, it will +need to be deleted. + +``` +$ ./bin/runmr mapreduce.UniqueColumns --table examples.unique --reducers 1 --output /tmp/unique +``` + +When the mapReduce job completes, examine the output. + +``` +$ hdfs dfs -cat /tmp/unique/part-r-0 +cf:fam1 +cq:qual1 +``` + +The output displays the unique column family and column qualifier values. In +this case since all rows use the same values, there are only two values output. + +Note that since the example used only one reducer all output will be contained +within the single `part-r-0` file. If more than one reducer is used the output +will be spread among various `part-r-x` files. + +Go back to the shell and add some additional entries. + +```text +$ /path/to/accumulo shell -u username -p secret +username@instance> table unique +username@instance example.unique> insert row1 fam2 qual2 v2 +username@instance example.unique> insert row1 fam3 qual2 v2 +username@instance example.unique> insert row1 fam2 qual2 v2 +username@instance example.unique> insert row2 fam2 qual2 v2 +username@instance example.unique> insert row3 fam2 qual2 v2 +username@instance example.unique> insert row3 fam3 qual3 v2 +username@instance example.unique> insert row3 fam3 qual4 v2 +``` + +Re-running the command will now find any additional unique column values. + +```text +$ hdfs dfs -rm -r -f /tmp/unique +$ ./bin/runmr mapreduce.UniqueColumns --table examples.unique --reducers 1 --output /tmp/unique +$ hdfs dfs -cat /tmp/unique/part-r-0 +cf:fam1 +cf:fam2 +cf:fam3 +cq:qual1 +cq:qual2 +cq:qual3 +cq:qual4 +``` + +The output now includes the additional column values that were added during the last batch of inserts. + [UniqueColumns.java]: ../src/main/java/org/apache/accumulo/examples/mapreduce/UniqueColumns.java
[accumulo-examples] branch main updated: Create user documentation for the UniqueColumns example. (#114)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 97b9df3 Create user documentation for the UniqueColumns example. (#114) 97b9df3 is described below commit 97b9df35a2497cacde1c7c126e51d382ce454bec Author: Mark Owens AuthorDate: Wed Jan 18 09:24:13 2023 -0500 Create user documentation for the UniqueColumns example. (#114) The uniqueColumns example has had little to no documentation in the past. There was no indication as to how the example should be run or even what it actually did. This PR adds a more thorough description and gives step-by-step instructions on how to utilize the example. --- docs/uniquecols.md | 71 -- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/uniquecols.md b/docs/uniquecols.md index 46b6a30..99ad701 100644 --- a/docs/uniquecols.md +++ b/docs/uniquecols.md @@ -17,7 +17,74 @@ limitations under the License. # Apache Accumulo Unique Columns example The UniqueColumns examples ([UniqueColumns.java]) computes the unique set -of columns in a table and shows how a map reduce job can directly read a -tables files from HDFS. +of column family and column qualifiers in a table. It also demonstrates +how a mapReduce job can directly read a tables files from HDFS. + +Create a table and add rows that all have identical column family and column +qualifiers. + +``` +$ /path/to/accumulo shell -u username -p secret +username@instance> createnamespace examples +username@instance> createtable examples.unique +username@instance> examples.unique> insert row1 fam1 qual1 v1 +username@instance> examples.unique> insert row2 fam1 qual1 v2 +username@instance> examples.unique> insert row3 fam1 qual1 v3 +``` + +Exit the Accumulo shell and run the uniqueColumns mapReduce job against +this table. Note that if the output file already exists in HDFS, it will +need to be deleted. + +``` +$ ./bin/runmr mapreduce.UniqueColumns --table examples.unique --reducers 1 --output /tmp/unique +``` + +When the mapReduce job completes, examine the output. + +``` +$ hdfs dfs -cat /tmp/unique/part-r-0 +cf:fam1 +cq:qual1 +``` + +The output displays the unique column family and column qualifier values. In +this case since all rows use the same values, there are only two values output. + +Note that since the example used only one reducer all output will be contained +within the single `part-r-0` file. If more than one reducer is used the output +will be spread among various `part-r-x` files. + +Go back to the shell and add some additional entries. + +```text +$ /path/to/accumulo shell -u username -p secret +username@instance> table unique +username@instance example.unique> insert row1 fam2 qual2 v2 +username@instance example.unique> insert row1 fam3 qual2 v2 +username@instance example.unique> insert row1 fam2 qual2 v2 +username@instance example.unique> insert row2 fam2 qual2 v2 +username@instance example.unique> insert row3 fam2 qual2 v2 +username@instance example.unique> insert row3 fam3 qual3 v2 +username@instance example.unique> insert row3 fam3 qual4 v2 +``` + +Re-running the command will now find any additional unique column values. + +```text +$ hdfs dfs -rm -r -f /tmp/unique +$ ./bin/runmr mapreduce.UniqueColumns --table examples.unique --reducers 1 --output /tmp/unique +$ hdfs dfs -cat /tmp/unique/part-r-0 +cf:fam1 +cf:fam2 +cf:fam3 +cq:qual1 +cq:qual2 +cq:qual3 +cq:qual4 +``` + +The output now includes the additional column values that were added during the last batch of inserts. + [UniqueColumns.java]: ../src/main/java/org/apache/accumulo/examples/mapreduce/UniqueColumns.java
[accumulo-examples] branch 2.1 updated: Update POM version info for 2.1 examples branch
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new 9fda851 Update POM version info for 2.1 examples branch 9fda851 is described below commit 9fda85174031a87e224766c60c7162f054e179f2 Author: Mark Owens AuthorDate: Wed Nov 23 10:02:02 2022 -0500 Update POM version info for 2.1 examples branch Update the 2.1 version of the accumulo-examples POM file * Set the accumulo-examples project version to `2.1.0`. * Update the `accumulo.version` property to `2.1.0` This branch of the accumulo-examples project should be targeted to work against any version of the Accumulo 2.1.x line of development. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1384798..7fbec6a 100644 --- a/pom.xml +++ b/pom.xml @@ -24,11 +24,11 @@ org.apache.accumulo accumulo-examples - 2.1.1-SNAPSHOT + 2.1.0 Apache Accumulo Examples Example code and corresponding documentation for using Apache Accumulo -2.1.1-SNAPSHOT +2.1.0 contrib/Eclipse-Accumulo-Codestyle.xml 3.3.4 11
[accumulo-examples] branch main updated: Comment out portions of filedata example (#108)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new a0cac7c Comment out portions of filedata example (#108)
a0cac7c is described below
commit a0cac7cbb0cc8c62c690e33e12acbeab9c517cca
Author: Mark Owens
AuthorDate: Tue Nov 22 16:23:50 2022 -0500
Comment out portions of filedata example (#108)
With the example repo now targeting Accumulo-3.0.0-SNAPSHOT, the
filedata example now longer works. In addition, it prevents the
project from building. Several files have been commented in order to
allow the project to build.
A follow-on issue will be created to address the non-working example.
---
.../examples/filedata/CharacterHistogram.java | 224 +++
.../examples/filedata/ChunkInputFormat.java| 169 ++---
.../examples/filedata/ChunkInputFormatIT.java | 688 +++--
3 files changed, 542 insertions(+), 539 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
b/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
index 01c08bc..d24c4c9 100644
---
a/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
+++
b/src/main/java/org/apache/accumulo/examples/filedata/CharacterHistogram.java
@@ -1,112 +1,112 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.examples.filedata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Properties;
-
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.iterators.user.SummingArrayCombiner;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.examples.cli.ClientOpts;
-import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-
-import com.beust.jcommander.Parameter;
-
-/**
- * A MapReduce that computes a histogram of byte frequency for each file and
stores the histogram
- * alongside the file data. The {@link ChunkInputFormat} is used to read the
file data from
- * Accumulo.
- */
-public class CharacterHistogram {
-
- private static final String VIS = "vis";
-
- public static class HistMapper extends
Mapper>,InputStream,Text,Mutation> {
-private ColumnVisibility cv;
-
-@Override
-public void map(List> k, InputStream v, Context context)
-throws IOException, InterruptedException {
- Long[] hist = new Long[256];
- Arrays.fill(hist, 0L);
- int b = v.read();
- while (b >= 0) {
-hist[b] += 1L;
-b = v.read();
- }
- v.close();
- Mutation m = new Mutation(k.get(0).getKey().getRow());
- m.put("info", "hist", cv,
- new
Value(SummingArrayCombiner.STRING_ARRAY_ENCODER.encode(Arrays.asList(hist;
- context.write(new Text(), m);
-}
-
-@Override
-protected void setup(Context context) {
- cv = new ColumnVisibility(context.getConfiguration().get(VIS, ""));
-}
- }
-
- static class Opts extends ClientOpts {
-@Parameter(names = {"-t", "--table"}, required = true, description =
"table to use")
-String tableName;
-@Parameter(names = "--vis")
-String visibilities = "";
- }
-
- @SuppressWarnings("deprecation")
- public static void main(String[] args) throws Exception {
-Opts opts = new Opts();
-opts.parseArgs(CharacterHistogram.class.getName(), args);
-
-Job job = Job.getInstance(opts.getHadoopConfig());
-job.setJobName(CharacterHistogram.class.getSimpleName());
-job.setJarByClass(CharacterHistogram.class);
-
[accumulo-examples] branch main updated: Update accumulo-examples to target 3.0.0-SNAPSHOT (#106)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 5fcb5a1 Update accumulo-examples to target 3.0.0-SNAPSHOT (#106) 5fcb5a1 is described below commit 5fcb5a175ceae55b9d5c8a6632d6e524531fdf93 Author: Mark Owens AuthorDate: Tue Nov 22 16:03:09 2022 -0500 Update accumulo-examples to target 3.0.0-SNAPSHOT (#106) The main branch of the accumulo-example repository should now target the current development branch of Accumulo, which is 3.0.0-SNAPSHOT. A 2.1 branch has been created that will continue to target the 2.1.x line of Accumulo. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ee5d26a..c97bbde 100644 --- a/pom.xml +++ b/pom.xml @@ -24,11 +24,11 @@ org.apache.accumulo accumulo-examples - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT Apache Accumulo Examples Example code and corresponding documentation for using Apache Accumulo -2.1.0-SNAPSHOT +3.0.0-SNAPSHOT contrib/Eclipse-Accumulo-Codestyle.xml 3.3.4 11
[accumulo-examples] branch 2.1 updated: Update POM to target 2.1.1-SNAPSHOT (#105)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/2.1 by this push: new 7c0576b Update POM to target 2.1.1-SNAPSHOT (#105) 7c0576b is described below commit 7c0576bdce69fda454ded2ce2261aa7287be21dc Author: Mark Owens AuthorDate: Tue Nov 22 09:11:00 2022 -0500 Update POM to target 2.1.1-SNAPSHOT (#105) With the release of 2.1.0, the main Accumulo branch is now targeted for 3.x. The current accumulo-examples target the 2.1.x line. This update targets the 2.1 examples to run against the 2.1.1-SNAPSHOT version of the main Accumulo repository. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ee5d26a..1384798 100644 --- a/pom.xml +++ b/pom.xml @@ -24,11 +24,11 @@ org.apache.accumulo accumulo-examples - 2.1.0-SNAPSHOT + 2.1.1-SNAPSHOT Apache Accumulo Examples Example code and corresponding documentation for using Apache Accumulo -2.1.0-SNAPSHOT +2.1.1-SNAPSHOT contrib/Eclipse-Accumulo-Codestyle.xml 3.3.4 11
[accumulo-examples] branch 2.1 created (now d28cebb)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git at d28cebb Bring some dependency versions up to date with main accumulo project (#104) No new revisions were added by this update.
[accumulo] branch iter110 created (now 5548c9289a)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch iter110 in repository https://gitbox.apache.org/repos/asf/accumulo.git at 5548c9289a Add VFS type mapping for new jar mime type (#3069) No new revisions were added by this update.
[accumulo] branch main updated (4df8ae2640 -> 6c8a4088dd)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4df8ae2640 Update several dependencies to latest versions (#3022) add 6c8a4088dd Change List to SortedSet in RecoveryLogsIterator (#3021) No new revisions were added by this update. Summary of changes: .../accumulo/tserver/log/RecoveryLogsIterator.java | 16 +++- 1 file changed, 11 insertions(+), 5 deletions(-)
[accumulo] branch main updated: Remove unneeded TODO in TabletBalancer (#3018)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 74155ba491 Remove unneeded TODO in TabletBalancer (#3018)
74155ba491 is described below
commit 74155ba4911e7146dbcb64eb5888441b447ed2c0
Author: Mark Owens
AuthorDate: Wed Oct 12 13:36:27 2022 -0400
Remove unneeded TODO in TabletBalancer (#3018)
Trivial change to remove TODO comments within the TabletBalancer.java 'run'
method.
The TODO references an older Jira ticket ACCUMUO-2938 that has been marked
resolved and closed. The TODO is no longer relevant.
---
.../org/apache/accumulo/server/master/balancer/TabletBalancer.java| 4
1 file changed, 4 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
index 0db9d6521c..23a5f5d679 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/TabletBalancer.java
@@ -213,10 +213,6 @@ public abstract class TabletBalancer
@Override
public void run() {
balancerLog.warn("Not balancing due to {} outstanding migrations.",
migrations.size());
- /*
- * TODO ACCUMULO-2938 redact key extents in this output to avoid leaking
protected
- * information.
- */
balancerLog.debug("Sample up to 10 outstanding migrations: {}",
migrations.stream().limit(10).collect(toList()));
}
[accumulo-website] branch next-release updated (0a6ff174 -> 7deff72b)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch next-release in repository https://gitbox.apache.org/repos/asf/accumulo-website.git from 0a6ff174 Add information about the Scan Server process. (#335) add 7deff72b Replace older Tour with newer JShell Tour (#334) No new revisions were added by this update. Summary of changes: _data/tour-jshell.yml | 14 --- _data/tour.yml | 1 + _includes/nav.html | 2 +- _layouts/tour-jshell.html | 55 - pages/tour.md | 37 -- tour-jshell/authorizations-code2.md | 153 tour-jshell/authorizations2.md | 61 -- tour-jshell/basic-read-write2.md| 80 - tour-jshell/batch-scanner-code2.md | 57 - tour-jshell/batch-scanner2.md | 41 --- tour-jshell/client2.md | 53 - tour-jshell/conditional-writer-code2.md | 36 -- tour-jshell/conditional-writer2.md | 132 - tour-jshell/data-model-code2.md | 75 tour-jshell/data-model2.md | 39 -- tour-jshell/getting-started2.md | 129 tour-jshell/index.md| 26 tour-jshell/ranges-splits2.md | 44 --- tour-jshell/using-iterators2.md | 141 -- tour/authorizations-code.md | 203 ++-- tour/authorizations.md | 68 +-- tour/basic-read-write.md| 103 ++-- tour/batch-scanner-code.md | 80 +++-- tour/batch-scanner.md | 54 - tour/client.md | 63 ++ tour/conditional-writer-code.md | 43 +++ tour/conditional-writer.md | 177 ++-- tour/data-model-code.md | 101 +--- tour/data-model.md | 41 --- tour/getting-started.md | 136 + tour/index.md | 16 ++- tour/ranges-splits.md | 37 +++--- tour/using-iterators.md | 138 ++ 33 files changed, 835 insertions(+), 1601 deletions(-) delete mode 100644 _data/tour-jshell.yml delete mode 100644 _layouts/tour-jshell.html delete mode 100644 pages/tour.md delete mode 100644 tour-jshell/authorizations-code2.md delete mode 100644 tour-jshell/authorizations2.md delete mode 100644 tour-jshell/basic-read-write2.md delete mode 100644 tour-jshell/batch-scanner-code2.md delete mode 100644 tour-jshell/batch-scanner2.md delete mode 100644 tour-jshell/client2.md delete mode 100644 tour-jshell/conditional-writer-code2.md delete mode 100644 tour-jshell/conditional-writer2.md delete mode 100644 tour-jshell/data-model-code2.md delete mode 100644 tour-jshell/data-model2.md delete mode 100644 tour-jshell/getting-started2.md delete mode 100644 tour-jshell/index.md delete mode 100644 tour-jshell/ranges-splits2.md delete mode 100644 tour-jshell/using-iterators2.md
[accumulo-website] branch main updated: Minor spelling update to metrics-tracing blog post
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new 0b507ab0 Minor spelling update to metrics-tracing blog post 0b507ab0 is described below commit 0b507ab0c06a56abe3dbdfbc86b5567bfa4b7095 Author: Mark Owens AuthorDate: Fri Jul 22 10:45:14 2022 -0400 Minor spelling update to metrics-tracing blog post --- _posts/blog/2022-06-22-2.1.0-metrics-and-tracing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_posts/blog/2022-06-22-2.1.0-metrics-and-tracing.md b/_posts/blog/2022-06-22-2.1.0-metrics-and-tracing.md index 5b5264ce..dbbfaa84 100644 --- a/_posts/blog/2022-06-22-2.1.0-metrics-and-tracing.md +++ b/_posts/blog/2022-06-22-2.1.0-metrics-and-tracing.md @@ -15,7 +15,7 @@ Accumulo's metrics integration test uses a [TestStatsDRegistryFactory](https://g ## Metrics Example -This example uses a Docker container that contains Telegraf-InfluxDB-Grafana system. We will configure Accumulo to send metrics to the [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/) component running in the Docker image. Telegraf will persist the metrics in [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) and then we will visualize the metris using [Grafana](https://grafana.com/). This example assumes that you have installed Docker (or equivalent [...] +This example uses a Docker container that contains Telegraf-InfluxDB-Grafana system. We will configure Accumulo to send metrics to the [Telegraf](https://www.influxdata.com/time-series-platform/telegraf/) component running in the Docker image. Telegraf will persist the metrics in [InfluxDB](https://www.influxdata.com/products/influxdb-overview/) and then we will visualize the metrics using [Grafana](https://grafana.com/). This example assumes that you have installed Docker (or equivalent [...] 1. Download the Telegraf-Influx-Grafana (TIG) Docker image ```
[accumulo-website] branch next-release updated (110e622b -> 05eef21a)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch next-release in repository https://gitbox.apache.org/repos/asf/accumulo-website.git from 110e622b Address issue #2479 - Add FATE zookeeper structure documentation to 2.x docs (#328) add 4980f0d0 Add JShell version of the Accumulo Tour to website add 05eef21a Add JShell version of the Accumulo Tour to website No new revisions were added by this update. Summary of changes: Gemfile | 2 + Gemfile.lock| 2 + _config.yml | 7 ++ _data/tour-jshell.yml | 14 +++ _includes/nav.html | 4 +- _layouts/tour-jshell.html | 55 pages/tour.md | 37 tour-jshell/authorizations-code2.md | 153 tour-jshell/authorizations2.md | 61 + tour-jshell/basic-read-write2.md| 80 + tour-jshell/batch-scanner-code2.md | 57 tour-jshell/batch-scanner2.md | 41 + tour-jshell/client2.md | 53 +++ tour-jshell/conditional-writer-code2.md | 36 tour-jshell/conditional-writer2.md | 132 +++ tour-jshell/data-model-code2.md | 75 tour-jshell/data-model2.md | 39 tour-jshell/getting-started2.md | 129 +++ tour-jshell/index.md| 26 ++ tour-jshell/ranges-splits2.md | 44 + tour-jshell/using-iterators2.md | 141 + tour/index.md | 10 +-- 22 files changed, 1190 insertions(+), 8 deletions(-) create mode 100644 _data/tour-jshell.yml create mode 100644 _layouts/tour-jshell.html create mode 100644 pages/tour.md create mode 100644 tour-jshell/authorizations-code2.md create mode 100644 tour-jshell/authorizations2.md create mode 100644 tour-jshell/basic-read-write2.md create mode 100644 tour-jshell/batch-scanner-code2.md create mode 100644 tour-jshell/batch-scanner2.md create mode 100644 tour-jshell/client2.md create mode 100644 tour-jshell/conditional-writer-code2.md create mode 100644 tour-jshell/conditional-writer2.md create mode 100644 tour-jshell/data-model-code2.md create mode 100644 tour-jshell/data-model2.md create mode 100644 tour-jshell/getting-started2.md create mode 100644 tour-jshell/index.md create mode 100644 tour-jshell/ranges-splits2.md create mode 100644 tour-jshell/using-iterators2.md
[accumulo] branch main updated: Implement consistent Namespace and Table property handling (#2771)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6f5c2f07a4 Implement consistent Namespace and Table property handling
(#2771)
6f5c2f07a4 is described below
commit 6f5c2f07a44bbbda1c33fb094654fc5644dab461
Author: Mark Owens
AuthorDate: Mon Jun 13 12:12:02 2022 -0400
Implement consistent Namespace and Table property handling (#2771)
TablePropUtil and NamespacePropUtil handle invalid properties differently.
This update updates the two classes to handle properties in a similar manner.
They both will now throw an exception on an invalid property value.
This is the first step in what may be a multi-step process. With this
change, both TablePropUtil and NamespacePropUtil are very similar and could
possibly be combined into one method. I will create a follow-on ticket to look
into that possibility. But I wanted to get the initial task of making them
consistent completed first.
Closes #2633
---
.../accumulo/server/util/NamespacePropUtil.java| 12 ++-
.../apache/accumulo/server/util/TablePropUtil.java | 18 ++--
.../accumulo/test/ZooKeeperPropertiesIT.java | 116 +
3 files changed, 136 insertions(+), 10 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
index f141513126..da1f13f28c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/NamespacePropUtil.java
@@ -32,12 +32,20 @@ public class NamespacePropUtil extends
PropUtil {
super(context);
}
+ /**
+ * Helper method to set provided properties for the provided namespace.
+ *
+ * @throws IllegalStateException
+ * if an underlying exception (KeeperException,
InterruptException) or other failure to
+ * read properties from the cache / backend store
+ * @throws IllegalArgumentException
+ * if a provided property is not valid
+ */
@Override
public void setProperties(NamespaceId namespaceId, Map
properties) {
for (Map.Entry prop : properties.entrySet()) {
- // TODO reconcile with TablePropUtil see
https://github.com/apache/accumulo/issues/2633
if (!Property.isTablePropertyValid(prop.getKey(), prop.getValue())) {
-throw new IllegalArgumentException("Invalid table property for
namespace: " + namespaceId
+throw new IllegalArgumentException("Invalid property for namespace: "
+ namespaceId
+ " name: " + prop.getKey() + ", value: " + prop.getValue());
}
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
index b3fb54be3b..642af9af00 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/TablePropUtil.java
@@ -19,7 +19,6 @@
package org.apache.accumulo.server.util;
import java.util.Collection;
-import java.util.HashMap;
import java.util.Map;
import org.apache.accumulo.core.conf.Property;
@@ -39,15 +38,18 @@ public class TablePropUtil extends PropUtil {
* @throws IllegalStateException
* if an underlying exception (KeeperException,
InterruptException) or other failure to
* read properties from the cache / backend store
+ * @throws IllegalArgumentException
+ * if a provided property is not valid
*/
@Override
- public void setProperties(TableId tableId, Map props) {
-Map tempProps = new HashMap<>(props);
-// TODO reconcile with NamespacePropUtil see
https://github.com/apache/accumulo/issues/2633
-tempProps.entrySet().removeIf(e ->
!Property.isTablePropertyValid(e.getKey(), e.getValue()));
-
-context.getPropStore().putAll(TablePropKey.of(context, tableId), props);
-
+ public void setProperties(TableId tableId, Map properties) {
+for (Map.Entry prop : properties.entrySet()) {
+ if (!Property.isTablePropertyValid(prop.getKey(), prop.getValue())) {
+throw new IllegalArgumentException("Invalid property for table: " +
tableId + " name: "
++ prop.getKey() + ", value: " + prop.getValue());
+ }
+}
+context.getPropStore().putAll(TablePropKey.of(context, tableId),
properties);
}
@Override
diff --git
a/test/src/main/java/org/apache/accumulo/test/ZooKeeperPropertiesIT.java
b/test/src/main/java/org/apache/accumulo/test/ZooKeeperPropertiesIT.java
index 5805341229..efb8f0a8d6 100644
--- a/test/src/main/java/org/apache/accumulo/te
[accumulo-examples] branch main updated: Add link to the Accumulo Tour
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 516163f Add link to the Accumulo Tour 516163f is described below commit 516163f3ddadc87f9ffdca61afe08458c61055c2 Author: Mark Owens AuthorDate: Mon May 2 10:50:46 2022 -0400 Add link to the Accumulo Tour Add link to the Accumulo Tour to the introductory section of the Accumulo-Examples repository. This link provides several more very simple examples for someone interested in getting familiar with Accumulo. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 7637ce7..f2b23a0 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ greater. Examples within the `main` branch are designed to work with the version under development. Additional branches exist for previous releases of the Accumulo 2.x line. For example, the `2.0` branch contains examples specifically intended to work with that release version. +The [Accumulo Tour] also provides several simple introductory examples that may be of interest. + A collection of examples for Accumulo 1.10 can be found [here]. ## Setup instructions @@ -142,3 +144,4 @@ This repository can be used to test Accumulo release candidates. See [ti]: https://github.com/apache/accumulo-examples/workflows/QA/badge.svg [tl]: https://github.com/apache/accumulo-examples/actions [here]: https://accumulo.apache.org/1.10/examples +[Accumulo Tour]: https://accumulo.apache.org/tour/
[accumulo-examples] branch main updated: Updated table names with appropriate namespace
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 374667a Updated table names with appropriate namespace 374667a is described below commit 374667ac1496d34331521e66f6f72ec1b6e08766 Author: Mark Owens AuthorDate: Fri Apr 29 11:22:17 2022 -0400 Updated table names with appropriate namespace Updated dataTable, dirTable, and indexTable table names with the `examples` namespace prefixed. --- docs/dirlist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dirlist.md b/docs/dirlist.md index 800f7ac..720c96c 100644 --- a/docs/dirlist.md +++ b/docs/dirlist.md @@ -34,7 +34,7 @@ To begin, ingest some data with Ingest.java. $ ./bin/runex dirlist.Ingest --vis exampleVis --chunkSize 10 /local/username/workspace This may take some time if there are large files in the /local/username/workspace directory. If you use 0 instead of 10 as the `chunkSize`, the ingest will run much faster, but it will not put any file data into Accumulo (the dataTable will be empty). -Note that running this example will create tables dirTable, indexTable, and dataTable in Accumulo that you should delete when you have completed the example. +Note that running this example will create tables `examples.dirTable`, `examples.indexTable`, and `examples.dataTable` in Accumulo that you should delete when you have completed the example. If you modify a file or add new files in the directory ingested (e.g. /local/username/workspace), you can run Ingest again to add new information into the Accumulo tables. To browse the data ingested, use Viewer.java. Be sure to give the "username" user the authorizations to see the data (in this case, run
[accumulo] branch main updated: Resolve unused format arguments (#2655)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 295f82c4b0 Resolve unused format arguments (#2655)
295f82c4b0 is described below
commit 295f82c4b0341c1d12fb4ce4ba5dd3816cfa2213
Author: Mark Owens
AuthorDate: Thu Apr 28 13:40:34 2022 -0400
Resolve unused format arguments (#2655)
* Resolve unused format arguments
Updated instances where the number of format arguments did not
match number of arguments.
In CompactionCoordinator the compactorAddress was a third argument for
a statement containing only two format arguments. The compactorAddress
was dropped as it did not seem to be necessary.
In AuditedSecurityOperation there were two occasions of mismatched
formats and arguments. In both cases, an additional format argument was
added to allow both the tableName and tableId to be displayed in the
audit log.
Changes to AuditMessage formatting affected the associated IT test as
well. Updated the IT test to work with the previous changes.
---
.../accumulo/server/security/AuditedSecurityOperation.java | 4 ++--
.../apache/accumulo/coordinator/CompactionCoordinator.java | 2 +-
.../main/java/org/apache/accumulo/test/AuditMessageIT.java | 14 +-
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
b/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
index 1553616665..6837de038b 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
@@ -264,7 +264,7 @@ public class AuditedSecurityOperation extends
SecurityOperation {
}
public static final String CAN_DELETE_TABLE_AUDIT_TEMPLATE =
- "action: deleteTable; targetTable: %s;";
+ "action: deleteTable; targetTable: %s:%s";
@Override
public boolean canDeleteTable(TCredentials c, TableId tableId, NamespaceId
namespaceId)
@@ -682,7 +682,7 @@ public class AuditedSecurityOperation extends
SecurityOperation {
}
public static final String CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE =
- "action: %s; targetTable: %s;";
+ "action: %s; targetTable: %s:%s";
@Override
public boolean canOnlineOfflineTable(TCredentials credentials, TableId
tableId, FateOperation op,
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 5029972a80..1dc1a01f85 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -448,7 +448,7 @@ public class CompactionCoordinator extends AbstractServer
prioTserver.prio, compactorAddress, externalCompactionId);
if (null == job.getExternalCompactionId()) {
LOG.trace("No compactions found for queue {} on tserver {}, trying
next tserver", queue,
- tserver.getHostAndPort(), compactorAddress);
+ tserver.getHostAndPort());
QUEUE_SUMMARIES.removeSummary(tserver, queue, prioTserver.prio);
prioTserver = QUEUE_SUMMARIES.getNextTserver(queue);
diff --git a/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
b/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
index 82e91db4cf..b02515e3d0 100644
--- a/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
@@ -78,6 +78,7 @@ public class AuditMessageIT extends ConfigurableMacBase {
private static final String AUDIT_USER_2 = "AuditUser2";
private static final String PASSWORD = "password";
private static final String OLD_TEST_TABLE_NAME = "apples";
+ private static final String OLD_TEST_TABLE_NAME_ID = "1";
private static final String NEW_TEST_TABLE_NAME = "oranges";
private static final String THIRD_TEST_TABLE_NAME = "pears";
private static final Authorizations auths = new Authorizations("private",
"public");
@@ -361,7 +362,7 @@ public class AuditMessageIT extends ConfigurableMacBase {
assertEquals(1,
findAuditMessage(auditMessages,
String.format(AuditedSecurityOperation.CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE,
-"offlineTable", OLD_TEST_TABLE_NAME)));
+
[accumulo-examples] branch main updated: Update README.md to clarify use of 'main' vs '2.0' branch (#102)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 3d5e01d Update README.md to clarify use of 'main' vs '2.0' branch (#102) 3d5e01d is described below commit 3d5e01d445604a7fc4efaa90c7068bac83ce2919 Author: Mark Owens AuthorDate: Wed Apr 27 10:23:52 2022 -0400 Update README.md to clarify use of 'main' vs '2.0' branch (#102) Added additional information to the README to clarify the distinction between the 'main' branch and additional branches. Indicate use of 'main' branch as examples for current development version of Accumulo while each release of Accumulo will have its own separate branch. Also, added link to 1.10 examples. Closes #80 --- README.md | 10 ++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 98d044a..7637ce7 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,15 @@ limitations under the License. [![Build Status][ti]][tl] +## Introduction + +The Accumulo-Examples repository contains a collection of examples for Accumulo versions 2.0 and +greater. Examples within the `main` branch are designed to work with the version currently +under development. Additional branches exist for previous releases of the Accumulo 2.x line. For +example, the `2.0` branch contains examples specifically intended to work with that release version. + +A collection of examples for Accumulo 1.10 can be found [here]. + ## Setup instructions Follow the steps below to run the Accumulo examples: @@ -132,3 +141,4 @@ This repository can be used to test Accumulo release candidates. See [wordcount]: docs/wordcount.md [ti]: https://github.com/apache/accumulo-examples/workflows/QA/badge.svg [tl]: https://github.com/apache/accumulo-examples/actions +[here]: https://accumulo.apache.org/1.10/examples
[accumulo] branch main updated: Update ChangeSecret to work as documentation states (#2651)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1908f066fd Update ChangeSecret to work as documentation states (#2651)
1908f066fd is described below
commit 1908f066fd066096f9e26e8e4a4a8b7494014196
Author: Mark Owens
AuthorDate: Wed Apr 27 07:06:18 2022 -0400
Update ChangeSecret to work as documentation states (#2651)
* Update ChangeSecret to work as documentation states
There are inconsistencies in how ChangeSecret should be run, as well as
exceptions thrown when attempting to execute ChangeSecret.
In two locations, instructions for executing ChangeSecret indicate
./bin/accumulo org.apache.accumulo.server.util.ChangeSecret
But within accumulo-site.xml, instructions indicate
./bin/accumulo org.apache.accumulo.server.util.ChangeSecret
[oldpasswd] [newpasswd]
Neither one of the two methods work properly.
Running with no parameters throws a NullPointerException. Running with
the indicated parameters presents a usage message with an error
message provided. Additionally, the usage message makes no mention of
the old and new password parameters.
This PR updates site.xml to match the instructions within the rest of
the code, i.e., execute the command with no parameters.
It also updates the method by which the old and new parameters
are collected. This allowed the `Opts` class to be removed along
with other unneeded code.
The use of `password` is replaced with the string `secret` to
make the intent more clear.
Co-authored-by: Christopher Tubbs
---
.../apache/accumulo/server/util/ChangeSecret.java | 30 ++
.../src/test/resources/conf/accumulo-site.xml | 7 +++--
2 files changed, 11 insertions(+), 26 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index 560fe061e7..127720a644 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -38,6 +38,7 @@ import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.ServerDirs;
import org.apache.accumulo.server.cli.ServerUtilOpts;
import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -48,48 +49,33 @@ import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
-import com.beust.jcommander.Parameter;
-
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.context.Scope;
public class ChangeSecret {
- static class Opts extends ServerUtilOpts {
-@Parameter(names = "--old", description = "old zookeeper password",
password = true,
-hidden = true)
-String oldPass;
-@Parameter(names = "--new", description = "new zookeeper password",
password = true,
-hidden = true)
-String newPass;
- }
-
public static void main(String[] args) throws Exception {
var siteConfig = SiteConfiguration.auto();
var hadoopConf = new Configuration();
-Opts opts = new Opts();
+ServerUtilOpts opts = new ServerUtilOpts();
ServerContext context = opts.getServerContext();
try (var fs = context.getVolumeManager()) {
ServerDirs serverDirs = new ServerDirs(siteConfig, hadoopConf);
verifyHdfsWritePermission(serverDirs, fs);
- List argsList = new ArrayList<>(args.length + 2);
- argsList.add("--old");
- argsList.add("--new");
- argsList.addAll(Arrays.asList(args));
-
- opts.parseArgs(ChangeSecret.class.getName(), args);
+ String oldPass = String.valueOf(System.console().readPassword("Old
secret: "));
+ String newPass = String.valueOf(System.console().readPassword("New
secret: "));
Span span = TraceUtil.startSpan(ChangeSecret.class, "main");
try (Scope scope = span.makeCurrent()) {
-verifyAccumuloIsDown(context, opts.oldPass);
+verifyAccumuloIsDown(context, oldPass);
final InstanceId newInstanceId = InstanceId.of(UUID.randomUUID());
updateHdfs(serverDirs, fs, newInstanceId);
-rewriteZooKeeperInstance(context, newInstanceId, opts.oldPass,
opts.newPass);
-if (opts.oldPass != null) {
- deleteInstance(context, opts.oldPass);
+rewriteZooKeeperInstance(context, newInstanceId, oldPass, newPass);
+if (!StringUtils.isBlank
[accumulo-examples] branch main updated: Update documentation for Sample and Shard examples (#101)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new b34ed03 Update documentation for Sample and Shard examples (#101) b34ed03 is described below commit b34ed034d339dc8c5d6f8a77cd8c4b4371f22f83 Author: Mark Owens AuthorDate: Mon Apr 11 14:24:11 2022 -0400 Update documentation for Sample and Shard examples (#101) Update Sample.md to indicate shard example should be run before proceeding with sampling portion of the shard example. Added namespace to name of shard table in Sample.md. Modified output of shard.Query output from Shard example to reflect recent changes in source code. --- docs/sample.md | 8 ++-- docs/shard.md | 28 +++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/sample.md b/docs/sample.md index bdf5be6..33c9e81 100644 --- a/docs/sample.md +++ b/docs/sample.md @@ -149,8 +149,10 @@ configuration for sample scan to work. Shard Sampling Example -- +Note: Before continuing, you need to complete the Shard example, located [here][shard]. + The Shard example shows how to index and search files using Accumulo. That -example indexes documents into a table named `shard`. The indexing scheme used +example indexes documents into a table named `examples.shard`. The indexing scheme used in that example places the document name in the column qualifier. A useful sample of this indexing scheme should contain all data for any document in the sample. To accomplish this, the following commands build a sample for the @@ -184,9 +186,11 @@ Another way sample data could be used with the shard example is with a specialized iterator. In the examples source code there is an iterator named CutoffIntersectingIterator. This iterator first checks how many documents are found in the sample data. If too many documents are found in the sample data, -then it returns nothing. Otherwise it proceeds to query the full data set. +then it returns nothing. Otherwise, it proceeds to query the full data set. To experiment with this iterator, use the following command. The `--sampleCutoff` option below will cause the query to return nothing if based on the sample it appears a query would return more than 1000 documents. $ ./bin/runex shard.Query --sampleCutoff 1000 -t examples.shard import int | fgrep '.java' | wc + +[shard]: shard.md diff --git a/docs/shard.md b/docs/shard.md index 97a9d40..d779c6f 100644 --- a/docs/shard.md +++ b/docs/shard.md @@ -37,11 +37,29 @@ After creating the tables, index some files. The following command indexes all o The following command queries the index to find all files containing 'foo' and 'bar'. $ ./bin/runex shard.Query -t examples.shard foo bar - /local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/security/ColumnVisibilityTest.java - /local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/client/mock/MockConnectorTest.java - /local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/security/VisibilityEvaluatorTest.java - /local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/data/KeyExtentTest.java - /local/username/workspace/accumulo/src/core/src/test/java/accumulo/core/iterators/WholeRowIteratorTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/client/admin/NewTableConfigurationTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/data/KeyExtentTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/iterators/user/WholeRowIteratorTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/iterators/user/WholeColumnFamilyIteratorTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/data/KeyBuilderTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/security/ColumnVisibilityTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/conf/IterConfigUtilTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/clientImpl/TableOperationsHelperTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/clientImpl/mapreduce/BatchInputSplitTest.java + /path/to/accumulo/core/src/test/java/org/apache/accumulo/core/spi/balancer/HostRegexTableLoadBalancerReconfigurationTest.java + /
[accumulo-examples] branch main updated (30e613d -> bb4a118)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git from 30e613d Remove ACCUMULO_JAVA_OPTS in favor of JAVA_TOOL_OPTIONS (#98) add bb4a118 Update POM to use newer version of enforcer plugin (#100) No new revisions were added by this update. Summary of changes: pom.xml | 5 + 1 file changed, 5 insertions(+)
[accumulo] branch main updated: Update HostAndPortTest methods (#2559)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 660ed07 Update HostAndPortTest methods (#2559)
660ed07 is described below
commit 660ed07e2224c2bce3a6841970c17e83e6640da7
Author: Mark Owens
AuthorDate: Wed Mar 9 15:48:17 2022 -0500
Update HostAndPortTest methods (#2559)
Update the test methods for the HostAndPort class.
testCompareTo() contained an unused variable, hostPortSame, that is now
used.
testOrder() was updated to make use of the Streams API vs. individually
adding each test IP line by line.
---
.../apache/accumulo/core/util/HostAndPortTest.java | 75 +++---
1 file changed, 22 insertions(+), 53 deletions(-)
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
b/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
index c9483cd..d44e4de 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
@@ -24,6 +24,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
@@ -36,7 +38,7 @@ class HostAndPortTest {
assertTrue(hostAndPort1.compareTo(hostAndPort2) > 0);
HostAndPort hostPortSame = HostAndPort.fromString("www.test.com");
-assertTrue(hostAndPort1.compareTo(hostAndPort1) == 0);
+assertTrue(hostPortSame.compareTo(hostPortSame) == 0);
hostAndPort1 = HostAndPort.fromString("www.example.com");
hostAndPort2 = HostAndPort.fromString("www.example.com");
@@ -82,59 +84,26 @@ class HostAndPortTest {
@Test
void testOrder() {
-Set hostPortSet = new TreeSet<>();
-hostPortSet.add(HostAndPort.fromString("example.info"));
-hostPortSet.add(HostAndPort.fromString("192.12.2.1:80"));
-hostPortSet.add(HostAndPort.fromString("example.com:80"));
-hostPortSet.add(HostAndPort.fromString("a.bb.c.d"));
-hostPortSet.add(HostAndPort.fromString("12.1.2.1"));
-hostPortSet.add(HostAndPort.fromString("localhost:090"));
-hostPortSet.add(HostAndPort.fromString("example.com"));
-hostPortSet.add(HostAndPort.fromString("100.100.100.100"));
-hostPortSet.add(HostAndPort.fromString("www.example.com"));
-hostPortSet.add(HostAndPort.fromString("[2001:eb8::1]"));
-hostPortSet.add(HostAndPort.fromString("localhost:90"));
-hostPortSet.add(HostAndPort.fromString("[2001:eb8::1]:80"));
-hostPortSet.add(HostAndPort.fromString("2001:db8::1"));
-hostPortSet.add(HostAndPort.fromString("100.100.101.100"));
-hostPortSet.add(HostAndPort.fromString("2001:::1"));
-hostPortSet.add(HostAndPort.fromString("192.12.2.1"));
-hostPortSet.add(HostAndPort.fromString("192.12.2.1:81"));
-hostPortSet.add(HostAndPort.fromString("199.10.1.1:14"));
-hostPortSet.add(HostAndPort.fromString("10.100.100.100"));
-hostPortSet.add(HostAndPort.fromString("2.2.2.2:1"));
-hostPortSet.add(HostAndPort.fromString("192.12.2.1:79"));
-hostPortSet.add(HostAndPort.fromString("1.1.1.1:24"));
-hostPortSet.add(HostAndPort.fromParts("localhost", 01));
-hostPortSet.add(HostAndPort.fromString("1.1.1.1"));
-hostPortSet.add(HostAndPort.fromString("192.12.2.1:79"));
-hostPortSet.add(HostAndPort.fromString("a.b.c.d"));
-hostPortSet.add(HostAndPort.fromString("1.100.100.100"));
-hostPortSet.add(HostAndPort.fromString("2.2.2.2:"));
+Set hostPortSet = Stream
+.of("example.info", "192.12.2.1:80", "example.com:80", "a.bb.c.d",
"12.1.2.1",
+"localhost:090", "example.com", "100.100.100.100",
"www.example.com",
+"[2001:eb8::1]", "localhost:90", "[2001:eb8::1]:80",
"2001:db8::1", "100.100.101.100",
+"2001:::1", "192.12.2.1", "192.12.2.1:81", "199.10.1.1:14",
"10.100.100.100",
+"2.2.2.2:1", "192.12.2.1:79", "1.1.1.1:24", "1.1.1.1",
"192.12.2.1:79", "a.b.c.d",
+"1.100.100.100", "2.2.2.2:", "a.b.b.d", "www.example.com",
"www.alpha.org",
+
[accumulo] branch main updated (97af145 -> 6242c1d)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 97af145 Add compareTo method to HostAndPort class (#2551) add 6242c1d Create static private final Comparator for HostAndPort (#2556) No new revisions were added by this update. Summary of changes: core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java | 8 1 file changed, 4 insertions(+), 4 deletions(-)
[accumulo] branch main updated: Add compareTo method to HostAndPort class (#2551)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 97af145 Add compareTo method to HostAndPort class (#2551)
97af145 is described below
commit 97af14509d2755de29e2c0196201018f576cab68
Author: Mark Owens
AuthorDate: Wed Mar 9 12:41:12 2022 -0500
Add compareTo method to HostAndPort class (#2551)
The utility class, VerifyTabletAssignements, uses a TreeMap collection
to hold HostAndPort values. This requires the HostAndPort class to
implement the Comparable interface. This ticket adds the compareTo
method to HostAndPort. Given that the utility is seldom used, the
compareTo uses a simple String compare on the Host portion and an
integer compare on the Port value.
The new method uses Comparator.nullsFirst to deal with cases where the
parameter value is null. The HostAndPort value if prevented from being null
with an
Objects.requireNonNull statement in the fromString method.
A HostAndPortTest class was added to verify changes.
This was first found when running ErrorProne. The ErrorProne check for
this condition had been disabled due to this issue. The check has now
been re-enabled in the ErrorProne configuration within the POM file.
Co-authored-by: Brian Loss
---
.../org/apache/accumulo/core/util/HostAndPort.java | 15 ++-
.../apache/accumulo/core/util/HostAndPortTest.java | 147 +
pom.xml| 5 +-
3 files changed, 163 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
b/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
index a7affa5..875bcee 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
@@ -19,6 +19,7 @@ import static
com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import java.io.Serializable;
+import java.util.Comparator;
import java.util.Objects;
/**
@@ -59,7 +60,7 @@ import java.util.Objects;
* colons, and port numbers. Full validation of the host field (if desired) is
the caller's
* responsibility.
*/
-public final class HostAndPort implements Serializable {
+public final class HostAndPort implements Serializable,
Comparable {
/** Magic value indicating the absence of a port number. */
private static final int NO_PORT = -1;
@@ -281,4 +282,16 @@ public final class HostAndPort implements Serializable {
return hasPort() ? port : defaultPort;
}
+ /**
+ * HostAndPort must implement compareTo. This method orders HostAndPort
values using a String
+ * compare on the Host value with a secondary integer compare on the Port
value.
+ */
+ @Override
+ public int compareTo(HostAndPort other) {
+return Comparator
+.nullsFirst(
+Comparator.comparing(HostAndPort::getHost).thenComparingInt(h ->
h.getPortOrDefault(0)))
+.compare(this, other);
+ }
+
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
b/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
new file mode 100644
index 000..c9483cd
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/util/HostAndPortTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.junit.jupiter.api.Test;
+
+class HostAndPortTest {
+
+ @Test
+ void testCompareTo() {
+HostAndPort hostAndPort1 = HostAndPort.fromString("example.info");
+HostAndPort hostAndPort2 = HostAndPort.fromString("example.com");
+assertTrue(hostAndPort1.compareTo(hos
[accumulo] branch main updated: Fate op defer time is not used (#2539)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new edbae8e Fate op defer time is not used (#2539)
edbae8e is described below
commit edbae8e32cf1008ab898490dc00e776a56df09ab
Author: Mark Owens
AuthorDate: Thu Mar 3 07:46:07 2022 -0500
Fate op defer time is not used (#2539)
A comment was added to Fate.java to indicate how the deferTime variable
is being used. It clarifies that the value is used in the manner of an
exit code for the particular if/else loop after the comment.
But it also makes clear that the value of deferTime is used as a wait
value later on within the ZooStore class.
No code was modified with this ticket.
Co-authored-by: Christopher Tubbs
---
core/src/main/java/org/apache/accumulo/fate/Fate.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/core/src/main/java/org/apache/accumulo/fate/Fate.java
b/core/src/main/java/org/apache/accumulo/fate/Fate.java
index eeebb04..0ca8a0a 100644
--- a/core/src/main/java/org/apache/accumulo/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/fate/Fate.java
@@ -81,6 +81,9 @@ public class Fate {
try {
deferTime = op.isReady(tid, environment);
+ // Here, deferTime is only used to determine success (zero) or
failure (non-zero),
+ // proceeding on success and returning to the while loop on
failure.
+ // The value of deferTime is only used as a wait time in
ZooStore.unreserve
if (deferTime == 0) {
prevOp = op;
if (status == TStatus.SUBMITTED) {
[accumulo] branch main updated: Add two missing override annotations (#2526)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new b63e51d Add two missing override annotations (#2526)
b63e51d is described below
commit b63e51de09db8518302f88705d9b1eb7c503a6e6
Author: Mark Owens
AuthorDate: Thu Feb 24 17:14:14 2022 -0500
Add two missing override annotations (#2526)
Add two missing override annotations within the
MiniAccumuloClusterControl class.
---
.../org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index 6d9afd2..d0a580c 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@ -143,6 +143,7 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
}
}
+ @Override
public synchronized void startCoordinator(Class coordinator)
throws IOException {
if (coordinatorProcess == null) {
@@ -162,6 +163,7 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
}
}
+ @Override
public synchronized void startCompactors(Class
compactor, int limit,
String queueName) throws IOException {
synchronized (compactorProcesses) {
[accumulo] branch main updated: Add errorpone profile to main Accumulo pom. (#2447)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4dc8d3e Add errorpone profile to main Accumulo pom. (#2447)
4dc8d3e is described below
commit 4dc8d3effc9eeccd514317b3028bc10e1cf71c22
Author: Mark Owens
AuthorDate: Mon Feb 14 10:55:32 2022 -0500
Add errorpone profile to main Accumulo pom. (#2447)
Added a new profile, errorprone, to the pom.xml file. Use of the profile
allows the Google ErrorProne static analysis tool to examine the
Accumulo codebase and flag errors/warnings.
See https://errorprone.info for additional info and
https://errorprone.info/bugpatterns for a list of bug patterns available.
Users withing to use the tool can examine the bug pattern list and add
items of interest specifically to the profile. Any patterns or warnings that
should always be flagged can then be permanently added to the profile
as needed.
Co-authored-by: Christopher Tubbs
Co-authored-by: Dom G.
---
pom.xml | 60
1 file changed, 60 insertions(+)
diff --git a/pom.xml b/pom.xml
index c1e8f00..e1e3ae2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -122,6 +122,7 @@
5.1.0
${project.parent.basedir}/contrib/Eclipse-Accumulo-Codestyle.xml
+2.11.0
false
@@ -1623,5 +1624,64 @@
${reuseForks}
+
+
+ errorprone
+
+
+
+org.apache.maven.plugins
+maven-compiler-plugin
+
+
+-XDcompilePolicy=simple
+
+ -Xplugin:ErrorProne \
+ -XepExcludedPaths:.*/(proto|thrift|generated-sources)/.* \
+ -XepDisableWarningsInGeneratedCode \
+ -XepDisableAllWarnings \
+
+ -Xep:Incomparable:OFF \
+ -Xep:CheckReturnValue:OFF \
+ -Xep:MustBeClosedChecker:OFF \
+ -Xep:ReturnValueIgnored:OFF \
+ -Xep:UnicodeInCode:OFF \
+
+ -Xep:ExpectedExceptionChecker \
+ -Xep:MissingOverride \
+
+
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
+
-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+
-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
+
-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED
+
+
+
+ com.google.errorprone
+ error_prone_core
+ ${errorprone.version}
+
+
+ com.google.auto.service
+ auto-service
+ 1.0
+
+
+
+
+
+
+
[accumulo] branch main updated: Class can be static (#2480)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7088cde Class can be static (#2480)
7088cde is described below
commit 7088cdec4fab5bb590645ab6dc10fb7b2021f50e
Author: Mark Owens
AuthorDate: Thu Feb 10 15:16:40 2022 -0500
Class can be static (#2480)
Several classes were flagged by errorprone as inner classes which could
be made static. Per errorprone justification:
"An inner class should be static unless it references members of its
enclosing class. An inner class that is made non-static unnecessarily
uses more memory and does not make the intent of the class clear."
---
core/src/main/java/org/apache/accumulo/core/file/FileOperations.java | 4 ++--
core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java | 2 +-
.../src/main/java/org/apache/accumulo/server/client/BulkImporter.java | 4 ++--
.../src/main/java/org/apache/accumulo/server/log/WalStateManager.java | 2 +-
server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 2 +-
.../apache/accumulo/tserver/replication/AccumuloReplicaSystem.java| 2 +-
test/src/main/java/org/apache/accumulo/test/rpc/Mocket.java | 2 +-
7 files changed, 9 insertions(+), 9 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
b/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
index 494d433..ce4b678 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
@@ -163,7 +163,7 @@ public abstract class FileOperations {
return new ReaderBuilder();
}
- public class FileOptions {
+ public static class FileOptions {
// objects used by all
public final AccumuloConfiguration tableConfiguration;
public final String filename;
@@ -270,7 +270,7 @@ public abstract class FileOperations {
/**
* Helper class extended by both writers and readers.
*/
- public class FileHelper {
+ public static class FileHelper {
private AccumuloConfiguration tableConfiguration;
private String filename;
private FileSystem fs;
diff --git
a/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
b/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
index 11386a5..b5df0c0 100644
--- a/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
+++ b/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooCache.java
@@ -246,7 +246,7 @@ public class ZooCache {
this.externalWatcher = watcher;
}
- private abstract class ZooRunnable {
+ private abstract static class ZooRunnable {
/**
* Runs an operation against ZooKeeper. Retries are performed by the retry
method when
* KeeperExceptions occur.
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index 131160a..314707d 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -315,7 +315,7 @@ public class BulkImporter {
return Collections.emptySet();
}
- private class AssignmentInfo {
+ private static class AssignmentInfo {
public AssignmentInfo(KeyExtent keyExtent, Long estSize) {
this.ke = keyExtent;
this.estSize = estSize;
@@ -488,7 +488,7 @@ public class BulkImporter {
}
- private class PathSize {
+ private static class PathSize {
public PathSize(Path mapFile, long estSize) {
this.path = mapFile;
this.estSize = estSize;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
index 81aa047..1b20478 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java
@@ -69,7 +69,7 @@ import org.slf4j.LoggerFactory;
*/
public class WalStateManager {
- public class WalMarkerException extends Exception {
+ public static class WalMarkerException extends Exception {
private static final long serialVersionUID = 1L;
public WalMarkerException(Exception ex) {
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index d62f780..6ded0a2 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -182,7 +182,7 @@ public class Monitor extends AbstractServer
[accumulo] branch main updated: Utilize Uninterruptibles.joinUninterruptibly in ServiceLockIT (#2476)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 66ef783 Utilize Uninterruptibles.joinUninterruptibly in ServiceLockIT
(#2476)
66ef783 is described below
commit 66ef78338927d61c2624f10b377e3928527665e8
Author: Mark Owens
AuthorDate: Thu Feb 10 08:52:45 2022 -0500
Utilize Uninterruptibles.joinUninterruptibly in ServiceLockIT (#2476)
Replace t.join within the try/catch of ServiceLockIT with the use of
Uninterruptibles.joinUninterruptibly.
---
.../org/apache/accumulo/test/fate/zookeeper/ServiceLockIT.java | 10 +++---
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ServiceLockIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ServiceLockIT.java
index f6c5238..5a0a569 100644
---
a/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ServiceLockIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/fate/zookeeper/ServiceLockIT.java
@@ -59,6 +59,8 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.util.concurrent.Uninterruptibles;
+
@Category({ZooKeeperTestingServerTests.class})
public class ServiceLockIT {
@@ -622,13 +624,7 @@ public class ServiceLockIT {
workers.forEach(w -> assertNull(w.getException()));
assertEquals(0, zk.getChildren(parent.toString(), false).size());
- threads.forEach(t -> {
-try {
- t.join();
-} catch (InterruptedException e) {
- // ignore
-}
- });
+ threads.forEach(Uninterruptibles::joinUninterruptibly);
}
}
[accumulo] branch main updated (fc611bc -> 1632c8f)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from fc611bc Replace minor cases of UUID with InstanceId, minor checkstyle suggestions (#2471) add 1632c8f Expresion of type int may overflow before assigned to long. (#2477) No new revisions were added by this update. Summary of changes: .../main/java/org/apache/accumulo/core/file/rfile/bcfile/Utils.java | 6 +++--- .../src/main/java/org/apache/accumulo/compactor/Compactor.java | 2 +- .../test/replication/UnorderedWorkAssignerReplicationIT.java| 2 +- 3 files changed, 5 insertions(+), 5 deletions(-)
[accumulo] branch main updated: Remove occurences of unnecessary parentheses (#2460)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new d098125 Remove occurences of unnecessary parentheses (#2460)
d098125 is described below
commit d0981258046480aae778fc5d54aeab55599d62c9
Author: Mark Owens
AuthorDate: Thu Feb 3 12:33:52 2022 -0500
Remove occurences of unnecessary parentheses (#2460)
Remove instances of unneeded parentheses in situations where the code will
not be misinterpreted without them.
---
.../apache/accumulo/core/bloomfilter/BloomFilter.java| 2 +-
.../org/apache/accumulo/core/clientImpl/Credentials.java | 4 ++--
.../accumulo/core/clientImpl/TableOperationsImpl.java| 2 +-
.../java/org/apache/accumulo/core/conf/Property.java | 6 +++---
.../accumulo/core/conf/cluster/ClusterConfigParser.java | 2 +-
.../core/file/blockfile/cache/impl/ClassSize.java| 2 +-
.../org/apache/accumulo/core/file/rfile/CreateEmpty.java | 4 ++--
.../org/apache/accumulo/core/file/rfile/PrintInfo.java | 2 +-
.../org/apache/accumulo/core/iterators/ServerFilter.java | 2 +-
.../core/iterators/SynchronizedServerFilter.java | 2 +-
.../apache/accumulo/core/iterators/user/RegExFilter.java | 16
.../accumulo/core/spi/crypto/AESCryptoService.java | 8
.../main/java/org/apache/accumulo/fate/util/Retry.java | 2 +-
.../apache/accumulo/iteratortest/IteratorTestInput.java | 2 +-
.../java/org/apache/accumulo/server/ServerContext.java | 2 +-
.../accumulo/server/manager/state/TabletServerState.java | 2 +-
.../accumulo/server/security/SecurityOperation.java | 2 +-
.../java/org/apache/accumulo/compactor/Compactor.java| 2 +-
.../accumulo/manager/upgrade/UpgradeCoordinator.java | 2 +-
.../main/java/org/apache/accumulo/tserver/MemValue.java | 2 +-
.../org/apache/accumulo/tserver/ThriftClientHandler.java | 2 +-
.../java/org/apache/accumulo/tserver/tablet/Rate.java| 2 +-
.../java/org/apache/accumulo/shell/ShellCompletor.java | 4 ++--
.../apache/accumulo/shell/commands/ConfigCommand.java| 2 +-
.../apache/accumulo/shell/commands/HistoryCommand.java | 2 +-
.../java/org/apache/accumulo/test/AuditMessageIT.java| 2 +-
.../org/apache/accumulo/test/ConditionalWriterIT.java| 2 +-
.../apache/accumulo/test/NewTableConfigurationIT.java| 2 +-
.../main/java/org/apache/accumulo/test/VerifyIngest.java | 2 +-
.../accumulo/test/fate/zookeeper/ServiceLockIT.java | 2 +-
.../accumulo/test/functional/DeleteRowsSplitIT.java | 2 +-
.../apache/accumulo/test/functional/DurabilityIT.java| 2 +-
.../org/apache/accumulo/test/functional/TabletIT.java| 2 +-
.../java/org/apache/accumulo/test/util/CertUtils.java| 2 +-
.../main/java/org/apache/accumulo/test/util/SlowOps.java | 2 +-
35 files changed, 50 insertions(+), 50 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/bloomfilter/BloomFilter.java
b/core/src/main/java/org/apache/accumulo/core/bloomfilter/BloomFilter.java
index f1cb9d8..a251899 100644
--- a/core/src/main/java/org/apache/accumulo/core/bloomfilter/BloomFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/bloomfilter/BloomFilter.java
@@ -208,7 +208,7 @@ public class BloomFilter extends Filter {
}
if (super.getSerialVersion() == super.getVersion()) {
- ObjectInputStream ois = new ObjectInputStream((DataInputStream) (in));
+ ObjectInputStream ois = new ObjectInputStream((DataInputStream) in);
try {
bits = (BitSet) ois.readObject();
} catch (ClassNotFoundException e) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/Credentials.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/Credentials.java
index 9ee7e28..5958353 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/Credentials.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/Credentials.java
@@ -162,10 +162,10 @@ public class Credentials {
return false;
Credentials other = Credentials.class.cast(obj);
boolean pEq = getPrincipal() == null ? (other.getPrincipal() == null)
-: (getPrincipal().equals(other.getPrincipal()));
+: getPrincipal().equals(other.getPrincipal());
if (!pEq)
return false;
-return getToken() == null ? (other.getToken() == null) :
(getToken().equals(other.getToken()));
+return getToken() == null ? (other.getToken() == null) :
getToken().equals(other.getToken());
}
@Override
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 892805d..949f3bc 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main
[accumulo] branch main updated: Remove self-assignment statement. (#2445)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 2658dea Remove self-assignment statement. (#2445)
2658dea is described below
commit 2658deaf326c58454e260a9b6777ced15ec095dc
Author: Mark Owens
AuthorDate: Tue Feb 1 11:05:44 2022 -0500
Remove self-assignment statement. (#2445)
Removed a self-assignment of the hostPortString variable to itself.
If the hostPortString is non-null, nothing happens and the variable is
unchanged. If the hostPortString is null, a NullPointerException is thrown.
There is no need to assign it to itself.
Change suggested by errorprone analysis.
Added a message to the requireNonNull method indicating the cause of the
NullPointerException.
Also, added java.util.Objects import remove need for fully qualified path
to requireNonNull.
---
core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
b/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
index d6ba8da..a7affa5 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/HostAndPort.java
@@ -19,6 +19,7 @@ import static
com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import java.io.Serializable;
+import java.util.Objects;
/**
* This class was copied from Guava release 23.0 to replace the older Guava 14
version that had been
@@ -143,7 +144,7 @@ public final class HostAndPort implements Serializable {
* if nothing meaningful could be parsed.
*/
public static HostAndPort fromString(String hostPortString) {
-hostPortString = java.util.Objects.requireNonNull(hostPortString);
+Objects.requireNonNull(hostPortString, "hostPortString variable was
null!");
String host;
String portString = null;
boolean hasBracketlessColons = false;
[accumulo] branch main updated: Add some missing @Override annotations. (#2444)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e20aae4 Add some missing @Override annotations. (#2444)
e20aae4 is described below
commit e20aae49c6ae1e68de443b17fb9f2979dfb04744
Author: Mark Owens
AuthorDate: Tue Feb 1 09:01:43 2022 -0500
Add some missing @Override annotations. (#2444)
Added several @Override annotations to six Accumulo classes.
---
.../org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java | 4
.../main/java/org/apache/accumulo/core/constraints/Constraint.java| 3 +++
.../apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java | 1 +
.../org/apache/accumulo/server/conf/codec/VersionedPropGzipCodec.java | 1 +
.../org/apache/accumulo/server/manager/state/MetaDataStateStore.java | 1 +
.../java/org/apache/accumulo/tserver/compactions/PrintableTable.java | 1 +
6 files changed, 11 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java
index 1e45d01..1264fc4 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/CloneConfigurationImpl.java
@@ -53,23 +53,27 @@ public class CloneConfigurationImpl implements
CloneConfiguration, CloneConfigur
public CloneConfigurationImpl() {}
+ @Override
public boolean isFlush() {
Preconditions.checkState(built);
return flush;
}
+ @Override
public Map getPropertiesToSet() {
Preconditions.checkState(built);
return (propertiesToSet == null ? Collections.emptyMap()
: Collections.unmodifiableMap(propertiesToSet));
}
+ @Override
public Set getPropertiesToExclude() {
Preconditions.checkState(built);
return (propertiesToExclude == null ? Collections.emptySet()
: Collections.unmodifiableSet(propertiesToExclude));
}
+ @Override
public boolean isKeepOffline() {
Preconditions.checkState(built);
return keepOffline;
diff --git
a/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
b/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
index b24d613..bad1268 100644
--- a/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
+++ b/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
@@ -52,6 +52,7 @@ public interface Constraint extends
org.apache.accumulo.core.data.constraints.Co
*
* @return user
*/
+@Override
String getUser();
/**
@@ -59,6 +60,7 @@ public interface Constraint extends
org.apache.accumulo.core.data.constraints.Co
*
* @return authorizations
*/
+@Override
AuthorizationContainer getAuthorizationsContainer();
}
@@ -69,6 +71,7 @@ public interface Constraint extends
org.apache.accumulo.core.data.constraints.Co
* numeric violation code
* @return matching violation description
*/
+ @Override
String getViolationDescription(short violationCode);
/**
diff --git
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
index d05244b..28099f2 100644
---
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
+++
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
@@ -42,6 +42,7 @@ public class MapReduceClientOpts extends ClientOpts {
private static final Logger log =
LoggerFactory.getLogger(MapReduceClientOpts.class);
+ @Override
public Properties getClientProps() {
Properties props = super.getClientProps();
// For MapReduce, Kerberos credentials don't make it to the Mappers and
Reducers,
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/codec/VersionedPropGzipCodec.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/codec/VersionedPropGzipCodec.java
index 139469d..2a2ef0c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/codec/VersionedPropGzipCodec.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/codec/VersionedPropGzipCodec.java
@@ -71,6 +71,7 @@ public class VersionedPropGzipCodec extends
VersionedPropCodec {
return encodingOpts.getEncodingVersion() ==
EncodingOptions.EncodingVersion_1_0;
}
+ @Override
Map decodePayload(final InputStream inStream, final
EncodingOptions encodingOpts)
throws IOException {
// read the property map keys, values
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/
[accumulo] branch main updated: Refactor listSplits operation when using maxSplits (#2381)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 83211c2 Refactor listSplits operation when using maxSplits (#2381)
83211c2 is described below
commit 83211c207cc74c30e25433b595026d31752ba2be
Author: Mark Owens
AuthorDate: Mon Dec 13 09:17:35 2021 -0500
Refactor listSplits operation when using maxSplits (#2381)
Refactored listSplits method in TableOperationsImpl. This change affects
the listSplits command which takes maxSplits as an option.
* Renamed variable names to enhance readability.
* Added documentation for the method.
* Replaced while-loop with if-loop after determining while-loop was only
run at most once each time.
* Created IT test for method in ShellIT class.
Closes #2371
Co-authored-by: Keith Turner
---
.../core/clientImpl/TableOperationsImpl.java | 66 --
.../java/org/apache/accumulo/test/ShellIT.java | 23
2 files changed, 71 insertions(+), 18 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index c95910f..202860a 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -689,28 +689,59 @@ public class TableOperationsImpl extends
TableOperationsHelper {
}
+ /**
+ * This version of listSplits is called when the maxSplits options is
provided. If the value of
+ * maxSplits is greater than the number of existing splits, then all splits
are returned and no
+ * additional processing is performed.
+ *
+ * But, if the value of maxSplits is less than the number of existing
splits, maxSplit split
+ * values are returned. These split values are "evenly" selected from the
existing splits based
+ * upon the algorithm implemented in the method.
+ *
+ * A stepSize is calculated based upon the number of splits requested and
the total split count. A
+ * running sum adjusted by this stepSize is calculated as each split is
parsed. Once this sum
+ * exceeds a value of 1, the current split point is selected to be returned.
The sum is then
+ * decremented by 1 and the process continues until all existing splits have
been parsed or
+ * maxSplits splits have been selected.
+ *
+ * @param tableName
+ * the name of the table
+ * @param maxSplits
+ * specifies the maximum number of splits to return
+ * @return a Collection containing a subset of evenly selected splits
+ */
@Override
- public Collection listSplits(String tableName, int maxSplits)
+ public Collection listSplits(final String tableName, final int
maxSplits)
throws TableNotFoundException, AccumuloSecurityException {
// tableName is validated in _listSplits
-List endRows = _listSplits(tableName);
-if (endRows.size() <= maxSplits)
- return endRows;
-
-double r = (maxSplits + 1) / (double) (endRows.size());
-double pos = 0;
-ArrayList subset = new ArrayList<>(maxSplits);
-int j = 0;
-for (int i = 0; i < endRows.size() && j < maxSplits; i++) {
- pos += r;
- while (pos > 1) {
-subset.add(endRows.get(i));
-j++;
-pos -= 1;
- }
+final List existingSplits = _listSplits(tableName);
+
+// As long as maxSplits is equal to or larger than the number of current
splits, the existing
+// splits are returned and no additional processing is necessary.
+if (existingSplits.size() <= maxSplits) {
+ return existingSplits;
}
-return subset;
+// When the number of maxSplits requested is less than the number of
existing splits, the
+// following code populates the splitsSubset list 'evenly' from the
existing splits
+ArrayList splitsSubset = new ArrayList<>(maxSplits);
+final int SELECTION_THRESHOLD = 1;
+
+// stepSize can never be greater than 1 due to the if-loop check above.
+final double stepSize = (maxSplits + 1) / (double) (existingSplits.size());
+double selectionTrigger = 0.0;
+
+for (Text existingSplit : existingSplits) {
+ if (splitsSubset.size() >= maxSplits) {
+break;
+ }
+ selectionTrigger += stepSize;
+ if (selectionTrigger > SELECTION_THRESHOLD) {
+splitsSubset.add(existingSplit);
+selectionTrigger -= 1;
+ }
+}
+return splitsSubset;
}
@Override
@@ -727,7 +758,6 @@ public class TableOperationsImpl extends
TableOperationsHelper {
// should not happen
throw new AssertionError(e);
}
-
}
@Override
diff --git a/test/src/main/java/or
[accumulo] branch 1.10 updated: Fix incorrect scan range output in getsplits command (#2370)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new b94ce9a Fix incorrect scan range output in getsplits command (#2370)
b94ce9a is described below
commit b94ce9ac73328488f8f8dd55dfa77e3795353515
Author: Mark Owens
AuthorDate: Mon Nov 29 13:16:54 2021 -0500
Fix incorrect scan range output in getsplits command (#2370)
In the Accumulo shell, calling getsplits with the verbose option can result
in incorret output. It occurs when the tableId of the table happens to be a
single character and there are other tables where the tableId starts with the
same character. This results in the output of getsplits displaying splits for
the other tables as well.
Added a couple of grammatical fixes based upon @millerruntime suggestions.
Closes #2356
---
.../accumulo/shell/commands/GetSplitsCommand.java | 2 +
.../org/apache/accumulo/test/ShellServerIT.java| 75 +-
2 files changed, 76 insertions(+), 1 deletion(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
index 0eae20b..a89d797 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
@@ -81,6 +81,8 @@ public class GetSplitsCommand extends Command {
final Text start =
new
Text(shellState.getConnector().tableOperations().tableIdMap().get(tableName));
final Text end = new Text(start);
+// do not append the ';' until the end value above is assigned.
+start.append(new byte[] {';'}, 0, 1);
end.append(new byte[] {'<'}, 0, 1);
scanner.setRange(new Range(start, end));
for (Iterator> iterator = scanner.iterator();
iterator.hasNext();) {
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
index 8f009f3..5c874ab 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java
@@ -327,7 +327,7 @@ public class ShellServerIT extends SharedMiniClusterBase {
@Override
public int defaultTimeoutSeconds() {
-return 60;
+return 180;
}
@Test
@@ -2063,4 +2063,77 @@ public class ShellServerIT extends SharedMiniClusterBase
{
return null;
}
+ // This test addresses a bug (#2356) where if a table with a tableId of
character length 1
+ // exists and another table(s) exist starting with the same character but
with a tableId of
+ // length > 1, the verbose version of the getsplits command will return
information from multiple
+ // tables when a lexicographical ordered table with an earlier ID is queried.
+ //
+ // In order to test, enough tables need to be created until the required
condition exists.
+ // Since table ID counts increment using 1..9a..z, this test creates tables
in groups of 36
+ // until a second table meeting the criteria is created.
+ // It then adds splits to the single digit ID table and one of the others.
It performs a
+ // "getsplits -v" command on the single digit ID table and verifies that
data from the other
+ // table is not present.
+ //
+ // Due to the number for createtable calls, this test will time out if a
match is not found
+ // within some number of operations. Therefore, if a match is not found
within the creation
+ // of the first 360 or so tables, the test exits with no results. In initial
runs of the ITs
+ // this never occurred.
+ @Test
+ public void testGetSplitsScanRange() throws Exception {
+Shell.log.debug("Starting testGetSplitsScanRange test --");
+int idCycleLen = 36;
+int maxLoopcnt = 10;
+int postModifier = 0;
+int loopCnt = 0;
+
+String[] tables = new String[2];
+Connector conn = getCluster().getConnector(getPrincipal(), getToken());
+while (loopCnt++ < maxLoopcnt) {
+ Map idMap = conn.tableOperations().tableIdMap();
+ if (findIds(tables, idMap)) {
+break;
+ }
+ createTables(idCycleLen, postModifier++);
+}
+if (loopCnt >= maxLoopcnt) {
+ Shell.log.warn("Warning: Unable to find needed tables...exiting test
without verifying.");
+ return;
+}
+// add splits to the two tables
+ts.exec("addsplits -t " + tables[0] + " a c e", true);
+ts.exec("addsplits -t " + tables[1] + " g i t", true);
+// first table should contain supplied string
+ts.exec("getsplits -v -t " + tables[0], true, "(e, +inf) Default Tablet",
[accumulo] branch main updated: Fix incorrect scan range output in getsplits command (#2369)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7e97566 Fix incorrect scan range output in getsplits command (#2369)
7e97566 is described below
commit 7e97566a8b1f0b464753d9e7099afb1bdfa5b043
Author: Mark Owens
AuthorDate: Mon Nov 29 12:44:45 2021 -0500
Fix incorrect scan range output in getsplits command (#2369)
In the Accumulo shell, calling getsplits with the verbose option can result
in incorret output. It occurs when the tableId of the table happens to be a
single character and there are other tables where the tableId starts with the
same character. This results in the output of getsplits displaying splits for
the other tables as well.
Closes #2356
Co-authored-by: Mike Miller
---
.../accumulo/shell/commands/GetSplitsCommand.java | 2 +
.../java/org/apache/accumulo/test/ShellIT.java | 76 +-
2 files changed, 77 insertions(+), 1 deletion(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
index 64ab003..79e9746 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/GetSplitsCommand.java
@@ -72,6 +72,8 @@ public class GetSplitsCommand extends Command {
final Text start =
new
Text(shellState.getAccumuloClient().tableOperations().tableIdMap().get(tableName));
final Text end = new Text(start);
+// do not append the ';' until the end value above is assigned.
+start.append(new byte[] {';'}, 0, 1);
end.append(new byte[] {'<'}, 0, 1);
scanner.setRange(new Range(start, end));
for (final Entry next : scanner) {
diff --git a/test/src/main/java/org/apache/accumulo/test/ShellIT.java
b/test/src/main/java/org/apache/accumulo/test/ShellIT.java
index 139247a..0136f6c 100644
--- a/test/src/main/java/org/apache/accumulo/test/ShellIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ShellIT.java
@@ -31,6 +31,7 @@ import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.TimeZone;
import org.apache.accumulo.harness.SharedMiniClusterBase;
@@ -56,7 +57,7 @@ public class ShellIT extends SharedMiniClusterBase {
@Override
protected int defaultTimeoutSeconds() {
-return 30;
+return 180;
}
@BeforeClass
@@ -515,4 +516,77 @@ public class ShellIT extends SharedMiniClusterBase {
exec("deletetable t -f", true, "Table: [t] has been deleted");
}
+
+ // This test addresses a bug (#2356) where if a table with a tableId of
character length 1
+ // exists and another table(s) exist starting with the same character but
with a tableId of
+ // length > 1, the verbose version of the getsplits command will return
information from multiple
+ // tables when a lexicographical ordered table with an earlier ID is queried.
+ //
+ // In order to test, enough tables need to be created until the required
condition exists.
+ // Since table ID counts increment using 1..9a..z, this test creates tables
in groups of 36
+ // until a second table meeting the criteria is created.
+ // It then adds splits to the single digit ID table and one of the others.
It performs a
+ // "getsplits -v" command on the single digit ID table and verifies that
data from the other
+ // table is not present.
+ //
+ // Due to the number for createtable calls, this test will time out if a
match is not found
+ // within some number of operations. Therefore, if a match is not found
within the creation
+ // of the first 360 or so tables, the test exits with no results. In initial
runs of the ITs
+ // this never occurred.
+ @Test
+ public void testGetSplitsScanRange() throws Exception {
+Shell.log.debug("Starting testGetSplitsScanRange test --");
+int idCycleLen = 36;
+int postModifier = 0;
+int maxLoopcnt = 10;
+int loopCnt = 0;
+
+String[] tables = new String[2];
+while (loopCnt++ < maxLoopcnt) {
+ Map idMap =
shell.getAccumuloClient().tableOperations().tableIdMap();
+ if (findIds(tables, idMap)) {
+break;
+ }
+ createTables(idCycleLen, postModifier++);
+}
+if (loopCnt >= maxLoopcnt) {
+ Shell.log.warn("Warning: Unable to find needed tables...exiting test
without verifying.");
+ return;
+}
+// add splits to the two tables
+exec("addsplits -t " + tables[0] + " a c e", true);
+exec("addsplits -t " + tables[1] + " g i t", true);
[accumulo] branch main updated: Add missing format argument. (#2246)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new e55b961 Add missing format argument. (#2246)
e55b961 is described below
commit e55b961c10bde3c985a2fefde16965ab7562417e
Author: Mark Owens
AuthorDate: Wed Aug 25 14:36:51 2021 -0400
Add missing format argument. (#2246)
Add missing argument to log statement. The format call references one
argument but no argument was supplied.
Close #2246
---
.../src/main/java/org/apache/accumulo/compactor/Compactor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index b63e1bd..c72e286 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -212,7 +212,7 @@ public class Compactor extends AbstractServer implements
CompactorService.Iface
var cancelId = Long.parseLong(new String(id, UTF_8));
if (cancelId >= job.getUserCompactionId()) {
- LOG.info("Cancelling compaction {} because user compaction was
canceled");
+ LOG.info("Cancelling compaction {} because user compaction was
canceled", ecid);
JOB_HOLDER.cancel(job.getExternalCompactionId());
return;
}
[accumulo] branch main updated: Update trace code in GarbageCollectionAlgorithm (#2216)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 93bafa8 Update trace code in GarbageCollectionAlgorithm (#2216)
93bafa8 is described below
commit 93bafa8f96d211ead5a09535d6d5aaf32145065d
Author: Mark Owens
AuthorDate: Mon Aug 2 08:26:07 2021 -0400
Update trace code in GarbageCollectionAlgorithm (#2216)
Update location of TraceScope code in GarbageCollectionAlgorithm. With the
GCA code now using an iterator rather tnan continue point, the seperate call to
getCandidates with GCA is no longer needed as its only purpose purpose was to
allow the GCE method to be traced. That method was removed. The trace was
instead placed around the call to readCandidatesThatFitInMemory as that is the
method responsible for collecting and batching up deletion candidates.
---
.../apache/accumulo/gc/GarbageCollectionAlgorithm.java| 15 +--
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 643f46c..12152fc 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@ -264,14 +264,6 @@ public class GarbageCollectionAlgorithm {
for (TableId delTableId : tableIdsWithDeletes) {
gce.deleteTableDirIfEmpty(delTableId);
}
-
- }
-
- private Iterator getCandidates(GarbageCollectionEnvironment gce)
- throws TableNotFoundException, IOException {
-try (TraceScope candidatesSpan = Trace.startSpan("getCandidates")) {
- return gce.getCandidates();
-}
}
private void confirmDeletesTrace(GarbageCollectionEnvironment gce,
@@ -292,10 +284,13 @@ public class GarbageCollectionAlgorithm {
public void collect(GarbageCollectionEnvironment gce) throws
TableNotFoundException, IOException {
-Iterator candidatesIter = getCandidates(gce);
+Iterator candidatesIter = gce.getCandidates();
while (candidatesIter.hasNext()) {
- List batchOfCandidates =
gce.readCandidatesThatFitInMemory(candidatesIter);
+ List batchOfCandidates;
+ try (TraceScope candidatesSpan = Trace.startSpan("getCandidates")) {
+batchOfCandidates = gce.readCandidatesThatFitInMemory(candidatesIter);
+ }
deleteBatch(gce, batchOfCandidates);
}
}
[accumulo] branch main updated: Remove continue point from Garbage Collector (#2214)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 485114a Remove continue point from Garbage Collector (#2214)
485114a is described below
commit 485114a4760028e70d2eb4ed23b795ed386fbaa1
Author: Mark Owens
AuthorDate: Thu Jul 29 08:07:56 2021 -0400
Remove continue point from Garbage Collector (#2214)
Updated Garbage Collection code to no longer use a continue point when
processing deletion candidates. The GC now uses an iterator that lasts during
the lifetime of a GC cycle.
The GarbageCollectionTest was updated to work with the update, as was the
GC integration test.
Closes #1351
---
.../accumulo/core/metadata/schema/Ample.java | 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 14 +--
.../accumulo/server/util/ListVolumesUsed.java | 2 +-
.../accumulo/gc/GarbageCollectionAlgorithm.java| 41 +++---
.../accumulo/gc/GarbageCollectionEnvironment.java | 25 ++--
.../apache/accumulo/gc/SimpleGarbageCollector.java | 22 ++--
.../apache/accumulo/gc/GarbageCollectionTest.java | 139 -
.../test/functional/GarbageCollectorIT.java| 3 +
8 files changed, 185 insertions(+), 63 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index fd5634e..18bc12a 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -198,7 +198,7 @@ public interface Ample {
throw new UnsupportedOperationException();
}
- default Iterator getGcCandidates(DataLevel level, String
continuePoint) {
+ default Iterator getGcCandidates(DataLevel level) {
throw new UnsupportedOperationException();
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index 1f7eefe..5ef7237 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -32,9 +32,7 @@ import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -163,7 +161,7 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
}
@Override
- public Iterator getGcCandidates(DataLevel level, String
continuePoint) {
+ public Iterator getGcCandidates(DataLevel level) {
if (level == DataLevel.ROOT) {
var zooReader = new ZooReader(context.getZooKeepers(),
context.getZooKeepersSessionTimeOut());
byte[] json;
@@ -173,19 +171,9 @@ public class ServerAmpleImpl extends AmpleImpl implements
Ample {
throw new RuntimeException(e);
}
Stream candidates =
RootGcCandidates.fromJson(json).stream().sorted();
-
- if (continuePoint != null && !continuePoint.isEmpty()) {
-candidates = candidates.dropWhile(candidate ->
candidate.compareTo(continuePoint) <= 0);
- }
-
return candidates.iterator();
} else if (level == DataLevel.METADATA || level == DataLevel.USER) {
Range range = DeletesSection.getRange();
- if (continuePoint != null && !continuePoint.isEmpty()) {
-String continueRow = DeletesSection.encodeRow(continuePoint);
-range = new Range(new Key(continueRow).followingKey(PartialKey.ROW),
true,
-range.getEndKey(), range.isEndKeyInclusive());
- }
Scanner scanner;
try {
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
index 3eb8c80..212822f 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
@@ -78,7 +78,7 @@ public class ListVolumesUsed {
+ " deletes section (volume replacement occurs at deletion time)");
volumes.clear();
-Iterator delPaths = context.getAmple().getGcCandidates(level, "");
+Iterator delPaths = context.getAmple().getGcCandidates(level);
while (delPaths.hasNext()) {
[accumulo] branch main updated (196fda5 -> f906510)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 196fda5 Add Compactor and Coordinator to testExpectedClasses in KeywordStartIT (#2139) add f906510 Test converting client props to ConditionalWriterConfig (#2135) No new revisions were added by this update. Summary of changes: .../accumulo/core/clientImpl/ClientContext.java| 33 +++-- .../core/clientImpl/ClientContextTest.java | 55 ++ 2 files changed, 74 insertions(+), 14 deletions(-)
[accumulo] branch main updated (8a636a3 -> c11ee7b)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 8a636a3 External Compactions feature (#1451) add c11ee7b Unit test converting client props to BatchWriterConfig (#2126) No new revisions were added by this update. Summary of changes: .../accumulo/core/clientImpl/ClientContext.java| 49 +++-- .../core/clientImpl/ClientContextTest.java | 80 ++ 2 files changed, 107 insertions(+), 22 deletions(-)
[accumulo] branch main updated: Make LogReader more discoverable (#2106)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1a2ef83 Make LogReader more discoverable (#2106)
1a2ef83 is described below
commit 1a2ef83cd7159b78f33377d917799c6ad299c9e2
Author: Mark Owens
AuthorDate: Thu May 13 12:44:42 2021 -0400
Make LogReader more discoverable (#2106)
Use the KeywordExecutable service to make wal-info (the WAL LogReader
class) available as part of the Accumulo command line. Can be used by running:
accumulo wal-info
Closes #2100
---
.../apache/accumulo/tserver/logger/LogReader.java | 40 +-
.../apache/accumulo/test/start/KeywordStartIT.java | 3 ++
2 files changed, 34 insertions(+), 9 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
index 17415bc..5c0ce29 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java
@@ -22,7 +22,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.DataInputStream;
import java.io.EOFException;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -37,6 +36,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.accumulo.tserver.log.DfsLogger;
import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException;
import org.apache.accumulo.tserver.log.RecoveryLogReader;
@@ -47,10 +47,14 @@ import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+@AutoService(KeywordExecutable.class)
+public class LogReader implements KeywordExecutable {
-public class LogReader {
private static final Logger log = LoggerFactory.getLogger(LogReader.class);
static class Opts extends Help {
@@ -73,19 +77,37 @@ public class LogReader {
* @param args
* - first argument is the file to print
*/
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws Exception {
+new LogReader().execute(args);
+ }
+
+ @Override
+ public String keyword() {
+return "wal-info";
+ }
+
+ @Override
+ public String description() {
+return "Prints WAL Info";
+ }
+
+ @SuppressFBWarnings(value = "DM_EXIT",
+ justification = "System.exit is fine here because it's a utility class
executed by a main()")
+ @Override
+ public void execute(String[] args) throws Exception {
Opts opts = new Opts();
-opts.parseArgs(LogReader.class.getName(), args);
+opts.parseArgs("accumulo wal-info", args);
+if (opts.files.isEmpty()) {
+ System.err.println("No WAL files were given");
+ System.exit(1);
+}
+
var siteConfig = SiteConfiguration.auto();
try (var fs = VolumeManagerImpl.get(siteConfig, new Configuration())) {
Matcher rowMatcher = null;
KeyExtent ke = null;
Text row = null;
- if (opts.files.isEmpty()) {
-new JCommander(opts).usage();
-return;
- }
if (opts.row != null) {
row = new Text(opts.row);
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
index 87e8d31..f808ecd 100644
--- a/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/start/KeywordStartIT.java
@@ -60,6 +60,7 @@ import org.apache.accumulo.tracer.TraceServer;
import org.apache.accumulo.tracer.TracerExecutable;
import org.apache.accumulo.tserver.TServerExecutable;
import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.logger.LogReader;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
@@ -115,6 +116,7 @@ public class KeywordStartIT {
expectSet.put("minicluster", MiniClusterExecutable.class);
expectSet.put("monitor", MonitorExecutable.class);
expectSet.put("rfile-info", PrintInfo.class);
+expectSet.put("wal-info", LogReader.class);
expectSet.put("shell", Shell.class);
expectSet.put(&q
[accumulo-examples] branch main updated: Update combiner.md
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 965581b Update combiner.md
965581b is described below
commit 965581ba2070fbf1d2283db4206b9844a892239b
Author: Mark Owens
AuthorDate: Wed May 12 14:30:39 2021 -0400
Update combiner.md
Updated Combiner.md file to:
* Correct a typo
* Include the 'examples' namespace as part of the table name when setting
the iterator.
---
docs/combiner.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/combiner.md b/docs/combiner.md
index 7ee9cd1..76cb5fe 100644
--- a/docs/combiner.md
+++ b/docs/combiner.md
@@ -37,13 +37,13 @@ tar distribution.
-
username@instance> createnamespace examples
username@instance> createtable examples.runners
-username@instance exampoles.runners> setiter -t runners -p 10 -scan -minc
-majc -n decStats -class org.apache.accumulo.examples.combiner.StatsCombiner
+username@instance examples.runners> setiter -t examples.runners -p 10
-scan -minc -majc -n decStats -class
org.apache.accumulo.examples.combiner.StatsCombiner
Combiner that keeps track of min, max, sum, and count
--> set StatsCombiner parameter all, set to true to apply Combiner
to every column, otherwise leave blank. if true, columns option will be
ignored.:
--> set StatsCombiner parameter columns, [:]{,[:]} escape non aplhanum chars using %.: stat
--> set StatsCombiner parameter reduceOnFullCompactionOnly, If
true, only reduce on full major compactions. Defaults to false. :
--> set StatsCombiner parameter radix, radix/base of the numbers:
10
-username@instance examples.runners> setiter -t runners -p 11 -scan -minc
-majc -n hexStats -class org.apache.accumulo.examples.combiner.StatsCombiner
+username@instance examples.runners> setiter -t examples.runners -p 11
-scan -minc -majc -n hexStats -class
org.apache.accumulo.examples.combiner.StatsCombiner
Combiner that keeps track of min, max, sum, and count
--> set StatsCombiner parameter all, set to true to apply Combiner
to every column, otherwise leave blank. if true, columns option will be
ignored.:
--> set StatsCombiner parameter columns, [:]{,[:]} escape non-alphanum chars using %.: hstat
[accumulo] branch main updated: Remove duplicated word in TESTING.md (#2090)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new afc85ae Remove duplicated word in TESTING.md (#2090) afc85ae is described below commit afc85ae53bd0a7f65a5588a64a1ce477b895e179 Author: Mark Owens AuthorDate: Mon May 10 12:07:49 2021 -0400 Remove duplicated word in TESTING.md (#2090) Remove an unneeded instance of the word 'to'. It had inadvertently been typed two times in a row within a sentence. --- TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TESTING.md b/TESTING.md index b9d7f43..0162552 100644 --- a/TESTING.md +++ b/TESTING.md @@ -25,7 +25,7 @@ Unit tests can be run by invoking `mvn package` at the root of the Apache Accumu `test` phase of the [Maven lifecycle][lifecycle]. The `test` phase cannot be run directly, because not all of Accumulo's modules are Java artifacts, and therefore will not be resolvable by their sibling modules until they are created in their `package` phase. To avoid building against stale artifacts from previous builds that may have been published to a -remote server or installed to your local Maven repository, always build with the `package` phase to to run the unit +remote server or installed to your local Maven repository, always build with the `package` phase to run the unit tests. The [maven-surefire-plugin][surefire] is bound to the `test` phase of the Maven lifecycle by default and will run the JUnit tests. To execute the unit tests, simply build the project by running:
[accumulo-examples] branch main updated: Avoid checking Accumulo table exists before creation (#74)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new ef2e04e Avoid checking Accumulo table exists before creation (#74) ef2e04e is described below commit ef2e04e78cc08ce1d1a4b3f67947e5fe1f874c89 Author: Mark Owens AuthorDate: Fri May 7 08:16:44 2021 -0400 Avoid checking Accumulo table exists before creation (#74) The initial focus of this ticket was to remove the check for table existence and just create and catch and then ignore TableExistsException instead. After initial work (see previous comments), it was decided to update the code to work more realistically. Primarily, do not ignore the exceptions and at least alert the user that the table existed prior and let the user decide whether to remove the table and re-run the example. Another concern was the fear of interfering with an existing table on a users system. The primary changes then became to update table creation and provide feedback via logs to situations where prior tables already existed. In order to prevent table name collision, the examples were modified to make use of Accumulo namespaces. The classes and documentation were update to create an 'examples' namespace wherein all the table are created. Along the way several other smaller tweaks were made also. Some of these are listed below. * The process of table creation was re-factored into a Commons class. All examples now use a method in that class to create both the namespace and tablename. A couple of constants used throughout the example classes are defined there as well. * The bloom classes now have a couple of methods that helped to remove some redundant code. * An unneeded import was removed from the CharacterHistogram.java class. * Most of the use of System.out.println was replace with logging instead. * Update SequentialBatchWriter to exit if required table for scanning does not exist. * A majority of the documentation was updated to included the creation of the necessary 'examples' namespace. * The config command was updated to use the table.class.loader.context rather than the deprecated table.classpath.context. * Update the constraints example to work with the new location of the contraints classes in Accumulo. * Update filedata documentation to note that the ChunkCombiner class must be avaiable in the accumulo lib directory or on the classpath somewhere in order to scan the create examples.dataTable. Closes #13 --- .gitignore | 1 + docs/batch.md | 4 +- docs/bloom.md | 14 ++-- docs/bulkIngest.md | 2 +- docs/classpath.md | 27 docs/combiner.md | 29 docs/compactionStrategy.md | 29 docs/constraints.md| 2 - docs/deleteKeyValuePair.md | 57 +--- docs/dirlist.md| 16 ++--- docs/export.md | 37 ++- docs/filedata.md | 13 ++-- docs/filter.md | 41 ++-- docs/helloworld.md | 4 +- docs/isolation.md | 6 +- docs/regex.md | 12 ++-- docs/reservations.md | 9 +-- docs/rgbalancer.md | 19 +++--- docs/rowhash.md| 13 ++-- docs/sample.md | 71 ++-- docs/shard.md | 13 ++-- docs/tabletofile.md| 22 +++ docs/terasort.md | 9 ++- docs/visibility.md | 77 +++--- docs/wordcount.md | 12 ++-- .../java/org/apache/accumulo/spark/CopyPlus5K.java | 18 - .../java/org/apache/accumulo/examples/Common.java | 51 ++ .../accumulo/examples/bloom/BloomBatchScanner.java | 27 .../accumulo/examples/bloom/BloomCommon.java | 13 .../accumulo/examples/bloom/BloomFilters.java | 60 + .../examples/bloom/BloomFiltersNotFound.java | 38 ++- .../examples/client/RandomBatchScanner.java| 29 .../accumulo/examples/client/ReadWriteExample.java | 45 +++-- .../accumulo/examples/client/RowOperations.java| 36 -- .../examples/client/SequentialBatch
[accumulo] branch main updated: Move ignoreEmptyDir opt to ImportOptions interface (#2045)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 35319d4 Move ignoreEmptyDir opt to ImportOptions interface (#2045)
35319d4 is described below
commit 35319d441f699178d5facb1b71d4279e6bdfe3e0
Author: Mark Owens
AuthorDate: Thu Apr 29 11:14:17 2021 -0400
Move ignoreEmptyDir opt to ImportOptions interface (#2045)
Update the importdirectory ignoreEmptyDir option to use the bulk import
fluent API. The ignoreEmptyDir option is moved into the ImportOptions interface
and can be set during the call to the load method.
Added @since 2.1.0 tag to ignoreEmptyDir method.
---
.../core/client/admin/TableOperations.java | 33 +-
.../core/clientImpl/TableOperationsImpl.java | 8 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 13 +
.../shell/commands/ImportDirectoryCommand.java | 4 +--
.../shell/commands/ImportDirectoryCommandTest.java | 8 +++---
.../apache/accumulo/test/functional/BulkIT.java| 19 +
.../apache/accumulo/test/functional/BulkNewIT.java | 17 ---
7 files changed, 48 insertions(+), 54 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index 78b99fe..c6ac0e9 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -717,6 +717,13 @@ public interface TableOperations {
ImportMappingOptions tableTime(boolean value);
/**
+ * Ignores empty bulk import source directory, rather than throwing an
IllegalArgumentException.
+ *
+ * @since 2.1.0
+ */
+ImportMappingOptions ignoreEmptyDir(boolean ignore);
+
+/**
* Loads the files into the table.
*/
void load()
@@ -802,32 +809,6 @@ public interface TableOperations {
* @since 2.0.0
*/
default ImportDestinationArguments importDirectory(String directory) {
-return importDirectory(directory, false);
- }
-
- /**
- * Bulk import the files in a directory into a table. Files can be created
using
- * {@link RFile#newWriter()}.
- *
- * This new method of bulk import examines files in the current process
outside of holding a table
- * lock. The old bulk import method ({@link #importDirectory(String, String,
String, boolean)})
- * examines files on the server side while holding a table read lock. This
version of the method
- * provides a boolean argument which will prevent an exception from being
thrown if the supplied
- * directory contains no files.
- *
- * This API supports adding files to online and offline tables.
- *
- * For example, to bulk import files from the directory 'dir1' into the
table 'table1' use the
- * following code.
- *
- *
- * client.tableOperations().importDirectory("dir1",
true).to("table1").load();
- *
- *
- * @since 2.0.0
- */
- default ImportDestinationArguments importDirectory(final String directory,
- final boolean ignoreEmptyDir) {
throw new UnsupportedOperationException();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 4b630e2..0c103b3 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -2003,12 +2003,6 @@ public class TableOperationsImpl extends
TableOperationsHelper {
@Override
public ImportDestinationArguments importDirectory(String directory) {
-return importDirectory(directory, false);
- }
-
- @Override
- public ImportDestinationArguments importDirectory(final String directory,
- final boolean ignoreEmptyDir) {
-return new BulkImport(directory, ignoreEmptyDir, context);
+return new BulkImport(directory, context);
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
index ec469d2..995b73e 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
@@ -97,24 +97,19 @@ public class BulkImport implements
ImportDestinationArguments, ImportMappingOpti
private static final Logger log = LoggerFactory.getLogger(BulkImport.class);
private boolean setTime = false;
+ private boolean ignoreEmptyDir = false;
private Executor executor = null;
private final String dir;
private
[accumulo] branch main updated: Add bulk import option to ignore empty dirs (#2039)
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new da2b099 Add bulk import option to ignore empty dirs (#2039)
da2b099 is described below
commit da2b0997421feb709b280020e940689cff2dc2bb
Author: Mark Owens
AuthorDate: Wed Apr 28 09:27:47 2021 -0400
Add bulk import option to ignore empty dirs (#2039)
Update the new bulk import operations to allow empty source directories to
be ignored rather than throwing an exception. There is now a '-i' option that
will log an info message indicating that the source directory was empty and no
files were imported rather than throwing an IllegalArgumentException.
This will support the use case of users who want to write idempotent bulk
import code.
---
.../core/client/admin/TableOperations.java | 26
.../core/clientImpl/TableOperationsImpl.java | 8 ++-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 17 +-
.../shell/commands/ImportDirectoryCommand.java | 13 ++--
.../apache/accumulo/shell/commands/OptUtil.java| 20 +++
.../shell/commands/ImportDirectoryCommandTest.java | 10 +++-
.../org/apache/accumulo/test/ShellServerIT.java| 69 +++---
.../apache/accumulo/test/functional/BulkIT.java| 9 +++
.../apache/accumulo/test/functional/BulkNewIT.java | 20 +++
9 files changed, 174 insertions(+), 18 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
index d8cf16f..78b99fe 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java
@@ -802,6 +802,32 @@ public interface TableOperations {
* @since 2.0.0
*/
default ImportDestinationArguments importDirectory(String directory) {
+return importDirectory(directory, false);
+ }
+
+ /**
+ * Bulk import the files in a directory into a table. Files can be created
using
+ * {@link RFile#newWriter()}.
+ *
+ * This new method of bulk import examines files in the current process
outside of holding a table
+ * lock. The old bulk import method ({@link #importDirectory(String, String,
String, boolean)})
+ * examines files on the server side while holding a table read lock. This
version of the method
+ * provides a boolean argument which will prevent an exception from being
thrown if the supplied
+ * directory contains no files.
+ *
+ * This API supports adding files to online and offline tables.
+ *
+ * For example, to bulk import files from the directory 'dir1' into the
table 'table1' use the
+ * following code.
+ *
+ *
+ * client.tableOperations().importDirectory("dir1",
true).to("table1").load();
+ *
+ *
+ * @since 2.0.0
+ */
+ default ImportDestinationArguments importDirectory(final String directory,
+ final boolean ignoreEmptyDir) {
throw new UnsupportedOperationException();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index a97ef8b..de64166 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -2003,6 +2003,12 @@ public class TableOperationsImpl extends
TableOperationsHelper {
@Override
public ImportDestinationArguments importDirectory(String directory) {
-return new BulkImport(directory, context);
+return importDirectory(directory, false);
+ }
+
+ @Override
+ public ImportDestinationArguments importDirectory(final String directory,
+ final boolean ignoreEmptyDir) {
+return new BulkImport(directory, ignoreEmptyDir, context);
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
index 94fdf8a..ec469d2 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
@@ -103,12 +103,18 @@ public class BulkImport implements
ImportDestinationArguments, ImportMappingOpti
private final ClientContext context;
private String tableName;
+ private boolean ignoreEmptyDir = false;
private LoadPlan plan = null;
public BulkImport(String directory, ClientContext context) {
+this(directory, false, context);
+ }
+
+ public BulkImport(String directory, boolean ignoreEmptyDirectory,
ClientContext context) {
this.context = context
[accumulo-examples] branch main updated: Update SequentialBatchWriter.java
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 6c3a912 Update SequentialBatchWriter.java 6c3a912 is described below commit 6c3a9120ac4b124275a85d1f2fec2e7931cc443d Author: Mark Owens AuthorDate: Fri Mar 26 10:05:47 2021 -0400 Update SequentialBatchWriter.java Fix org.apache.accumulo.core.client import statement in SequentialBatchWriter. Replace the '*' catch-all with specified imports. --- .../apache/accumulo/examples/client/SequentialBatchWriter.java| 8 +++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/accumulo/examples/client/SequentialBatchWriter.java b/src/main/java/org/apache/accumulo/examples/client/SequentialBatchWriter.java index 2dd5667..ea8ca2d 100644 --- a/src/main/java/org/apache/accumulo/examples/client/SequentialBatchWriter.java +++ b/src/main/java/org/apache/accumulo/examples/client/SequentialBatchWriter.java @@ -18,7 +18,13 @@ package org.apache.accumulo.examples.client; import java.util.Random; -import org.apache.accumulo.core.client.*; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.examples.cli.ClientOpts;
[accumulo-examples] branch main updated: Update pom.xml
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new c2f3002 Update pom.xml c2f3002 is described below commit c2f300251afc57742292506e1429074b6fc45f17 Author: Mark Owens AuthorDate: Thu Mar 25 14:46:36 2021 -0400 Update pom.xml Excluding jute when the ZK version if prior to 3.6 can result in a NoClassDefFoundError for org/apache/jute/Record. --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3601f32..2a43708 100644 --- a/pom.xml +++ b/pom.xml @@ -327,7 +327,6 @@ org.apache.logging.log4j:log4j-1.2-api:jar: org.apache.logging.log4j:log4j-slf4j-impl:jar: javax.activation:javax.activation-api:jar: - org.apache.zookeeper:zookeeper-jute:jar: org.apache.hadoop:hadoop-annotations:jar: org.apache.hadoop:hadoop-auth:jar: org.apache.hadoop:hadoop-mapreduce-client-common:jar:
[accumulo-examples] branch main updated: Update env.sh.example
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 6723497 Update env.sh.example
6723497 is described below
commit 672349771ecb4299f7b696ad294a5be1eef77030
Author: Mark Owens
AuthorDate: Thu Mar 25 08:31:11 2021 -0400
Update env.sh.example
Update env.sh.example file so that the version of ZooKeeper used by
Accumulo-Examples repo matches version of current accumulo-snapshot. Reverted
from ZK 3.6.2 to 3.5.9.
---
conf/env.sh.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/env.sh.example b/conf/env.sh.example
index 9691521..eb02bee 100644
--- a/conf/env.sh.example
+++ b/conf/env.sh.example
@@ -40,7 +40,7 @@ fi
# Versions set below will be what is included in the shaded jar
export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`"
export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`"
-export ZOOKEEPER_VERSION=3.6.2
+export ZOOKEEPER_VERSION=3.5.9
# Path to shaded test jar
ex_home=$( cd "$( dirname "$conf_dir" )" && pwd )
export EXAMPLES_JAR_PATH="${ex_home}/target/accumulo-examples-shaded.jar"
[accumulo-examples] branch main updated: Update pom.xml
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 02ab78f Update pom.xml 02ab78f is described below commit 02ab78f83df63f05ba1cefddafb622fe6a7b318f Author: Mark Owens AuthorDate: Thu Mar 25 08:29:57 2021 -0400 Update pom.xml Update pom in order to have Accumulo-Examples zookeeper version match version of current accumulo-snapshot. Reverted from ZK 3.6.2 to 3.5.9. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8c18530..438fcf0 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 11 11 11 -3.6.2 +3.5.9
[accumulo-examples] branch main updated: Update BulkIngestExample
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new be2545d Update BulkIngestExample
be2545d is described below
commit be2545d4c864477ee1fa9e58ee5ba5861ea857b5
Author: Mark Owens
AuthorDate: Wed Mar 24 14:08:28 2021 -0400
Update BulkIngestExample
Correct invalid return type from main method in BulkIngestExample.
---
.../apache/accumulo/examples/mapreduce/bulk/BulkIngestExample.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/BulkIngestExample.java
b/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/BulkIngestExample.java
index d5b3aa1..6b27309 100644
---
a/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/BulkIngestExample.java
+++
b/src/main/java/org/apache/accumulo/examples/mapreduce/bulk/BulkIngestExample.java
@@ -102,13 +102,13 @@ public final class BulkIngestExample {
}
}
- public static int main(String[] args) throws Exception {
+ public static void main(String[] args) throws Exception {
ClientOpts opts = new ClientOpts();
opts.parseArgs(BulkIngestExample.class.getName(), args);
FileSystem fs = FileSystem.get(opts.getHadoopConfig());
generateTestData(fs);
-return ingestTestData(fs, opts);
+ingestTestData(fs, opts);
}
private static void generateTestData(FileSystem fs) throws IOException {
[accumulo-examples] branch main updated: Update pom.xml
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 13ba110 Update pom.xml 13ba110 is described below commit 13ba1107197fa5721a76b4d237d438514a3911ad Author: Mark Owens AuthorDate: Tue Mar 23 15:21:32 2021 -0400 Update pom.xml Update ZooKeeper version in Accumulo-Examples pom file to 3.6.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f2b43e5..8c18530 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ 11 11 11 -3.4.14 +3.6.2
[accumulo-examples] branch main updated: Update env.sh.example
This is an automated email from the ASF dual-hosted git repository.
jmark99 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new b8f28b9 Update env.sh.example
b8f28b9 is described below
commit b8f28b9ae2a3a37b63c9d7bcdb97830107a96153
Author: Mark Owens
AuthorDate: Tue Mar 23 15:20:18 2021 -0400
Update env.sh.example
Update env.sh.example configuration file to use ZooKeeper 3.6.2
---
conf/env.sh.example | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/env.sh.example b/conf/env.sh.example
index 1f1461b..9691521 100644
--- a/conf/env.sh.example
+++ b/conf/env.sh.example
@@ -40,7 +40,7 @@ fi
# Versions set below will be what is included in the shaded jar
export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`"
export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`"
-export ZOOKEEPER_VERSION=3.4.13
+export ZOOKEEPER_VERSION=3.6.2
# Path to shaded test jar
ex_home=$( cd "$( dirname "$conf_dir" )" && pwd )
export EXAMPLES_JAR_PATH="${ex_home}/target/accumulo-examples-shaded.jar"
[accumulo-examples] branch main updated (1ca755b -> 3fbd7de)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git. from 1ca755b Update wordcount.md add 40c5d90 Update Accumulo-Examples pom new 3fbd7de Update Accumulo-Examples pom to use jdk 11 and latest Accumulo snapshot The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml| 22 +++--- .../apache/accumulo/examples/dirlist/Viewer.java | 9 + .../examples/mapreduce/bulk/BulkIngestExample.java | 2 ++ 3 files changed, 18 insertions(+), 15 deletions(-)
[accumulo-examples] 01/01: Update Accumulo-Examples pom to use jdk 11 and latest Accumulo snapshot
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git commit 3fbd7de6bf7a95ae57655ecc8c623ae1ccc5db3e Merge: 1ca755b 40c5d90 Author: Mark Owens AuthorDate: Wed Mar 17 07:45:40 2021 -0400 Update Accumulo-Examples pom to use jdk 11 and latest Accumulo snapshot Update Accumulo-Examples pom to use jdk 11 and latest Accumulo snapshot. Update pom.xml to make use of jdk 11 as well as refer to latest Accumulo snapshot. Made two minor changes to the Viewer and BulkIngestExample classes also. - update pom.xml to refer to Accumulo-2.1.0-SNAPSHOT - update pom.xml to use jdk 11 - update Viewer.java to run with jdk 11 - add info message to BulkIngestExample class Closes #68 pom.xml| 22 +++--- .../apache/accumulo/examples/dirlist/Viewer.java | 9 + .../examples/mapreduce/bulk/BulkIngestExample.java | 2 ++ 3 files changed, 18 insertions(+), 15 deletions(-)
[accumulo-examples] branch main updated: Update wordcount.md
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 1ca755b Update wordcount.md 1ca755b is described below commit 1ca755b5ab155a7149c43e26d08b22c74f5e71d9 Author: Mark Owens AuthorDate: Tue Mar 16 11:35:23 2021 -0400 Update wordcount.md Corrected type in final scan command. Scan table wordCount2 rather than table wordCount. --- docs/wordcount.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/wordcount.md b/docs/wordcount.md index 3f9cb4b..3e3cc52 100644 --- a/docs/wordcount.md +++ b/docs/wordcount.md @@ -62,8 +62,8 @@ After the MapReduce job completes, query the `wordCount2` table. The results sho be the same as before: $ accumulo shell -username@instance> table wordCount -username@instance wordCount> scan -b the +username@instance> table wordCount2 +username@instance wordCount2> scan -b the the count:20080906 []75 their count:20080906 []2 ...
[accumulo-examples] branch main updated: Update rgbalancer.md
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git The following commit(s) were added to refs/heads/main by this push: new 03dd655 Update rgbalancer.md 03dd655 is described below commit 03dd655075b1db7bf3423ba0954f3e5f5445fa25 Author: Mark Owens AuthorDate: Mon Mar 8 11:43:16 2021 -0500 Update rgbalancer.md Updated the regex pattern. With version 2.1 the double backslashes for the regex through a BadArgumentexception. Assume it is related to change from using Jline ConsoleReader to using LineReader. --- docs/rgbalancer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rgbalancer.md b/docs/rgbalancer.md index 7f897c0..daa1a3b 100644 --- a/docs/rgbalancer.md +++ b/docs/rgbalancer.md @@ -95,7 +95,7 @@ commands below. The configured regular expression selects the first two digits from a tablets end row as the group id. Tablets that don't match and the default tablet are configured to be in group 04. -root@accumulo testRGB> config -t testRGB -s table.custom.balancer.group.regex.pattern=(\\d\\d).* +root@accumulo testRGB> config -t testRGB -s table.custom.balancer.group.regex.pattern=(\d\d).* root@accumulo testRGB> config -t testRGB -s table.custom.balancer.group.regex.default=04 root@accumulo testRGB> config -t testRGB -s table.balancer=org.apache.accumulo.server.master.balancer.RegexGroupBalancer
