[accumulo] branch main updated: Print warning in shell when user does not have permission to get config (#3015)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8a53e8dab5 Print warning in shell when user does not have permission
to get config (#3015)
8a53e8dab5 is described below
commit 8a53e8dab51aca58a04b513116763bbe4817f0a2
Author: Dave Marion
AuthorDate: Fri Oct 14 07:16:38 2022 -0400
Print warning in shell when user does not have permission to get config
(#3015)
Modified ConfigCommand to only show the user properties that they have
permission to see. When a user is unable to see the properties at a
specific level warnings are printed in the shell letting them know what
permission they are missing and that the properties that they are seeing
for the table or namespace may actually be set at a higher level that they
can't see. Override information is still shown, but only for levels that
the user
has access to.
---
.../core/clientImpl/NamespaceOperationsImpl.java | 6 +-
.../core/clientImpl/TableOperationsHelper.java | 4 +-
.../core/clientImpl/TableOperationsImpl.java | 4 +-
.../server/client/ClientServiceHandler.java| 14 ++-
.../org/apache/accumulo/server/util/Admin.java | 3 +-
shell/pom.xml | 4 +
.../accumulo/shell/commands/ConfigCommand.java | 113 ++---
.../accumulo/test/NewTableConfigurationIT.java | 3 +-
8 files changed, 126 insertions(+), 25 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
index 88b1e116ee..cd0dd2b580 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
@@ -285,12 +285,14 @@ public class NamespaceOperationsImpl extends
NamespaceOperationsHelper {
@Override
public Map getConfiguration(final String namespace)
- throws AccumuloException, NamespaceNotFoundException {
+ throws AccumuloException, AccumuloSecurityException,
NamespaceNotFoundException {
EXISTING_NAMESPACE_NAME.validate(namespace);
try {
return ThriftClientTypes.CLIENT.execute(context, client -> client
.getNamespaceConfiguration(TraceUtil.traceInfo(),
context.rpcCreds(), namespace));
+} catch (AccumuloSecurityException e) {
+ throw e;
} catch (AccumuloException e) {
Throwable t = e.getCause();
if (t instanceof ThriftTableOperationException) {
@@ -395,7 +397,7 @@ public class NamespaceOperationsImpl extends
NamespaceOperationsHelper {
}
private void checkLocalityGroups(String namespace, String propChanged)
- throws AccumuloException, NamespaceNotFoundException {
+ throws AccumuloSecurityException, AccumuloException,
NamespaceNotFoundException {
EXISTING_NAMESPACE_NAME.validate(namespace);
if (LocalityGroupUtil.isLocalityGroupProperty(propChanged)) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java
index 27f7a9f4a5..1748592792 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java
@@ -81,7 +81,7 @@ public abstract class TableOperationsHelper implements
TableOperations {
@Override
public IteratorSetting getIteratorSetting(String tableName, String name,
IteratorScope scope)
- throws AccumuloException, TableNotFoundException {
+ throws AccumuloSecurityException, AccumuloException,
TableNotFoundException {
EXISTING_TABLE_NAME.validate(tableName);
checkArgument(name != null, "name is null");
checkArgument(scope != null, "scope is null");
@@ -113,7 +113,7 @@ public abstract class TableOperationsHelper implements
TableOperations {
@Override
public Map> listIterators(String tableName)
- throws AccumuloException, TableNotFoundException {
+ throws AccumuloSecurityException, AccumuloException,
TableNotFoundException {
EXISTING_TABLE_NAME.validate(tableName);
Map> result = new TreeMap<>();
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 65222bf5f9..1f66da5630 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -1098,7 +1098,7 @@ public class TableOperationsImpl extends
TableOperationsHelper {
}
void checkLocalityGr
[accumulo-testing] branch main updated: Standardize formatting of shell scripts (#245)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new c803776 Standardize formatting of shell scripts (#245)
c803776 is described below
commit c80377616e04fcb513f5d3be3dae7b73871e7561
Author: Christopher Tubbs
AuthorDate: Fri Oct 14 09:30:56 2022 -0400
Standardize formatting of shell scripts (#245)
* Standardize shebang lines for bash scripts
* Run `shfmt -ln bash -l -d -i 2 -ci -s .` to format scripts
---
bin/agitator | 41 +
bin/build | 4 +-
bin/cingest| 6 +-
bin/gcs| 3 +-
bin/mapred | 3 +-
bin/monitor| 3 +-
bin/performance| 22 +++--
bin/rwalk | 4 +-
conf/env.sh| 23 +++--
.../files/azure-format-lvm-data-disk.sh| 27 +++---
.../files/update-hosts-genders.sh | 10 +--
src/main/docker/docker-entry | 8 +-
test/automation/automateEC2.sh | 100 ++---
test/automation/cluster_props.sh | 1 -
test/merkle-replication/configure-replication.sh | 57 ++--
test/merkle-replication/ingest-data.sh | 18 ++--
test/merkle-replication/merkle-env.sh | 1 +
test/merkle-replication/verify-data.sh | 60 ++---
test/stress/reader.sh | 12 +--
test/stress/start-readers.sh | 12 +--
test/stress/start-writers.sh | 12 +--
test/stress/stop-readers.sh| 14 +--
test/stress/stop-writers.sh| 14 +--
test/stress/stress-env.sh | 4 +-
test/stress/writer.sh | 12 +--
test/test1/verify_test.sh | 10 +--
test/test1/verify_test_2.sh| 10 +--
test/test2/concurrent.sh | 84 +
test/test3/bigrow.sh | 10 +--
test/test4/bulk_import_test.sh | 1 -
test/upgrade/upgrade_test.sh | 8 +-
31 files changed, 300 insertions(+), 294 deletions(-)
diff --git a/bin/agitator b/bin/agitator
index 50157b5..23d2bbd 100755
--- a/bin/agitator
+++ b/bin/agitator
@@ -18,8 +18,8 @@
# under the License.
#
-bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-at_home=$( cd "$( dirname "$bin_dir" )" && pwd )
+bin_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+at_home=$(cd "$(dirname "$bin_dir")" && pwd)
source "${bin_dir}/build"
function print_usage() {
@@ -44,22 +44,23 @@ function start_app_agitator() {
local max_kill=$5
local start_cmd=$6
local kill_cmd=$7
- local hosts_array; read -a hosts_array < <(get_app_hosts "$app_name")
+ local hosts_array
+ read -a hosts_array < <(get_app_hosts "$app_name")
local num_hosts=${#hosts_array[@]}
local node_to_kill
nodes_to_kill_array=()
local T
local ENV_VARS="ACCUMULO_HOME=$ACCUMULO_HOME ZOOKEEPER_HOME=$ZOOKEEPER_HOME
HADOOP_HOME=$HADOOP_HOME JAVA_HOME=$JAVA_HOME"
- if (( num_hosts == 0 )); then
+ if ((num_hosts == 0)); then
echo "ERROR: No hosts were found in env for $app_name"
exit 1
fi
- if (( max_kill > num_hosts )); then
+ if ((max_kill > num_hosts)); then
echo "ERROR: Max kill $max_kill is greater than number of hosts $num_hosts"
exit 1
fi
- if (( max_kill < min_kill )); then
+ if ((max_kill < min_kill)); then
echo "ERROR: Max kill $max_kill is less than min kill $min_kill"
exit 1
fi
@@ -72,7 +73,7 @@ function start_app_agitator() {
sleep $((kill_sleep_time * 60))
T="$(date +'%Y%m%d %H:%M:%S')"
-if ((max_kill == 1)) ; then
+if ((max_kill == 1)); then
node_to_kill=${hosts_array[0]}
echo "$T Killing $app_name at $node_to_kill"
ssh "$node_to_kill" "$kill_cmd"
@@ -100,7 +101,7 @@ function start_app_agitator() {
sleep $((restart_sleep_time * 60))
T="$(date +'%Y%m%d %H:%M:%S')"
-if ((max_kill == 1)) ; then
+if ((max_kill == 1)); then
echo "$T Restarting $app_name at $node_to_kill"
ssh "$node_to_kill" "bash -c '${ENV_VARS} $start_cmd'"
else
@@ -120,7 +121,8 @@ function start_agitator() {
elif hash parallel-ssh 2>/dev/null; then
export PSSH=parallel-ssh
else
-echo >&2 "The agitator requires pssh/parallel-ssh to be installed.
Aborting."; exit 1;
+echo >&2 "The agitator requires pssh/parallel-ssh to be installed.
Aborting."
[accumulo] branch main updated: Fix bug with concurrent merge and bulk import (#3014)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new f6c943e70d Fix bug with concurrent merge and bulk import (#3014)
f6c943e70d is described below
commit f6c943e70dd14183985faeeefc5b554fcc731fff
Author: Keith Turner
AuthorDate: Fri Oct 14 14:41:30 2022 +0100
Fix bug with concurrent merge and bulk import (#3014)
This fixes #3006
---
.../clientImpl/bulk/ConcurrentKeyExtentCache.java | 25 +-
1 file changed, 20 insertions(+), 5 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
index 03d9bda441..ec154e7dbe 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
@@ -35,14 +35,19 @@ import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.bulk.BulkImport.KeyExtentCache;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.schema.TabletDeletedException;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
class ConcurrentKeyExtentCache implements KeyExtentCache {
+ private static Logger log =
LoggerFactory.getLogger(ConcurrentKeyExtentCache.class);
+
private static final Text MAX = new Text();
private Set rowsToLookup = Collections.synchronizedSet(new
HashSet<>());
@@ -120,12 +125,22 @@ class ConcurrentKeyExtentCache implements KeyExtentCache {
for (Text lookupRow : lookupRows) {
if (getFromCache(lookupRow) == null) {
-Iterator iter = lookupExtents(lookupRow).iterator();
-while (iter.hasNext()) {
- KeyExtent ke2 = iter.next();
- if (inCache(ke2))
+while (true) {
+ try {
+Iterator iter = lookupExtents(lookupRow).iterator();
+while (iter.hasNext()) {
+ KeyExtent ke2 = iter.next();
+ if (inCache(ke2))
+break;
+ updateCache(ke2);
+}
break;
- updateCache(ke2);
+ } catch (TabletDeletedException tde) {
+// tablets were merged away in the table, start over and try
again
+log.debug("While trying to obtain a tablet location for bulk
import, a tablet was "
++ "deleted. If this was caused by a concurrent merge
tablet "
++ "operation, this is okay. Otherwise, it could be a
problem.", tde);
+ }
}
}
}
[accumulo] branch main updated (f6c943e70d -> 4df8ae2640)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from f6c943e70d Fix bug with concurrent merge and bulk import (#3014) add 4df8ae2640 Update several dependencies to latest versions (#3022) No new revisions were added by this update. Summary of changes: pom.xml | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-)
[accumulo-testing] branch main updated: Update OpenTelemetery and Micrometer versions with Accumulo latest (#247)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 9bcb27c Update OpenTelemetery and Micrometer versions with Accumulo
latest (#247)
9bcb27c is described below
commit 9bcb27c81260577f55a5423457ef62a34c923dbd
Author: Christopher L. Shannon
AuthorDate: Fri Oct 14 11:43:13 2022 -0400
Update OpenTelemetery and Micrometer versions with Accumulo latest (#247)
---
contrib/terraform-testing-infrastructure/README.md| 4 ++--
.../modules/config-files/templates/hadoop_bashrc.tftpl| 2 +-
.../modules/config-files/templates/install_sw.sh.tftpl| 8
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/contrib/terraform-testing-infrastructure/README.md
b/contrib/terraform-testing-infrastructure/README.md
index 79626e9..1fdb10a 100644
--- a/contrib/terraform-testing-infrastructure/README.md
+++ b/contrib/terraform-testing-infrastructure/README.md
@@ -326,11 +326,11 @@ This Terraform configuration:
and builds the software using Maven, then untars the binary tarball to
`${software_root}/accumulo/accumulo-${accumulo_version}`
5. Downloads the [OpenTelemetry](https://opentelemetry.io/) Java Agent jar
file and copies it to
-
`${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar`
+
`${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.19.0.jar`
6. Copies the Accumulo `test` jar to
`${software_root}/accumulo/accumulo-${accumulo_version}/lib`
so that `org.apache.accumulo.test.metrics.TestStatsDRegistryFactory` is
on the classpath
7. Downloads the [Micrometer](https://micrometer.io/) StatsD Registry jar
file and copies it to
-
`${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.9.4.jar`
+
`${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.9.5.jar`
8. Clones, if necessary, the Apache Accumulo Testing Git repo from
`${accumulo_testing_repo}`
into `${software_root}/sources/accumulo-testing-repo`. It switches to the
`${accumulo_testing_branch_name}` branch and builds the software using
Maven.
diff --git
a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
index 11eba3e..9004b9f 100644
---
a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
+++
b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl
@@ -28,7 +28,7 @@ export
ACCUMULO_HOME=${software_root}/accumulo/accumulo-${accumulo_version}
export ACCUMULO_LOG_DIR=${accumulo_dir}/logs
export M2_HOME=${software_root}/apache-maven/apache-maven-${maven_version}
-export
ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar
-Dotel.traces.exporter=jaeger -Dotel.metrics.exporter=none
-Dotel.logs.exporter=none
-Dotel.exporter.jaeger.endpoint=http://${manager_ip}:14250
-Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125"
+export
ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.19.0.jar
-Dotel.traces.exporter=jaeger -Dotel.metrics.exporter=none
-Dotel.logs.exporter=none
-Dotel.exporter.jaeger.endpoint=http://${manager_ip}:14250
-Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125"
# User specific environment and startup programs
PATH=$PATH:$HOME/.local/bin:$HOME/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ACCUMULO_HOME/bin:$M2_HOME/bin
diff --git
a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
index 2e2d7f5..2a58c35 100644
---
a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
+++
b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl
@@ -110,8 +110,8 @@ openssl rand -out
${software_root}/accumulo/accumulo-${accumulo_version}/conf/pe
#
# OpenTelemetry dependencies
#
-if [ ! -f
${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar
]; then
- wget
https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/1.18.0/opentelemetry-javaagent-1.18.0.jar
-O
${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar
+if [ ! -f
${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.19.0.jar
]; then
+ wget
https://searc
[accumulo-testing] branch main updated: More carefully update license headers
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new 6d6f4e2 More carefully update license headers 6d6f4e2 is described below commit 6d6f4e2cf1d64f44fc7912ec5042592c4e4a7a98 Author: Christopher Tubbs AuthorDate: Fri Oct 14 12:05:11 2022 -0400 More carefully update license headers Fix issues with #244 by more carefully updating license headers in Terraform templates Remove problematic, or potentially problematic (and definitely unnecessary) license headers in cloud-init, genders, bash profile, and hosts Terraform template files --- .../cloud-init-config/templates/cloud-init.tftpl | 19 --- .../config-files/templates/core-site.xml.tftpl | 39 +++--- .../modules/config-files/templates/genders.tftpl | 19 --- .../templates/hadoop_bash_profile.tftpl| 19 --- .../config-files/templates/hadoop_bashrc.tftpl | 19 --- .../config-files/templates/hdfs-site.xml.tftpl | 39 +++--- .../modules/config-files/templates/hosts.tftpl | 19 --- .../templates/initialize_accumulo.sh.tftpl | 3 +- .../templates/initialize_hadoop.sh.tftpl | 3 +- .../config-files/templates/install_sw.sh.tftpl | 3 +- .../config-files/templates/yarn-site.xml.tftpl | 39 +++--- pom.xml| 4 ++- 12 files changed, 66 insertions(+), 159 deletions(-) diff --git a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl index f7932db..0593b10 100644 --- a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl @@ -1,22 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - #cloud-config # # Don't log key information diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl index 7ec0e32..7df9093 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl @@ -1,24 +1,25 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - + fs.defaultFS diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl index 9747cb8..15d2043 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl @@ -1,22 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License")
[accumulo] branch main updated (4df8ae2640 -> 6c8a4088dd)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4df8ae2640 Update several dependencies to latest versions (#3022) add 6c8a4088dd Change List to SortedSet in RecoveryLogsIterator (#3021) No new revisions were added by this update. Summary of changes: .../accumulo/tserver/log/RecoveryLogsIterator.java | 16 +++- 1 file changed, 11 insertions(+), 5 deletions(-)
[accumulo] branch main updated: Make use of GitHub Actions environment files (#3024)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 04ac8b222d Make use of GitHub Actions environment files (#3024)
04ac8b222d is described below
commit 04ac8b222d866fab2552087cbffa150c2d402a3c
Author: Christopher Tubbs
AuthorDate: Fri Oct 14 16:52:49 2022 -0400
Make use of GitHub Actions environment files (#3024)
set-output was previously used to create the IT matrix, but it is now
deprecated. So, this change makes use of the environment files instead.
See
https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#environment-files
Note: the emitted key=value should not quote the value, or else it will be
parsed as a String instead of a JSON type. This took me so long to figure
out, so I'm sharing it here. Also, if you change the name of the output
property, make sure you update the mapping of the step output to the job
output to use the new name.
---
.github/workflows/maven-full-its.yaml | 2 +-
contrib/ci/it-matrix.sh | 42 ---
2 files changed, 30 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/maven-full-its.yaml
b/.github/workflows/maven-full-its.yaml
index 047ded2fee..a70eea19a7 100644
--- a/.github/workflows/maven-full-its.yaml
+++ b/.github/workflows/maven-full-its.yaml
@@ -72,7 +72,7 @@ jobs:
needs: fastbuild
timeout-minutes: 5
outputs:
- matrix: ${{ steps.set-matrix.outputs.matrix }}
+ matrix: ${{ steps.set-matrix.outputs.CUSTOM_MATRIX }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
diff --git a/contrib/ci/it-matrix.sh b/contrib/ci/it-matrix.sh
index f84c93fc57..6e0f809ce6 100755
--- a/contrib/ci/it-matrix.sh
+++ b/contrib/ci/it-matrix.sh
@@ -22,16 +22,32 @@ testsPerJob=15
if [[ -n $1 && $1 =~ ^[0-9]*$ ]]; then
testsPerJob=$1
fi
-echo "Creating matrix (tests per job: $testsPerJob)..."
-
-gitRootDir=$(git rev-parse --show-toplevel)
-# this only works because our test paths don't have spaces; we should keep it
that way
-count=0
-echo -n '::set-output name=matrix::{"profile":['
-for x in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java \; |
sort -u | xargs -n "$testsPerJob" | tr ' ' ','); do
- [[ $count -gt 0 ]] && echo -n ','
- echo -n "{\"name\":\"task_$count\",\"its\":\"$x\"}"
- ((count = count + 1))
-done
-echo ']}'
-echo "Finished creating matrix ($count tasks)"
+
+# set these to /dev/null if they aren't defined in the environment
+GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}"
+GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}"
+
+function createTestMatrix() {
+
+ local count=0
+ local chunk=$1
+ local batch
+ local gitRootDir
+
+ { echo "Creating matrix (tests per job: $chunk)..." | tee -a
"$GITHUB_STEP_SUMMARY"; } 1>&2
+
+ gitRootDir=$(git rev-parse --show-toplevel)
+
+ # this only works because our test paths don't have spaces; we should keep
it that way
+ echo -n '{"profile":['
+ for batch in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java
\; | sort -u | xargs -n "$chunk" | tr ' ' ','); do
+[[ $count -gt 0 ]] && echo -n ','
+echo -n '{"name":"task_'"$count"'","its":"'"$batch"'"}'
+((count = count + 1))
+ done
+ echo ']}'
+
+ { echo "Finished creating matrix ($count tasks)" | tee
"$GITHUB_STEP_SUMMARY"; } 1>&2
+}
+
+echo "CUSTOM_MATRIX=$(createTestMatrix "$testsPerJob")" | tee -a
"$GITHUB_OUTPUT"
