[accumulo] branch main updated: Print warning in shell when user does not have permission to get config (#3015)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 8a53e8dab5 Print warning in shell when user does not have permission to get config (#3015) 8a53e8dab5 is described below commit 8a53e8dab51aca58a04b513116763bbe4817f0a2 Author: Dave Marion AuthorDate: Fri Oct 14 07:16:38 2022 -0400 Print warning in shell when user does not have permission to get config (#3015) Modified ConfigCommand to only show the user properties that they have permission to see. When a user is unable to see the properties at a specific level warnings are printed in the shell letting them know what permission they are missing and that the properties that they are seeing for the table or namespace may actually be set at a higher level that they can't see. Override information is still shown, but only for levels that the user has access to. --- .../core/clientImpl/NamespaceOperationsImpl.java | 6 +- .../core/clientImpl/TableOperationsHelper.java | 4 +- .../core/clientImpl/TableOperationsImpl.java | 4 +- .../server/client/ClientServiceHandler.java| 14 ++- .../org/apache/accumulo/server/util/Admin.java | 3 +- shell/pom.xml | 4 + .../accumulo/shell/commands/ConfigCommand.java | 113 ++--- .../accumulo/test/NewTableConfigurationIT.java | 3 +- 8 files changed, 126 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java index 88b1e116ee..cd0dd2b580 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java @@ -285,12 +285,14 @@ public class NamespaceOperationsImpl extends NamespaceOperationsHelper { @Override public Map getConfiguration(final String namespace) - throws AccumuloException, NamespaceNotFoundException { + throws AccumuloException, AccumuloSecurityException, NamespaceNotFoundException { EXISTING_NAMESPACE_NAME.validate(namespace); try { return ThriftClientTypes.CLIENT.execute(context, client -> client .getNamespaceConfiguration(TraceUtil.traceInfo(), context.rpcCreds(), namespace)); +} catch (AccumuloSecurityException e) { + throw e; } catch (AccumuloException e) { Throwable t = e.getCause(); if (t instanceof ThriftTableOperationException) { @@ -395,7 +397,7 @@ public class NamespaceOperationsImpl extends NamespaceOperationsHelper { } private void checkLocalityGroups(String namespace, String propChanged) - throws AccumuloException, NamespaceNotFoundException { + throws AccumuloSecurityException, AccumuloException, NamespaceNotFoundException { EXISTING_NAMESPACE_NAME.validate(namespace); if (LocalityGroupUtil.isLocalityGroupProperty(propChanged)) { diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java index 27f7a9f4a5..1748592792 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsHelper.java @@ -81,7 +81,7 @@ public abstract class TableOperationsHelper implements TableOperations { @Override public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) - throws AccumuloException, TableNotFoundException { + throws AccumuloSecurityException, AccumuloException, TableNotFoundException { EXISTING_TABLE_NAME.validate(tableName); checkArgument(name != null, "name is null"); checkArgument(scope != null, "scope is null"); @@ -113,7 +113,7 @@ public abstract class TableOperationsHelper implements TableOperations { @Override public Map> listIterators(String tableName) - throws AccumuloException, TableNotFoundException { + throws AccumuloSecurityException, AccumuloException, TableNotFoundException { EXISTING_TABLE_NAME.validate(tableName); Map> result = new TreeMap<>(); diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java index 65222bf5f9..1f66da5630 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java @@ -1098,7 +1098,7 @@ public class TableOperationsImpl extends TableOperationsHelper { } void checkLocalityGr
[accumulo-testing] branch main updated: Standardize formatting of shell scripts (#245)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new c803776 Standardize formatting of shell scripts (#245) c803776 is described below commit c80377616e04fcb513f5d3be3dae7b73871e7561 Author: Christopher Tubbs AuthorDate: Fri Oct 14 09:30:56 2022 -0400 Standardize formatting of shell scripts (#245) * Standardize shebang lines for bash scripts * Run `shfmt -ln bash -l -d -i 2 -ci -s .` to format scripts --- bin/agitator | 41 + bin/build | 4 +- bin/cingest| 6 +- bin/gcs| 3 +- bin/mapred | 3 +- bin/monitor| 3 +- bin/performance| 22 +++-- bin/rwalk | 4 +- conf/env.sh| 23 +++-- .../files/azure-format-lvm-data-disk.sh| 27 +++--- .../files/update-hosts-genders.sh | 10 +-- src/main/docker/docker-entry | 8 +- test/automation/automateEC2.sh | 100 ++--- test/automation/cluster_props.sh | 1 - test/merkle-replication/configure-replication.sh | 57 ++-- test/merkle-replication/ingest-data.sh | 18 ++-- test/merkle-replication/merkle-env.sh | 1 + test/merkle-replication/verify-data.sh | 60 ++--- test/stress/reader.sh | 12 +-- test/stress/start-readers.sh | 12 +-- test/stress/start-writers.sh | 12 +-- test/stress/stop-readers.sh| 14 +-- test/stress/stop-writers.sh| 14 +-- test/stress/stress-env.sh | 4 +- test/stress/writer.sh | 12 +-- test/test1/verify_test.sh | 10 +-- test/test1/verify_test_2.sh| 10 +-- test/test2/concurrent.sh | 84 + test/test3/bigrow.sh | 10 +-- test/test4/bulk_import_test.sh | 1 - test/upgrade/upgrade_test.sh | 8 +- 31 files changed, 300 insertions(+), 294 deletions(-) diff --git a/bin/agitator b/bin/agitator index 50157b5..23d2bbd 100755 --- a/bin/agitator +++ b/bin/agitator @@ -18,8 +18,8 @@ # under the License. # -bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -at_home=$( cd "$( dirname "$bin_dir" )" && pwd ) +bin_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +at_home=$(cd "$(dirname "$bin_dir")" && pwd) source "${bin_dir}/build" function print_usage() { @@ -44,22 +44,23 @@ function start_app_agitator() { local max_kill=$5 local start_cmd=$6 local kill_cmd=$7 - local hosts_array; read -a hosts_array < <(get_app_hosts "$app_name") + local hosts_array + read -a hosts_array < <(get_app_hosts "$app_name") local num_hosts=${#hosts_array[@]} local node_to_kill nodes_to_kill_array=() local T local ENV_VARS="ACCUMULO_HOME=$ACCUMULO_HOME ZOOKEEPER_HOME=$ZOOKEEPER_HOME HADOOP_HOME=$HADOOP_HOME JAVA_HOME=$JAVA_HOME" - if (( num_hosts == 0 )); then + if ((num_hosts == 0)); then echo "ERROR: No hosts were found in env for $app_name" exit 1 fi - if (( max_kill > num_hosts )); then + if ((max_kill > num_hosts)); then echo "ERROR: Max kill $max_kill is greater than number of hosts $num_hosts" exit 1 fi - if (( max_kill < min_kill )); then + if ((max_kill < min_kill)); then echo "ERROR: Max kill $max_kill is less than min kill $min_kill" exit 1 fi @@ -72,7 +73,7 @@ function start_app_agitator() { sleep $((kill_sleep_time * 60)) T="$(date +'%Y%m%d %H:%M:%S')" -if ((max_kill == 1)) ; then +if ((max_kill == 1)); then node_to_kill=${hosts_array[0]} echo "$T Killing $app_name at $node_to_kill" ssh "$node_to_kill" "$kill_cmd" @@ -100,7 +101,7 @@ function start_app_agitator() { sleep $((restart_sleep_time * 60)) T="$(date +'%Y%m%d %H:%M:%S')" -if ((max_kill == 1)) ; then +if ((max_kill == 1)); then echo "$T Restarting $app_name at $node_to_kill" ssh "$node_to_kill" "bash -c '${ENV_VARS} $start_cmd'" else @@ -120,7 +121,8 @@ function start_agitator() { elif hash parallel-ssh 2>/dev/null; then export PSSH=parallel-ssh else -echo >&2 "The agitator requires pssh/parallel-ssh to be installed. Aborting."; exit 1; +echo >&2 "The agitator requires pssh/parallel-ssh to be installed. Aborting."
[accumulo] branch main updated: Fix bug with concurrent merge and bulk import (#3014)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new f6c943e70d Fix bug with concurrent merge and bulk import (#3014) f6c943e70d is described below commit f6c943e70dd14183985faeeefc5b554fcc731fff Author: Keith Turner AuthorDate: Fri Oct 14 14:41:30 2022 +0100 Fix bug with concurrent merge and bulk import (#3014) This fixes #3006 --- .../clientImpl/bulk/ConcurrentKeyExtentCache.java | 25 +- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java index 03d9bda441..ec154e7dbe 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java @@ -35,14 +35,19 @@ import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.bulk.BulkImport.KeyExtentCache; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.metadata.schema.TabletDeletedException; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; class ConcurrentKeyExtentCache implements KeyExtentCache { + private static Logger log = LoggerFactory.getLogger(ConcurrentKeyExtentCache.class); + private static final Text MAX = new Text(); private Set rowsToLookup = Collections.synchronizedSet(new HashSet<>()); @@ -120,12 +125,22 @@ class ConcurrentKeyExtentCache implements KeyExtentCache { for (Text lookupRow : lookupRows) { if (getFromCache(lookupRow) == null) { -Iterator iter = lookupExtents(lookupRow).iterator(); -while (iter.hasNext()) { - KeyExtent ke2 = iter.next(); - if (inCache(ke2)) +while (true) { + try { +Iterator iter = lookupExtents(lookupRow).iterator(); +while (iter.hasNext()) { + KeyExtent ke2 = iter.next(); + if (inCache(ke2)) +break; + updateCache(ke2); +} break; - updateCache(ke2); + } catch (TabletDeletedException tde) { +// tablets were merged away in the table, start over and try again +log.debug("While trying to obtain a tablet location for bulk import, a tablet was " ++ "deleted. If this was caused by a concurrent merge tablet " ++ "operation, this is okay. Otherwise, it could be a problem.", tde); + } } } }
[accumulo] branch main updated (f6c943e70d -> 4df8ae2640)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from f6c943e70d Fix bug with concurrent merge and bulk import (#3014) add 4df8ae2640 Update several dependencies to latest versions (#3022) No new revisions were added by this update. Summary of changes: pom.xml | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-)
[accumulo-testing] branch main updated: Update OpenTelemetery and Micrometer versions with Accumulo latest (#247)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new 9bcb27c Update OpenTelemetery and Micrometer versions with Accumulo latest (#247) 9bcb27c is described below commit 9bcb27c81260577f55a5423457ef62a34c923dbd Author: Christopher L. Shannon AuthorDate: Fri Oct 14 11:43:13 2022 -0400 Update OpenTelemetery and Micrometer versions with Accumulo latest (#247) --- contrib/terraform-testing-infrastructure/README.md| 4 ++-- .../modules/config-files/templates/hadoop_bashrc.tftpl| 2 +- .../modules/config-files/templates/install_sw.sh.tftpl| 8 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contrib/terraform-testing-infrastructure/README.md b/contrib/terraform-testing-infrastructure/README.md index 79626e9..1fdb10a 100644 --- a/contrib/terraform-testing-infrastructure/README.md +++ b/contrib/terraform-testing-infrastructure/README.md @@ -326,11 +326,11 @@ This Terraform configuration: and builds the software using Maven, then untars the binary tarball to `${software_root}/accumulo/accumulo-${accumulo_version}` 5. Downloads the [OpenTelemetry](https://opentelemetry.io/) Java Agent jar file and copies it to - `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar` + `${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.19.0.jar` 6. Copies the Accumulo `test` jar to `${software_root}/accumulo/accumulo-${accumulo_version}/lib` so that `org.apache.accumulo.test.metrics.TestStatsDRegistryFactory` is on the classpath 7. Downloads the [Micrometer](https://micrometer.io/) StatsD Registry jar file and copies it to - `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.9.4.jar` + `${software_root}/accumulo/accumulo-${accumulo_version}/lib/micrometer-registry-statsd-1.9.5.jar` 8. Clones, if necessary, the Apache Accumulo Testing Git repo from `${accumulo_testing_repo}` into `${software_root}/sources/accumulo-testing-repo`. It switches to the `${accumulo_testing_branch_name}` branch and builds the software using Maven. diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl index 11eba3e..9004b9f 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/hadoop_bashrc.tftpl @@ -28,7 +28,7 @@ export ACCUMULO_HOME=${software_root}/accumulo/accumulo-${accumulo_version} export ACCUMULO_LOG_DIR=${accumulo_dir}/logs export M2_HOME=${software_root}/apache-maven/apache-maven-${maven_version} -export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar -Dotel.traces.exporter=jaeger -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.jaeger.endpoint=http://${manager_ip}:14250 -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125" +export ACCUMULO_JAVA_OPTS="-javaagent:${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.19.0.jar -Dotel.traces.exporter=jaeger -Dotel.metrics.exporter=none -Dotel.logs.exporter=none -Dotel.exporter.jaeger.endpoint=http://${manager_ip}:14250 -Dtest.meter.registry.host=${manager_ip} -Dtest.meter.registry.port=8125" # User specific environment and startup programs PATH=$PATH:$HOME/.local/bin:$HOME/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ACCUMULO_HOME/bin:$M2_HOME/bin diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl index 2e2d7f5..2a58c35 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/install_sw.sh.tftpl @@ -110,8 +110,8 @@ openssl rand -out ${software_root}/accumulo/accumulo-${accumulo_version}/conf/pe # # OpenTelemetry dependencies # -if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar ]; then - wget https://search.maven.org/remotecontent?filepath=io/opentelemetry/javaagent/opentelemetry-javaagent/1.18.0/opentelemetry-javaagent-1.18.0.jar -O ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.18.0.jar +if [ ! -f ${software_root}/accumulo/accumulo-${accumulo_version}/lib/opentelemetry-javaagent-1.19.0.jar ]; then + wget https://searc
[accumulo-testing] branch main updated: More carefully update license headers
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new 6d6f4e2 More carefully update license headers 6d6f4e2 is described below commit 6d6f4e2cf1d64f44fc7912ec5042592c4e4a7a98 Author: Christopher Tubbs AuthorDate: Fri Oct 14 12:05:11 2022 -0400 More carefully update license headers Fix issues with #244 by more carefully updating license headers in Terraform templates Remove problematic, or potentially problematic (and definitely unnecessary) license headers in cloud-init, genders, bash profile, and hosts Terraform template files --- .../cloud-init-config/templates/cloud-init.tftpl | 19 --- .../config-files/templates/core-site.xml.tftpl | 39 +++--- .../modules/config-files/templates/genders.tftpl | 19 --- .../templates/hadoop_bash_profile.tftpl| 19 --- .../config-files/templates/hadoop_bashrc.tftpl | 19 --- .../config-files/templates/hdfs-site.xml.tftpl | 39 +++--- .../modules/config-files/templates/hosts.tftpl | 19 --- .../templates/initialize_accumulo.sh.tftpl | 3 +- .../templates/initialize_hadoop.sh.tftpl | 3 +- .../config-files/templates/install_sw.sh.tftpl | 3 +- .../config-files/templates/yarn-site.xml.tftpl | 39 +++--- pom.xml| 4 ++- 12 files changed, 66 insertions(+), 159 deletions(-) diff --git a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl index f7932db..0593b10 100644 --- a/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/cloud-init-config/templates/cloud-init.tftpl @@ -1,22 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - #cloud-config # # Don't log key information diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl index 7ec0e32..7df9093 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/core-site.xml.tftpl @@ -1,24 +1,25 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - + fs.defaultFS diff --git a/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl b/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl index 9747cb8..15d2043 100644 --- a/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl +++ b/contrib/terraform-testing-infrastructure/modules/config-files/templates/genders.tftpl @@ -1,22 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License")
[accumulo] branch main updated (4df8ae2640 -> 6c8a4088dd)
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4df8ae2640 Update several dependencies to latest versions (#3022) add 6c8a4088dd Change List to SortedSet in RecoveryLogsIterator (#3021) No new revisions were added by this update. Summary of changes: .../accumulo/tserver/log/RecoveryLogsIterator.java | 16 +++- 1 file changed, 11 insertions(+), 5 deletions(-)
[accumulo] branch main updated: Make use of GitHub Actions environment files (#3024)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/main by this push: new 04ac8b222d Make use of GitHub Actions environment files (#3024) 04ac8b222d is described below commit 04ac8b222d866fab2552087cbffa150c2d402a3c Author: Christopher Tubbs AuthorDate: Fri Oct 14 16:52:49 2022 -0400 Make use of GitHub Actions environment files (#3024) set-output was previously used to create the IT matrix, but it is now deprecated. So, this change makes use of the environment files instead. See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#environment-files Note: the emitted key=value should not quote the value, or else it will be parsed as a String instead of a JSON type. This took me so long to figure out, so I'm sharing it here. Also, if you change the name of the output property, make sure you update the mapping of the step output to the job output to use the new name. --- .github/workflows/maven-full-its.yaml | 2 +- contrib/ci/it-matrix.sh | 42 --- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.github/workflows/maven-full-its.yaml b/.github/workflows/maven-full-its.yaml index 047ded2fee..a70eea19a7 100644 --- a/.github/workflows/maven-full-its.yaml +++ b/.github/workflows/maven-full-its.yaml @@ -72,7 +72,7 @@ jobs: needs: fastbuild timeout-minutes: 5 outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + matrix: ${{ steps.set-matrix.outputs.CUSTOM_MATRIX }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/contrib/ci/it-matrix.sh b/contrib/ci/it-matrix.sh index f84c93fc57..6e0f809ce6 100755 --- a/contrib/ci/it-matrix.sh +++ b/contrib/ci/it-matrix.sh @@ -22,16 +22,32 @@ testsPerJob=15 if [[ -n $1 && $1 =~ ^[0-9]*$ ]]; then testsPerJob=$1 fi -echo "Creating matrix (tests per job: $testsPerJob)..." - -gitRootDir=$(git rev-parse --show-toplevel) -# this only works because our test paths don't have spaces; we should keep it that way -count=0 -echo -n '::set-output name=matrix::{"profile":[' -for x in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java \; | sort -u | xargs -n "$testsPerJob" | tr ' ' ','); do - [[ $count -gt 0 ]] && echo -n ',' - echo -n "{\"name\":\"task_$count\",\"its\":\"$x\"}" - ((count = count + 1)) -done -echo ']}' -echo "Finished creating matrix ($count tasks)" + +# set these to /dev/null if they aren't defined in the environment +GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}" +GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/null}" + +function createTestMatrix() { + + local count=0 + local chunk=$1 + local batch + local gitRootDir + + { echo "Creating matrix (tests per job: $chunk)..." | tee -a "$GITHUB_STEP_SUMMARY"; } 1>&2 + + gitRootDir=$(git rev-parse --show-toplevel) + + # this only works because our test paths don't have spaces; we should keep it that way + echo -n '{"profile":[' + for batch in $(find "$gitRootDir" -name '*IT.java' -exec basename '{}' .java \; | sort -u | xargs -n "$chunk" | tr ' ' ','); do +[[ $count -gt 0 ]] && echo -n ',' +echo -n '{"name":"task_'"$count"'","its":"'"$batch"'"}' +((count = count + 1)) + done + echo ']}' + + { echo "Finished creating matrix ($count tasks)" | tee "$GITHUB_STEP_SUMMARY"; } 1>&2 +} + +echo "CUSTOM_MATRIX=$(createTestMatrix "$testsPerJob")" | tee -a "$GITHUB_OUTPUT"