This is an automated email from the ASF dual-hosted git repository.
ajfabbri pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ed29a6a75e1 HADOOP-19877: run s3a integration and scale tests in CI
(#8467)
ed29a6a75e1 is described below
commit ed29a6a75e17a9f2ae13aea620f20f2c094e165b
Author: Aaron Fabbri <[email protected]>
AuthorDate: Thu May 14 14:08:23 2026 -0700
HADOOP-19877: run s3a integration and scale tests in CI (#8467)
- create composite actions for creating builder image
- Trigger when haddop-aws code or related workflows change
- Specify localstack container in workflow `services:` instead of using the
3rd-party setup-localstack action; it had more issues than it was worth.
- Separate container OS vs. runner OS params
- Pass localstack license (dontated) CI key secret through to workflow
- Faster build; skip javadoc
- Add s3a test excludes
- Don't use path-style access for s3
- Sanity check local maven repo path
- Fix cache path issue: container vs host path
- Try to skip waiting for localstack readiness
- Actions cache is immutable, use accordingly.
- Consolodate common MAVEN_OPTS
- Address broken mvn verify return status..
- Update test-reporter action and filter artifact download
- Rename test exclude file, localstack-specific..
- Update aws integration test reporting workflow
- Consistent workflow name report_foo.yml
- Fix test report to avoid parsing failsafe summary
- Mind cancellations, tweak report workflow
- Use commit status to improve report usability
- Naming tweaks: concise and consistent
- Rename input to `toolchain_branch` for clarity
- Remove unnecessary `ports:` section for localstack service.
- Tweak s3a report name / context
* ci: add missing apache headers, fix typo in test excludes
* ci: pin localstack version
* ci/aws: remove checks: write permission
---
.github/actions/build_image/action.yml | 80 +++++++
.github/actions/build_image_url/action.yml | 44 ++++
.../gha-tests/hadoop-aws-localstack-excludes.txt | 79 +++++++
.github/workflows/cloud_aws.yml | 44 ++++
.github/workflows/report_cloud_aws.yml | 88 ++++++++
.github/workflows/templates/auth-keys.xml.tmpl | 63 ++++++
.github/workflows/tmpl_build_and_test.yml | 1 +
.github/workflows/tmpl_cloud_aws.yml | 230 +++++++++++++++++++++
8 files changed, 629 insertions(+)
diff --git a/.github/actions/build_image/action.yml
b/.github/actions/build_image/action.yml
new file mode 100644
index 00000000000..6044416117e
--- /dev/null
+++ b/.github/actions/build_image/action.yml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Create hadoop build image
+
+inputs:
+ branch:
+ description: Git branch to use.
+ required: true
+ os:
+ description: Runner OS name
+ required: true
+ build_image_url:
+ description: URL of build (infra) image.
+ required: true
+
+outputs:
+ uid:
+ description: User ID this action ran as.
+ value: ${{ steps.variables.outputs.uid }}
+
+runs:
+ using: composite
+ steps:
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #
v4.0.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ github.token }}
+ - name: Set up Docker Buildx
+ uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+ - name: Build and push ${{ inputs.os }} base build image for ${{
inputs.branch }}
+ id: docker_build_base
+ uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294
# v7.0.0
+ with:
+ context: ./dev-support/docker/
+ file: ./dev-support/docker/Dockerfile_${{ inputs.os }}
+ push: true
+ tags: ${{ inputs.build_image_url }}-base
+ cache-from: type=registry,ref=ghcr.io/apache/hadoop/gha-build-${{
inputs.os }}-image-cache:${{ inputs.branch }}
+ - name: User-specific Dockerfile
+ shell: bash
+ run: |
+ USER_ID=$(id -u "${USER}")
+ GROUP_ID=$(id -g "${USER}")
+ cat > /tmp/Dockerfile.gha <<UserSpecificDocker
+ FROM ${{ inputs.build_image_url }}-base
+ RUN rm -f /var/log/faillog /var/log/lastlog
+ RUN groupadd --non-unique -g ${GROUP_ID} ${USER}
+ RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER}
+ RUN echo "${USER} ALL=NOPASSWD: ALL" >
"/etc/sudoers.d/hadoop-build-${USER_ID}"
+ UserSpecificDocker
+ - name: Build and push ${{ inputs.os }} build image for ${{ inputs.branch
}}
+ id: docker_build_gha
+ uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294
# v7.0.0
+ with:
+ context: ./dev-support/docker/
+ file: /tmp/Dockerfile.gha
+ push: true
+ tags: ${{ inputs.build_image_url }}
+ - name: Set up Outputs
+ id: variables
+ shell: bash
+ run: |
+ echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT
diff --git a/.github/actions/build_image_url/action.yml
b/.github/actions/build_image_url/action.yml
new file mode 100644
index 00000000000..fc253e02c84
--- /dev/null
+++ b/.github/actions/build_image_url/action.yml
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: Get Build Image URL
+description:
+
+inputs:
+ branch:
+ description: Git branch to use.
+ required: true
+ os:
+ description: Operating system to run the build on
+ required: true
+
+outputs:
+ build_image_url:
+ description: URL of build (infra) image.
+ value: ${{ steps.compute.outputs.image_url }}
+
+runs:
+ using: composite
+ steps:
+ - id: compute
+ shell: bash
+ run: |
+ # Convert to lowercase to meet Docker repo name requirement
+ REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]'
'[:lower:]')
+ OS=$(echo "${{ inputs.os }}" | tr '[:upper:]' '[:lower:]')
+ echo "image_url=ghcr.io/${REPO_OWNER}/gha-build-${OS}:${{
inputs.branch }}-${{ github.run_id }}" \
+ >> $GITHUB_OUTPUT
+
diff --git a/.github/gha-tests/hadoop-aws-localstack-excludes.txt
b/.github/gha-tests/hadoop-aws-localstack-excludes.txt
new file mode 100644
index 00000000000..d16ff8957a4
--- /dev/null
+++ b/.github/gha-tests/hadoop-aws-localstack-excludes.txt
@@ -0,0 +1,79 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Initial stab at excluding tests that won't run in our
+# CI container environment with Localstack mocks of AWS services.
+
+# TODO see if we can enable any of these...
+
+# tests that depend on public S3 buckets
+**/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
+**/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java
+**/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
+**/org/apache/hadoop/fs/s3a/tools/ITestMarkerTool.java
+**/ITestS3AAnalyticsAcceleratorStreamReading.java
+**/ITestS3AEndpointRegion.java
+
+
+# Tests requiring IAM roles / STS
+# We should be able to re-enable some of these. See:
+# https://docs.localstack.cloud/aws/services/sts/
+**/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java
+**/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
+**/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFilesystem.java
+**/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java
+**/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
+
+
+# failures that need to be investigated
+
+# Two methods fail: 1. testUpdateDeepDirectoryStructureNoChange():
+# AssertionFailedError: Files Skipped value 0 too below minimum 1 ==>
+# expected: <true> but was: <false>
+# 2. testUpdateDeepDirectoryStructureToRemote():
+# AssertionFailedError: Files Copied value 2 above maximum 1 ==> expected:
<true> but was: <false>
+**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
+
+# A number of failures with vectored read tests
+**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java
+
+# Access key errors:
+# (test
case)->AbstractS3ATestBase.setup:111->AbstractFSContractTestBase.setup:197->AbstractFSContractTestBase.mkdirs:355
+# ยป AccessDenied s3a://hadoop-ci/job-00/test: getFileStatus on
+# s3a://hadoop-ci/job-00/test:
+# software.amazon.awssdk.services.s3.model.S3Exception: The AWS Access Key Id
you
+# provided does not exist in our records. (Service: S3, Status Code: 403
+**/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
+**/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
+
+# Localstack issue (guessing lack of persistence of upload parts across
sessions)
+**/org/apache/hadoop/fs/s3a/commit/ITestUploadRecovery.java
+**/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMultipartUploader.java
+
+# Error: ITestConnectionTimeouts.testObjectUploadTimeouts:258 Expected a
+# java.lang.Exception to be thrown, but got the result: :
"01234567890123456789..."
+**/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java
+
+#
ITestS3AAWSCredentialsProvider.testBadCredentials:183->lambda$testBadCredentials$0:184
+# ->createFailingFS:160 Expected exception - got S3AFileSystem{
+**/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
+
+# testSeeksWithLruEviction java.util.concurrent.TimeoutException: timed out
+# after 180 seconds
+**/org/apache/hadoop/fs/s3a/ITestS3APrefetchingLruEviction.java
diff --git a/.github/workflows/cloud_aws.yml b/.github/workflows/cloud_aws.yml
new file mode 100644
index 00000000000..7d48413fb0d
--- /dev/null
+++ b/.github/workflows/cloud_aws.yml
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Cloud-AWS"
+
+on:
+ pull_request:
+ paths:
+ - 'hadoop-tools/hadoop-aws/**'
+ - '.github/workflows/*cloud_aws.yml'
+ - '.github/actions/build_image**'
+ - '.github/gha-tests/hadoop-aws*excludes.txt'
+
+jobs:
+ run-aws-integration:
+ # Security: write privileges are needed to update PR status and upload
test results.
+ # Package write is for building toolchain container images on demand, but
ghcr.io access is
+ # scoped to the repository the actions run on.
+ permissions:
+ packages: write
+ contents: read
+ name: Run
+ uses: ./.github/workflows/tmpl_cloud_aws.yml
+ secrets: inherit # inherit LOCALSTACK_CI_KEY
+ with:
+ java: 25
+ os: ubuntu_24
+ runner_os: ubuntu-24.04
diff --git a/.github/workflows/report_cloud_aws.yml
b/.github/workflows/report_cloud_aws.yml
new file mode 100644
index 00000000000..5ea9037dda3
--- /dev/null
+++ b/.github/workflows/report_cloud_aws.yml
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Cloud-AWS Test Summary
+
+on:
+ workflow_run:
+ workflows: ["Cloud-AWS"]
+ types:
+ - completed
+
+permissions:
+ contents: read
+ actions: read
+ checks: write
+ statuses: write
+
+jobs:
+ report:
+ runs-on: ubuntu-latest
+ if: github.event.workflow_run.event == 'pull_request'
+ steps:
+ - name: S3A Test Report
+ id: report
+ uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 #
v3.0.0
+ with:
+ artifact: /s3a-test-reports-java-.*/ # see tmpl_cloud_aws.yml
+ name: S3A Integration Tests
+ path: "**/TEST-*.xml"
+ reporter: java-junit
+ only-summary: 'false'
+ # use-actions-summary is currently mutually exclusive with check run
+ # creation in dorny/test-reporter.:
+ # For now, try check run method.
+ use-actions-summary: 'false'
+ list-suites: 'failed'
+ list-tests: 'failed'
+ # list-files: 'none'
+ fail-on-error: 'false'
+
+ - name: Post S3A commit status
+ uses: actions/github-script@v7
+ if: always()
+ with:
+ script: |
+ const conclusion = '${{ steps.report.outputs.conclusion }}';
+ const passed = '${{ steps.report.outputs.passed }}';
+ const failed = '${{ steps.report.outputs.failed }}';
+ const skipped = '${{ steps.report.outputs.skipped }}';
+ const url = '${{ steps.report.outputs.url_html }}';
+
+ // Map dorny/test-reporter conclusion to a GitHub commit-status
state.
+ let state, description;
+ if (conclusion === 'failure') {
+ state = 'failure';
+ description = `${passed} pass, ${failed} fail, ${skipped} skip`;
+ } else if (conclusion === 'success') {
+ state = 'success';
+ description = `${passed} pass, ${failed} fail, ${skipped} skip`;
+ } else {
+ state = 'error';
+ description = 'Test report unavailable (no results found or
reporter error)';
+ }
+
+ const params = {
+ ...context.repo,
+ sha: context.payload.workflow_run.head_sha,
+ state,
+ description,
+ context: 'S3A Test Summary',
+ };
+ if (url) params.target_url = url;
+
+ await github.rest.repos.createCommitStatus(params);
diff --git a/.github/workflows/templates/auth-keys.xml.tmpl
b/.github/workflows/templates/auth-keys.xml.tmpl
new file mode 100644
index 00000000000..c9a5cb2c91b
--- /dev/null
+++ b/.github/workflows/templates/auth-keys.xml.tmpl
@@ -0,0 +1,63 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+
+<!-- Localstack config for CI -->
+<configuration>
+ <property>
+ <name>test.fs.s3a.name</name>
+ <value>s3a://hadoop-ci/</value>
+ </property>
+ <property>
+ <name>fs.contract.test.fs.s3a</name>
+ <value>s3a://hadoop-ci/</value>
+ </property>
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>test</value>
+ </property>
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>test</value>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <!-- hostname alias and /etc/hosts entry configured in tmpl_cloud_aws.yml
-->
+ <value>http://s3.localstack:4566</value>
+ </property>
+ <!-- localstack needs path style access, or virtual-hosted style (above)
+ <property>
+ <name>fs.s3a.path.style.access</name>
+ <value>true</value>
+ </property>
+ -->
+ <property>
+ <name>fs.s3a.connection.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.s3a.endpoint.region</name>
+ <value>us-west-2</value>
+ </property>
+ <property>
+ <!-- causes CORS failures with localstack by deafult -->
+ <name>fs.s3a.bucket.abd-bucket.audit.enabled</name>
+ <value>false</value>
+ </property>
+</configuration>
diff --git a/.github/workflows/tmpl_build_and_test.yml
b/.github/workflows/tmpl_build_and_test.yml
index 155ad60f392..7df87b2428a 100644
--- a/.github/workflows/tmpl_build_and_test.yml
+++ b/.github/workflows/tmpl_build_and_test.yml
@@ -72,6 +72,7 @@ env:
-Dsurefire.rerunFailingTestsCount=2
jobs:
+ # TODO: use common composite actions in .github/actions/
precondition:
name: Preparation
runs-on: ubuntu-24.04
diff --git a/.github/workflows/tmpl_cloud_aws.yml
b/.github/workflows/tmpl_cloud_aws.yml
new file mode 100644
index 00000000000..18edf2bab3a
--- /dev/null
+++ b/.github/workflows/tmpl_cloud_aws.yml
@@ -0,0 +1,230 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: s3a integration
+on:
+ workflow_call:
+ inputs:
+ java:
+ required: false
+ type: string
+ default: 17
+ toolchain_branch:
+ required: false
+ type: string
+ description: Branch to use for toolchain image build
+ default: trunk
+ os:
+ required: false
+ type: string
+ description: OS for container to run the build in
+ default: ubuntu_24
+ runner_os:
+ required: false
+ type: string
+ description: OS tag for runner (e.g., Linux, ubuntu-24.04)
+ default: ubuntu_24.04
+
+# Security: Minimal defaults for workflow.
+permissions: {}
+
+concurrency:
+ group: >-
+ cloud-aws
+ ${{ github.workflow }}
+ ${{ github.repository == 'apache/hadoop' && github.run_id || github.ref }}
+ ${{ inputs.java }}
+ ${{ inputs.toolchain_branch }}
+ ${{ inputs.os }}
+ cancel-in-progress: true
+
+env:
+ BUCKET_NAME: hadoop-ci
+
+jobs:
+ precondition:
+ runs-on: ${{ inputs.runner_os }}
+ outputs:
+ build_image_url: ${{ steps.img.outputs.build_image_url }}
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ # Full fetch so build image URL can be computed for any branch
+ fetch-depth: 0
+ - uses: ./.github/actions/build_image_url
+ id: img
+ with:
+ os: ${{ inputs.os }}
+ branch: ${{ inputs.toolchain_branch }}
+ - name: debug base_image_url
+ run: |
+ echo "precondition url: ${{ steps.img.outputs.build_image_url }}"
+
+ build-image:
+ name: Toolchain image (JDK${{ inputs.java }}, ${{ inputs.os }}-${{
inputs.toolchain_branch }})
+ runs-on: ${{ inputs.runner_os }}
+ needs: [ precondition ]
+ permissions:
+ packages: write
+ outputs:
+ uid: ${{ steps.build_img.outputs.uid }}
+ steps:
+ - name: debug build url
+ run: |
+ echo "Build image URL: ${{
needs.precondition.outputs.build_image_url }}"
+ - uses: actions/checkout@v6
+ - uses: ./.github/actions/build_image
+ id: build_img
+ with:
+ branch: ${{ inputs.toolchain_branch }}
+ os: ${{ inputs.os }}
+ build_image_url: ${{ needs.precondition.outputs.build_image_url }}
+
+ test:
+ name: S3A Integration Tests (Java ${{ inputs.java }})
+ needs: [ precondition, build-image ]
+ runs-on: ${{ inputs.runner_os }}
+ permissions:
+ # Security: Minimal permissions for the test runner. Reporting happens in
+ # report_cloud_aws.yml.
+ contents: read
+ services:
+ localstack:
+ image: localstack/localstack:2026.04.2
+ # Despite examples showing a `ports:` section, "You don't need to
+ # configure any ports for service containers. By default, all
+ # containers that are part of the same Docker network expose all ports
+ # to each other, and no ports are exposed outside of the Docker
+ # network." See:
+ #
https://docs.github.com/en/actions/tutorials/use-containerized-services/use-docker-service-containers#running-jobs-in-a-container
+ env:
+ SERVICES: s3,kms
+ AWS_DEFAULT_REGION: us-west-2
+ AWS_ACCESS_KEY_ID: test
+ AWS_SECRET_ACCESS_KEY: test
+ LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_CI_KEY }}
+ LOCALSTACK_HOST: s3.localstack
+
+ # Performance: Disable image's health check (localstack readiness): it
typically takes less
+ # than a minute, and the Maven build that runs first takes longer than
that.
+ # Also need to specify a dummy health-cmd or the github runner fails.
+ options: >-
+ --health-cmd "exit 0"
+ --health-interval 1s
+ --health-retries 1
+ --network-alias s3.localstack
+
+ container:
+ image: ${{ needs.precondition.outputs.build_image_url }}
+ options: >-
+ --user ${{ needs.build-image.outputs.uid }}
+ env:
+ # mvn verify doesn't return failure exit code due to HADOOP-18040
+ # (which seems incorrect, but let's just override this for now)
+ MAVEN_OPTS: >-
+ -Dmaven.test.failure.ignore=false
+ -Dmaven.repo.local=.m2/repository
+ -Dcheckstyle.skip -Dspotbugs.skip -Denforcer.skip -Drat.skip
+ steps:
+ - uses: actions/checkout@v6
+ # Performance: Caching TODO: We need to create a centralized maven build
cache that is
+ # built on trunk. This will always miss on a new PR: Caches can't be
+ # shared between PR branches. PR branches *can* access caches from their
+ # base branch, though. See:
+ #
https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
+ # As-is, first run on a PR always misses. Subsequent cached builds see
>100% speedup.
+ - name: Restore build cache
+ id: restore_cache
+ uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae #
v5.0.5
+ with:
+ path: |
+ .m2/repository/
+ key: ${{ inputs.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{
inputs.toolchain_branch }}
+ # falls back to less-specific caches if exact key not found
+ restore-keys: |
+ ${{ inputs.os }}-maven-${{ hashFiles('**/pom.xml') }}-
+ ${{ inputs.os }}-maven-
+ - name: Setup JDK ${{ inputs.java }}
+ uses: actions/setup-java@v5
+ with:
+ distribution: zulu
+ java-version: ${{ inputs.java }}
+
+ - name: Maven Build
+ shell: bash
+ run: |
+ mkdir -p .m2/repository
+ ./mvnw -B -T 1C --no-transfer-progress -DskipTests -am -pl
hadoop-tools/hadoop-aws \
+ -Dmaven.javadoc.skip -Dcheckstyle.skip -Dspotbugs.skip
-Denforcer.skip -Drat.skip install
+
+ - name: Check local maven repository
+ shell: bash
+ run: |
+ if [ -d ".m2/repository" ]; then
+ echo "๐ Maven repository at $(pwd)/.m2/repository"
+ else
+ echo "๐ .m2/repository not found in $(pwd)"
+ exit 1
+ fi
+
+ - name: Save build cache
+ uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae #
v5.0.5
+ if: ${{ !cancelled() && steps.restore_cache.outputs.cache-hit !=
'true' }}
+ with:
+ path: |
+ .m2/repository/
+ key: ${{ steps.restore_cache.outputs.cache-primary-key }}
+
+ - name: Create localstack s3 bucket
+ shell: bash
+ run: |
+ # Use curl to interact with localstack if awslocal is not in the
container
+ curl -X PUT "http://localstack:4566/${{ env.BUCKET_NAME }}"
+
+ - name: Register bucket hostname
+ shell: bash
+ run: |
+ # Resolve s3.localstack IP and add entry for virtual-hosted bucket
FQDN
+ LSIP=$(getent hosts s3.localstack | awk '{print $1}')
+ echo "Registering ${{ env.BUCKET_NAME }}.s3.localstack as ${LSIP}"
+ echo "${LSIP} ${{ env.BUCKET_NAME }}.s3.localstack" | sudo tee -a
/etc/hosts
+
+ - name: S3A Integration Tests
+ shell: bash
+ env:
+ AWS_ACCESS_KEY_ID: test
+ AWS_SECRET_ACCESS_KEY: test
+ AWS_REGION: us-west-2
+ AWS_DEFAULT_REGION: us-west-2
+ run: |
+ cp .github/workflows/templates/auth-keys.xml.tmpl \
+ hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
+ ./mvnw verify -B -pl hadoop-tools/hadoop-aws \
+ -Dparallel-tests \
+ -DskipITs=false \
+ -Dtest=none \
+ -Dscale=true \
+
-Dfailsafe.excludesFile=$PWD/.github/gha-tests/hadoop-aws-localstack-excludes.txt
+
+ - name: Upload Test Reports
+ uses: actions/upload-artifact@v4
+ if: ${{ !cancelled() }}
+ with:
+ name: s3a-test-reports-java-${{ inputs.java }}-${{ inputs.os }}
+ path: |
+ **/target/failsafe-reports/*.xml
+ **/target/surefire-reports/*.xml
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
