[
https://issues.apache.org/jira/browse/HADOOP-19858?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18072086#comment-18072086
]
ASF GitHub Bot commented on HADOOP-19858:
-----------------------------------------
ajfabbri commented on code in PR #8412:
URL: https://github.com/apache/hadoop/pull/8412#discussion_r3054020649
##########
.github/workflows/tmpl_build_and_test.yml:
##########
@@ -0,0 +1,165 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Build and Test
+
+on:
+ workflow_call:
+ inputs:
+ java:
+ required: false
+ type: string
+ default: 17
+ branch:
+ required: false
+ type: string
+ description: Branch to run the build against
+ default: trunk
+ os:
+ required: false
+ type: string
+ description: Operating system to run the build on
+ default: ubuntu_24
+ jobs:
+ required: true
+ type: string
+ description: >-
+ Jobs to run, and should be in JSON Array format.
+ Candidates: "build-only".
+ default: '[]'
+
+concurrency:
+ group: build-and-test-${{ github.workflow }}-${{ github.head_ref ||
github.run_id }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_ARGS:
+ --batch-mode
+ --no-transfer-progress
+ -Pyarn-ui
+ -Pnative
+ -Drequire.test.libhadoop
+ -Drequire.fuse
+ -Drequire.openssl
+ -Drequire.snappy
+ -Drequire.valgrind
+ -Dmaven.test.failure.ignore=false
+
+jobs:
+ precondition:
+ name: Preparation
+ runs-on: ubuntu-24.04
+ outputs:
+ infra_image_url: ${{ steps.variables.outputs.infra_image_url }}
+ steps:
+ - name: Set up Outputs
+ id: variables
+ run: |
+ echo "infra_image_url=ghcr.io/hadoop/hadoop/gha-${{ inputs.os }}:${{
inputs.branch }}" >> $GITHUB_OUTPUT
+ infra-image:
+ name: Infra Image ${{ inputs.os }}-${{ inputs.branch }}
+ runs-on: ubuntu-24.04
+ needs: [ precondition ]
+ permissions:
+ packages: write
+ outputs:
+ uid: ${{ steps.variables.outputs.uid }}
+ steps:
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #
v4.0.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Checkout Hadoop repository
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+ repository: apache/hadoop
+ ref: ${{ inputs.branch }}
+ - name: Sync the current branch with the latest in Apache Hadoop
+ if: github.repository != 'apache/hadoop'
+ run: |
+ echo "APACHE_HADOOP_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
+ git fetch https://github.com/$GITHUB_REPOSITORY.git
${GITHUB_REF#refs/heads/}
+ git -c user.name='Apache Hadoop GitHub Actions Account' -c
user.email='[email protected]' merge --no-commit --progress --squash
FETCH_HEAD
+ git -c user.name='Apache Hadoop GitHub Actions Account' -c
user.email='[email protected]' commit -m "Merged commit" --allow-empty
+ - name: Set up Docker Buildx
+ uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+ - name: Build and push ${{ inputs.os }} base infra image for ${{
inputs.branch }}
+ id: docker_build_base
+ uses:
docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+ with:
+ context: ./dev-support/docker/
+ file: ./dev-support/docker/Dockerfile_${{ inputs.os }}
+ push: true
+ tags: ${{ needs.precondition.outputs.infra_image_url }}-base
+ - name: User-specific Dockerfile
+ run: |
+ USER_ID=$(id -u "${USER}")
+ GROUP_ID=$(id -g "${USER}")
+ cat > /tmp/Dockerfile.gha <<UserSpecificDocker
+ FROM ${{ needs.precondition.outputs.infra_image_url }}-base
+ RUN rm -f /var/log/faillog /var/log/lastlog
+ RUN groupadd --non-unique -g ${GROUP_ID} ${USER}
+ RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER}
+ RUN echo "${USER} ALL=NOPASSWD: ALL" >
"/etc/sudoers.d/hadoop-build-${USER_ID}"
+ UserSpecificDocker
+ - name: Build and push ${{ inputs.os }} infra image for ${{
inputs.branch }}
+ id: docker_build_gha
+ uses:
docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+ with:
+ context: ./dev-support/docker/
+ file: /tmp/Dockerfile.gha
+ push: true
+ tags: ${{ needs.precondition.outputs.infra_image_url }}
+ - name: Set up Outputs
+ id: variables
+ run: |
+ echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT
+ build-only:
+ name: Build Only (Java ${{ inputs.java }}) ${{ inputs.os }}-${{
inputs.branch }}
+ if: (!cancelled()) && contains(fromJSON(inputs.jobs), 'build-only')
+ runs-on: ubuntu-24.04
+ needs: [ precondition, infra-image ]
+ container:
+ image: ${{ needs.precondition.outputs.infra_image_url }}
+ options: --user ${{ needs.infra-image.outputs.uid }}
+ steps:
+ - name: Checkout Hadoop repository
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+ repository: apache/hadoop
+ ref: ${{ inputs.branch }}
+ - name: Sync the current branch with the latest in Apache Hadoop
Review Comment:
Same question here. Can we skip this step?
##########
.github/workflows/tmpl_build_and_test.yml:
##########
@@ -0,0 +1,166 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Build and Test
+
+on:
+ workflow_call:
+ inputs:
+ java:
+ required: false
+ type: string
+ default: 17
+ branch:
+ required: false
+ type: string
+ description: Branch to run the build against
+ default: trunk
+ os:
+ required: false
+ type: string
+ description: Operating system to run the build on
+ default: ubuntu_24
+ jobs:
+ required: true
+ type: string
+ description: >-
+ Jobs to run, and should be in JSON Array format.
+ Candidates: "build-only".
+ default: '[]'
+
+concurrency:
+ group: build-and-test-${{ github.workflow }}-${{ github.head_ref ||
github.run_id }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_ARGS:
+ --batch-mode
+ --no-transfer-progress
+ -Pyarn-ui
+ -Pnative
+ -Drequire.test.libhadoop
+ -Drequire.fuse
+ -Drequire.openssl
+ -Drequire.snappy
+ -Drequire.valgrind
+ -Dmaven.test.failure.ignore=false
+
+jobs:
+ precondition:
+ name: Preparation
+ runs-on: ubuntu-24.04
+ outputs:
+ infra_image_url: ${{ steps.variables.outputs.infra_image_url }}
+ steps:
+ - name: Set up Outputs
+ id: variables
+ run: |
+ echo "infra_image_url=ghcr.io/hadoop/hadoop/gha-${{ inputs.os }}:${{
inputs.branch }}" >> $GITHUB_OUTPUT
+ infra-image:
+ name: Infra Image ${{ inputs.os }}-${{ inputs.branch }}
+ runs-on: ubuntu-24.04
+ needs: [ precondition ]
+ permissions:
+ packages: write
+ outputs:
+ uid: ${{ steps.variables.outputs.uid }}
+ steps:
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #
v4.0.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Checkout Hadoop repository
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+ repository: apache/hadoop
+ ref: ${{ inputs.branch }}
+ - name: Sync the current branch with the latest in Apache Hadoop
+ if: github.repository != 'apache/hadoop'
+ run: |
+ echo "APACHE_HADOOP_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
+ git fetch https://github.com/$GITHUB_REPOSITORY.git
${GITHUB_REF#refs/heads/}
+ git -c user.name='Apache Hadoop GitHub Actions Account' -c
user.email='[email protected]' merge --no-commit --progress --squash
FETCH_HEAD
+ git -c user.name='Apache Hadoop GitHub Actions Account' -c
user.email='[email protected]' commit -m "Merged commit" --allow-empty
+ - name: Set up Docker Buildx
+ uses:
docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+ - name: Build and push ${{ inputs.os }} base infra image for ${{
inputs.branch }}
+ id: docker_build_base
+ uses:
docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+ with:
+ context: ./dev-support/docker/
+ file: ./dev-support/docker/Dockerfile_${{ inputs.os }}
+ push: true
+ tags: ${{ needs.precondition.outputs.infra_image_url }}-base
+ - name: User-specific Dockerfile
Review Comment:
Ah that is unfortunate. So in GH Runner, does this attempt to create the
root user? Is that necessary? Is the idea to support both root and non-root
users? (We can do non-root with self-hosted runners, which I may try out soon.)
##########
.github/workflows/tmpl_build_and_test.yml:
##########
@@ -0,0 +1,165 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Build and Test
+
+on:
+ workflow_call:
+ inputs:
+ java:
+ required: false
+ type: string
+ default: 17
+ branch:
+ required: false
+ type: string
+ description: Branch to run the build against
+ default: trunk
+ os:
+ required: false
+ type: string
+ description: Operating system to run the build on
+ default: ubuntu_24
+ jobs:
+ required: true
+ type: string
+ description: >-
+ Jobs to run, and should be in JSON Array format.
+ Candidates: "build-only".
+ default: '[]'
+
+concurrency:
+ group: build-and-test-${{ github.workflow }}-${{ github.head_ref ||
github.run_id }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_ARGS:
+ --batch-mode
+ --no-transfer-progress
+ -Pyarn-ui
+ -Pnative
+ -Drequire.test.libhadoop
+ -Drequire.fuse
+ -Drequire.openssl
+ -Drequire.snappy
+ -Drequire.valgrind
+ -Dmaven.test.failure.ignore=false
+
+jobs:
+ precondition:
+ name: Preparation
+ runs-on: ubuntu-24.04
+ outputs:
+ infra_image_url: ${{ steps.variables.outputs.infra_image_url }}
+ steps:
+ - name: Set up Outputs
+ id: variables
+ run: |
+ echo "infra_image_url=ghcr.io/hadoop/hadoop/gha-${{ inputs.os }}:${{
inputs.branch }}" >> $GITHUB_OUTPUT
+ infra-image:
+ name: Infra Image ${{ inputs.os }}-${{ inputs.branch }}
+ runs-on: ubuntu-24.04
+ needs: [ precondition ]
+ permissions:
+ packages: write
+ outputs:
+ uid: ${{ steps.variables.outputs.uid }}
+ steps:
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 #
v4.0.0
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ - name: Checkout Hadoop repository
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+ repository: apache/hadoop
+ ref: ${{ inputs.branch }}
+ - name: Sync the current branch with the latest in Apache Hadoop
Review Comment:
I don't understand this step. Does it only run on fork repositories?
Also I wouldn't want an automatic "back merge"--I prefer to rebase when
possible (i.e. I prefer to follow linux [kernel guidance on
git](https://www.kernel.org/doc/html/latest/maintainer/rebasing-and-merging.html).
> Set up build workflow in GitHub Actions
> ---------------------------------------
>
> Key: HADOOP-19858
> URL: https://issues.apache.org/jira/browse/HADOOP-19858
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: build
> Reporter: Cheng Pan
> Priority: Major
> Labels: pull-request-available
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
