This is an automated email from the ASF dual-hosted git repository. msciabarra pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new f717619c8 GitHub action (#5361) f717619c8 is described below commit f717619c8c25b54294cfa195732732e85c5d9431 Author: Michele Sciabarra <mich...@nuvolaris.io> AuthorDate: Fri Dec 16 16:33:32 2022 +0100 GitHub action (#5361) * github action initial commit * fix blank * debug * fix push * fix java * add setup * triggering build * run unit test * ooops * updating debugAction * waitifdebug * fix cleanup * final test * disable ngrok * reworked the PR - multiple jobs and slack * split in multiple build * github actions updated * scancode fixes * forcing a corresponding version of jdk in ci and docker image and renaming S3_LOG_BUCKET to AWS_BUCKET to fix tests * more fixes - only one test failing against 1.0.0 * finally passing all the tests on 1.0.0 * restored master variables * restored more master files * triggered on push and pull request * unit test too * wskadmin fixes * improve message * dummy enabled * better git message * cleanup the log messages * unnecessary change to master - even if beneficial * improved messages (and a test disabled) * reverting commented test * get rid of DummyKO no more needed Co-authored-by: Michele Sciabarra <mich...@sciabarra.com> --- .github/workflows/0-on-demand.yaml | 92 ++++++++++++++++++++++ .github/workflows/1-unit.yaml | 70 ++++++++++++++++ .github/workflows/2-system.yaml | 70 ++++++++++++++++ .github/workflows/3-multi-runtime.yaml | 70 ++++++++++++++++ .github/workflows/4-standalone.yaml | 70 ++++++++++++++++ .github/workflows/5-scheduler.yaml | 70 ++++++++++++++++ .github/workflows/6-performance.yaml | 84 ++++++++++++++++++++ .github/workflows/README.md | 56 +++++++++++++ .gitignore | 4 + common/scala/Dockerfile | 3 +- tools/github/checkAndUploadLogs.sh | 59 ++++++++++++++ tools/github/debugAction.sh | 69 ++++++++++++++++ tools/github/flake8.sh | 42 ++++++++++ .../Dockerfile => tools/github/runDummyTests.sh | 21 ++--- .../github/runLeanSystemTests.sh | 27 ++++--- .../github/runMultiRuntimeTests.sh | 27 ++++--- .../github/runSchedulerTests.sh | 26 +++--- tools/github/runStandaloneTests.sh | 64 +++++++++++++++ .../Dockerfile => tools/github/runSystemTests.sh | 27 ++++--- .../Dockerfile => tools/github/runUnitTests.sh | 25 +++--- tools/github/s3-upload.sh | 53 +++++++++++++ tools/github/scan.sh | 52 ++++++++++++ tools/github/setup.sh | 79 +++++++++++++++++++ .../Dockerfile => tools/github/waitIfDebug.sh | 32 ++++---- .../Dockerfile => tools/github/writeOnSlack.sh | 23 +++--- 25 files changed, 1112 insertions(+), 103 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml new file mode 100644 index 000000000..5d8d66ea3 --- /dev/null +++ b/.github/workflows/0-on-demand.yaml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: On Demand Tests + +on: + workflow_dispatch: + inputs: + enable_ngrok_debug: + description: "Enable Ngrok Debugging" + required: true + type: boolean + default: false + test_suite: + description: Select Test Suite to run + type: choice + options: + - Unit + - System + - MultiRuntime + - Standalone + - Scheduler + - Performance + - Dummy + +env: + # openwhisk env + TEST_SUITE: ${{ inputs.test_suite }} + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + + # (optional) you need to add as secrets an ngrok token and a password to debug a build on demand + NGROK_DEBUG: ${{ inputs.enable_ngrok_debug }} + NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_PASSWORD: ${{ secrets.NGROK_PASSWORD }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} + - name: Debug Action (if requested) + run: ./tools/github/debugAction.sh + - name: Wait for Debug (if requested) + run: ./tools/github/waitIfDebug.sh + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml new file mode 100644 index 000000000..144720625 --- /dev/null +++ b/.github/workflows/1-unit.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Unit Tests + +on: + # build on push + push: + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Unit + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml new file mode 100644 index 000000000..9295f921f --- /dev/null +++ b/.github/workflows/2-system.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: System Tests + +on: + # build on push + push: + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: System + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml new file mode 100644 index 000000000..6bdcf7703 --- /dev/null +++ b/.github/workflows/3-multi-runtime.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: MultiRuntime Tests + +on: + # build on push + push: + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: MultiRuntime + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml new file mode 100644 index 000000000..92a9a7d82 --- /dev/null +++ b/.github/workflows/4-standalone.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Standalone Tests + +on: + # build on push + push: + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Standalone + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml new file mode 100644 index 000000000..cb9234f20 --- /dev/null +++ b/.github/workflows/5-scheduler.yaml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Scheduler Tests + +on: + # build on push + push: + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Scheduler + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - id: tests + name: Run Tests + run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" + continue-on-error: true + - id: logs + name: Upload Logs + run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nlogs:' ${{ steps.logs.outputs.logs }} + $'\nreport:' ${{ steps.logs.outputs.report }} + - name: Results + run: test "${{ steps.tests.outcome }}" == "success" diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml new file mode 100644 index 000000000..cde8c5fbf --- /dev/null +++ b/.github/workflows/6-performance.yaml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: Performance Tests + +on: + # build on push + push: + # build on pull requests + pull_request: + +env: + # openwhisk env + TEST_SUITE: Performance + ANSIBLE_CMD: "ansible-playbook -i environments/local -e docker_image_prefix=testing" + GRADLE_PROJS_SKIP: "" + + ## secrets + # (optional) slack incoming wehbook for notifications + SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} + + # (optional) s3 log upload + AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + + # github + GH_BUILD: ${{ github.event_name }}-${{ github.sha }} + GH_BRANCH: ${{ github.head_ref || github.ref_name }} + +jobs: + openwhisk: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: "Setup" + run: ./tools/github/setup.sh + - run: ./tests/performance/preparation/deploy.sh + - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 2m + continue-on-error: true + - run: TERM=dumb ./tests/performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 2m + continue-on-error: true + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 4 1 2 2m + continue-on-error: true + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 4 1 2 2m + continue-on-error: true + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/noop.js 100 110 2 2m + continue-on-error: true + - run: TERM=dumb ./tests/performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" ./tests/performance/preparation/actions/async.js 100 110 2 2m + continue-on-error: true + - run: OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ApiV1Simulation + continue-on-error: true + - run: OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" PAUSE_BETWEEN_INVOKES="100" ./gradlew gatlingRun-org.apache.openwhisk.LatencySimulation + continue-on-error: true + - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation + continue-on-error: true + - run: OPENWHISK_HOST="172.17.0.1" API_KEY="$(cat ansible/files/auth.guest)" CONNECTIONS="100" REQUESTS_PER_SEC="1" ASYNC="true" ./gradlew gatlingRun-org.apache.openwhisk.BlockingInvokeOneActionSimulation + continue-on-error: true + # The following configuration does not make much sense. But we do not have enough users. But it's good to verify, that the test is still working. + - run: OPENWHISK_HOST="172.17.0.1" USERS="1" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-org.apache.openwhisk.ColdBlockingInvokeSimulation + continue-on-error: true + - name: Slack Notification + run: > + ./tools/github/writeOnSlack.sh + "[$TEST_SUITE]" ${{ steps.tests.outcome }} on ${GH_BUILD} + $'\nbranch:' $GH_BRANCH + $'\nmessage:' "$(git log -1 --oneline | cat)" + $'\nCheck GitHub logs for results' diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..55473d04e --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,56 @@ +# How to use those workflows + +There are a few [GitHub secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) to configure to fully leverage the build. + +You can use and set the followings secrets also in your fork. + +## Ngrok Debugging + +You can debug a GitHub Action build using [NGROK](https://ngrok.com/). + +It is disabled for automated build triggered by push and pull_requests. + +You can trigger a workflow run manually enabling ngrok debugging. + +It will open an ssh connection to the VM and keep it up and running for one hour. +The connection url is showns in the log for debugAction.sh + +You can then connect to the build vm, and debug it. +You need to use a password of your choice to access it. + +You can continue the build with `touch /tmp/continue`. +You can abort the build with `touch /tmp/abort`. + +To enable this option you have to register to Ngrok, using the fee account and get the NGROK Token. + +Then set the following secrets: + +- `NGROK_TOKEN` to the ngrok token. +- `NGROK_PASSWORD` to a password of choice to access the build with the ssh command generated. + +## Log Upload + +The build uploads the logs to an s3 bucket allowing to inspect them with a browser. + +You need to create the bucket with the following commands: + +``` +AWS_BUCKET=<name-of-your-bucket> +AWS_REGION=<the-region-you-use> +aws s3 mb s3://$AWS_BUCKET --region $AWS_REGION +aws s3 website s3://$AWS_BUCKET/ --index-document index.html +aws s3api put-bucket-acl --acl public-read --bucket $AWS_BUCKET +``` + +To enable upload to the created bucket you need to set the following secrets: + +- `AWS_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. +- `AWS_ACCESS_KEY_ID`: your aws access key. +- `AWS_SECRET_ACCESS_KEY`: your aws secret key. +- `AWS_REGION`: important: the region where your bucket is. + +## Slack notification + +If you want to get notified of what happens on slack, create an [Incoming Web Hook](https://api.slack.com/messaging/webhooks) and then set the following secret: + +- `SLACK_WEBHOOK`: the incoming webhook url provided by slack. diff --git a/.gitignore b/.gitignore index 284e8d225..7a15a629f 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,7 @@ core/routemgmt/deleteApi/utils.js core/routemgmt/getApi/apigw-utils.js core/routemgmt/getApi/package-lock.json core/routemgmt/getApi/utils.js + +# vscode metals +.bloop/ +.metals/ diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile index 6db867ff2..c7ebd2b9e 100644 --- a/common/scala/Dockerfile +++ b/common/scala/Dockerfile @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 +# if you change version of openjsk, also update tools/github/setup.sh to download the corresponding jdk +FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.12_7_openj9-0.27.0 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh new file mode 100755 index 000000000..79f169720 --- /dev/null +++ b/tools/github/checkAndUploadLogs.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# check variables +for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION +do + if test -z "${!i}" + then echo "Required Environment Variable Missing: $i" ; exit 1 + fi +done + +# Disable abort script at first error as we require the logs to be uploaded +# even if check and log collection fails +# set -e + + +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR + +LOG_NAME="$1" + +# tags is db only when the test is unit +TAGS="" +[[ "$2" == "Unit" ]] && TAGS="db" + +LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" +BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com" + +echo "Logs: ${BUCKET_URL}/index.html#${LOG_DIR}/" +echo "Reports: ${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" + +echo "logs=${BUCKET_URL}/index.html#${LOG_DIR}/" >>${GITHUB_OUTPUT:-/dev/stdin} +echo "report=${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" >>${GITHUB_OUTPUT:-/dev/stdin} + +ansible-playbook -i ansible/environments/local ansible/logs.yml + +./tools/build/checkLogs.py logs "$TAGS" + +./tools/github/s3-upload.sh "$PWD/logs" "$LOG_DIR" + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/debugAction.sh b/tools/github/debugAction.sh new file mode 100755 index 000000000..6ebe6e243 --- /dev/null +++ b/tools/github/debugAction.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [[ -z "$NGROK_DEBUG" ]] || [[ "$NGROK_DEBUG" == "false" ]] +then exit 0 +fi + +if [[ -z "$NGROK_TOKEN" ]] +then echo "Please set 'NGROK_TOKEN'" + exit 1 +fi + +if [[ -z "$NGROK_PASSWORD" ]] +then echo "Please set 'NGROK_PASSWORD'" + exit 1 +fi + +echo "### Install ngrok ###" +if ! test -e ./ngrok +then + wget -q https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-386.zip + unzip ngrok-stable-linux-386.zip + chmod +x ./ngrok +fi + +echo "### Update user: $USER password ###" +echo -e "$NGROK_PASSWORD\n$NGROK_PASSWORD" | sudo passwd "$USER" + +echo "### Start ngrok proxy for 22 port ###" + +rm -f .ngrok.log +./ngrok authtoken "$NGROK_TOKEN" +./ngrok tcp 22 --log ".ngrok.log" & + +sleep 10 +HAS_ERRORS=$(grep "command failed" < .ngrok.log) + +if [[ -z "$HAS_ERRORS" ]]; then + MSG="To connect: $(grep -o -E "tcp://(.+)" < .ngrok.log | sed "s/tcp:\/\//ssh $USER@/" | sed "s/:/ -p /")" + echo "" + echo "==========================================" + echo "$MSG" + echo "==========================================" + if test -n "$SLACK_WEBHOOK" + then + echo -n '{"text":' >/tmp/msg$$ + echo -n "$MSG" | jq -Rsa . >>/tmp/msg$$ + echo -n '}' >>/tmp/msg$$ + curl -X POST -H 'Content-type: application/json' --data "@/tmp/msg$$" "$SLACK_WEBHOOK" + fi +else + echo "$HAS_ERRORS" + exit 1 +fi diff --git a/tools/github/flake8.sh b/tools/github/flake8.sh new file mode 100755 index 000000000..98b7c0d9c --- /dev/null +++ b/tools/github/flake8.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pip3 install --user --upgrade flake8 + +# These files do not have a .py extension so flake8 will not scan them +declare -a PYTHON_FILES=("." + "./tools/admin/wskadmin" + "./tools/build/citool" + "./tools/build/redo") + +echo 'Flake8: first round (fast fail) stops the build if there are any Python 3 syntax errors...' +for i in "${PYTHON_FILES[@]}" +do + flake8 "$i" --select=E999,F821 --statistics + RETURN_CODE=$? + if [ $RETURN_CODE != 0 ]; then + echo 'Flake8 found Python 3 syntax errors above. See: https://docs.python.org/3/howto/pyporting.html' + exit $RETURN_CODE + fi +done + +echo 'Flake8: second round to find any other stylistic issues...' +for i in "${PYTHON_FILES[@]}" +do + flake8 "$i" --ignore=E,W503,W504,W605 --max-line-length=127 --statistics +done diff --git a/common/scala/Dockerfile b/tools/github/runDummyTests.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/runDummyTests.sh index 6db867ff2..11564531e --- a/common/scala/Dockerfile +++ b/tools/github/runDummyTests.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +set -e -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." -RUN mkdir /logs - -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh - -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +mkdir $ROOTDIR/logs +echo "<h1>$(date)</h1>" >$ROOTDIR/logs/now.html +exit 0 diff --git a/common/scala/Dockerfile b/tools/github/runLeanSystemTests.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/runLeanSystemTests.sh index 6db867ff2..372fbc1e4 --- a/common/scala/Dockerfile +++ b/tools/github/runLeanSystemTests.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +16,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_LEAN_SYSTEM" +export GRADLE_COVERAGE=true -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json -RUN mkdir /logs +./distDocker-lean.sh -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh +./setupLeanSystem.sh /ansible/files/runtimes-nodeonly.json -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +./runTests.sh diff --git a/common/scala/Dockerfile b/tools/github/runMultiRuntimeTests.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/runMultiRuntimeTests.sh index 6db867ff2..e2de0c59b --- a/common/scala/Dockerfile +++ b/tools/github/runMultiRuntimeTests.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +16,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_MULTI_RUNTIME" +export GRADLE_COVERAGE=true -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +./setupPrereq.sh -RUN mkdir /logs +./distDocker.sh -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh +./setupSystem.sh -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +./runTests.sh diff --git a/common/scala/Dockerfile b/tools/github/runSchedulerTests.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/runSchedulerTests.sh index 6db867ff2..000de8080 --- a/common/scala/Dockerfile +++ b/tools/github/runSchedulerTests.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +16,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SCHEDULER" +export GRADLE_COVERAGE=true -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json -RUN mkdir /logs +./distDocker.sh -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh +./setupSystem.sh /ansible/files/runtimes-nodeonly.json -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +./runTests.sh diff --git a/tools/github/runStandaloneTests.sh b/tools/github/runStandaloneTests.sh new file mode 100755 index 000000000..88dac0762 --- /dev/null +++ b/tools/github/runStandaloneTests.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_STANDALONE" +export GRADLE_COVERAGE=true + +cd $ROOTDIR/ansible +$ANSIBLE_CMD setup.yml +$ANSIBLE_CMD properties.yml -e manifest_file="/ansible/files/runtimes-nodeonly.json" +$ANSIBLE_CMD downloadcli-github.yml + +# Install kubectl +curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/v1.25.3/bin/linux/amd64/kubectl +chmod +x kubectl +sudo cp kubectl /usr/local/bin/kubectl + +# Install kind +curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.17.0/kind-linux-amd64 +chmod +x kind +sudo cp kind /usr/local/bin/kind + +kind create cluster --wait 5m +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl config set-context --current --namespace=default + +# This is required because it is timed out to pull the image during the test. +docker pull openwhisk/action-nodejs-v14:nightly +docker pull openwhisk/dockerskeleton:nightly +docker pull openwhisk/example:nightly +docker pull openwhisk/apigateway:0.11.0 + +cd $ROOTDIR +TERM=dumb ./gradlew :core:standalone:build \ + :core:monitoring:user-events:distDocker + +cd $ROOTDIR +TERM=dumb ./gradlew :core:standalone:cleanTest \ + :core:standalone:test \ + :core:monitoring:user-events:reportTestScoverage + +# Run test in end as it publishes the coverage also +cd $ROOTDIR/tools/travis +./runTests.sh diff --git a/common/scala/Dockerfile b/tools/github/runSystemTests.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/runSystemTests.sh index 6db867ff2..8325caddb --- a/common/scala/Dockerfile +++ b/tools/github/runSystemTests.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +16,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +set -e + +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." + +cd $ROOTDIR/tools/travis + +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_SYSTEM" +export GRADLE_COVERAGE=true -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +./setupPrereq.sh /ansible/files/runtimes-nodeonly.json -RUN mkdir /logs +./distDocker.sh -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh +./setupSystem.sh /ansible/files/runtimes-nodeonly.json -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +./runTests.sh diff --git a/common/scala/Dockerfile b/tools/github/runUnitTests.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/runUnitTests.sh index 6db867ff2..b10418c9e --- a/common/scala/Dockerfile +++ b/tools/github/runUnitTests.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +set -e -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." -RUN mkdir /logs +cd $ROOTDIR/tools/travis +export TESTCONTAINERS_RYUK_DISABLED="true" +export ORG_GRADLE_PROJECT_testSetName="REQUIRE_ONLY_DB" -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh +./setupPrereq.sh -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +cat "$ROOTDIR/tests/src/test/resources/application.conf" + +./distDocker.sh + +./runTests.sh diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh new file mode 100755 index 000000000..38c685a04 --- /dev/null +++ b/tools/github/s3-upload.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# check variables +for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +do + if test -z "${!i}" + then echo "Please set $i" ; exit 1 + fi +done + +if [[ -z "$1" ]] || [[ -z "$2" ]] +then echo "usage: <source-dir> <target-path>" ; exit 1 +fi + +FROM="$1" +TO="$2" + +BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html" +BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com/" + +# install rclone +if ! which rclone +then curl https://rclone.org/install.sh | sudo bash +fi + +RCLONE="rclone --config /dev/null \ + --s3-provider AWS \ + --s3-region $AWS_REGION \ + --s3-acl public-read \ + --s3-access-key-id $AWS_ACCESS_KEY_ID \ + --s3-secret-access-key $AWS_SECRET_ACCESS_KEY" + +curl -s "$BROWSER" |\ + sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\ + $RCLONE rcat ":s3:$AWS_BUCKET/index.html" + +$RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/" diff --git a/tools/github/scan.sh b/tools/github/scan.sh new file mode 100755 index 000000000..42b3fd573 --- /dev/null +++ b/tools/github/scan.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +# Build script for Travis-CI. +SECONDS=0 +SCRIPTDIR=$(cd $(dirname "$0") && pwd) +ROOTDIR="$SCRIPTDIR/../.." +HOMEDIR="$SCRIPTDIR/../../../" +UTILDIR="$HOMEDIR/openwhisk-utilities/" + +cd $ROOTDIR +./tools/github/flake8.sh # Check Python files for style and stop the build on syntax errors + +# clone the openwhisk utilities repo. +cd $HOMEDIR +git clone https://github.com/apache/openwhisk-utilities.git + +# run the scancode util. against project source code starting at its root +cd $UTILDIR +scancode/scanCode.py --config scancode/ASF-Release.cfg $ROOTDIR + +# run scalafmt checks +cd $ROOTDIR +TERM=dumb ./gradlew checkScalafmtAll + +# lint tests to all be actually runnable +MISSING_TESTS=$(grep -rL "RunWith" --include="*Tests.scala" tests || true) +if [ -n "$MISSING_TESTS" ] +then + echo "The following tests are missing the 'RunWith' annotation" + echo $MISSING_TESTS + exit 1 +fi + +echo "Time taken for ${0##*/} is $SECONDS secs" diff --git a/tools/github/setup.sh b/tools/github/setup.sh new file mode 100755 index 000000000..e72b04605 --- /dev/null +++ b/tools/github/setup.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#if [[ $TEST_SUITE =~ Dummy ]] +#then echo skipping setup ; exit 0 +#fi + +# retries a command for five times and exits with the non-zero exit if even after +# the retries the command did not succeed. +function retry() { + local exitcode=0 + for i in {1..5}; + do + exitcode=0 + "$@" && break || exitcode=$? && echo "$i. attempt failed. Will retry $((5-i)) more times!" && sleep 1; + done + if [ $exitcode -ne 0 ]; then + exit $exitcode + fi +} + +# setup docker to listen in port 4243 +sudo systemctl stop docker +sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service +sudo systemctl daemon-reload +sudo systemctl start docker + +# installing right version of jdk +JDK=https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.12%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_11.0.12_7_openj9-0.27.0.tar.gz +curl -sL $JDK | sudo tar xzvf - -C /usr/local +JAVA="$(which java)" +sudo mv "$JAVA" "$JAVA"."$(date +%s)" +sudo ln -sf /usr/local/jdk*/bin/java $JAVA +java -version + +# Python +python -m pip install --user couchdb + +# Ansible (warning you need jinja < 3.1 with this version) +python -m pip install --user 'jinja2<3.1' ansible==2.8.18 + +# Azure CosmosDB +python -m pip install --user pydocumentdb + +# Support the revises log upload script +python -m pip install --user humanize requests + +# Scan code before compiling the code +tools/github/scan.sh + +# Preload alpine 3.5 to avoid issues with depending images +retry docker pull alpine:3.5 + +# exit if dummy test suite skipping the long compilation when debugging +if [[ $TEST_SUITE =~ Dummy ]] +then echo skiping setup ; exit 0 +fi + +# Basic check that all code compiles and dependencies are downloaded correctly. +# Compiling the tests will compile all components as well. +# +# Downloads the gradle wrapper, dependencies and tries to compile the code. +# Retried 5 times in case there are network hiccups. +TERM=dumb retry ./gradlew :tests:compileTestScala diff --git a/common/scala/Dockerfile b/tools/github/waitIfDebug.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/waitIfDebug.sh index 6db867ff2..30f791b29 --- a/common/scala/Dockerfile +++ b/tools/github/waitIfDebug.sh @@ -1,3 +1,4 @@ +#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +if ! test -e .ngrok.log +then exit 0 +fi +echo "You have an hour to debug this build." +echo "Do touch /tmp/continue to continue." +echo "Do touch /tmp/abort to abort." -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +EXIT=0 +for i in $(seq 1 60) +do + if test -e /tmp/continue ; then EXIT=0 ; break ; fi + if test -e /tmp/abort ; then EXIT=1 ; break ; fi + echo "$i/60 still waiting..." + sleep 60 +done -RUN mkdir /logs - -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh - -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +killall ngrok +rm -f .ngrok.log /tmp/continue /tmp/abort +exit $EXIT diff --git a/common/scala/Dockerfile b/tools/github/writeOnSlack.sh old mode 100644 new mode 100755 similarity index 57% copy from common/scala/Dockerfile copy to tools/github/writeOnSlack.sh index 6db867ff2..da8f2a8f2 --- a/common/scala/Dockerfile +++ b/tools/github/writeOnSlack.sh @@ -1,3 +1,4 @@ +#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,20 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -FROM adoptopenjdk/openjdk11-openj9:x86_64-alpine-jdk-11.0.8_10_openj9-0.21.0 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +if test -z "$SLACK_WEBHOOK" +then echo "Please create an incoming webhook for slack and set SLACK_WEBHOOK" + exit 0 +fi -# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184 -RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories -RUN apk add --update sed curl bash && apk update && apk upgrade +echo -n '{"text":' >/tmp/msg$$ +echo -n "$@" | jq -Rsa . >>/tmp/msg$$ +echo -n '}' >>/tmp/msg$$ -RUN mkdir /logs - -COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh - -COPY copyJMXFiles.sh / -RUN chmod +x copyJMXFiles.sh +curl -X POST -H 'Content-type: application/json' --data "@/tmp/msg$$" "$SLACK_WEBHOOK"