This is an automated email from the ASF dual-hosted git repository. msciabarra pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new 21c9a6363 fixes to use different secrets, show the results in the log (#5367) 21c9a6363 is described below commit 21c9a6363fe14e581ec67da281e1762db02ae557 Author: Michele Sciabarra <mich...@nuvolaris.io> AuthorDate: Thu Dec 22 14:11:26 2022 +0100 fixes to use different secrets, show the results in the log (#5367) * fixes to use different secrets, show the results in the log * propagating aws variabales and disabiling a test if the key is there ut empty Co-authored-by: Michele Sciabarra <mich...@sciabarra.com> --- .github/workflows/0-on-demand.yaml | 10 +++++----- .github/workflows/1-unit.yaml | 9 +++++++-- .github/workflows/2-system.yaml | 10 +++++----- .github/workflows/3-multi-runtime.yaml | 10 +++++----- .github/workflows/4-standalone.yaml | 10 +++++----- .github/workflows/5-scheduler.yaml | 10 +++++----- .github/workflows/6-performance.yaml | 8 ++++---- .github/workflows/README.md | 18 +++++++++--------- .../org/apache/openwhisk/core/database/s3/S3Aws.scala | 2 +- tools/github/checkAndUploadLogs.sh | 10 +++++++--- tools/github/s3-upload.sh | 14 +++++++------- tools/github/setup.sh | 3 +++ 12 files changed, 63 insertions(+), 51 deletions(-) diff --git a/.github/workflows/0-on-demand.yaml b/.github/workflows/0-on-demand.yaml index 5d8d66ea3..a39a379d9 100644 --- a/.github/workflows/0-on-demand.yaml +++ b/.github/workflows/0-on-demand.yaml @@ -47,10 +47,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} @@ -74,7 +74,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/1-unit.yaml b/.github/workflows/1-unit.yaml index 144720625..c94061250 100644 --- a/.github/workflows/1-unit.yaml +++ b/.github/workflows/1-unit.yaml @@ -34,7 +34,12 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} + + # some tests need also this even if they are empty on pull_requests... AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -56,7 +61,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/2-system.yaml b/.github/workflows/2-system.yaml index 9295f921f..79c5ea6eb 100644 --- a/.github/workflows/2-system.yaml +++ b/.github/workflows/2-system.yaml @@ -34,10 +34,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} @@ -56,7 +56,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/3-multi-runtime.yaml b/.github/workflows/3-multi-runtime.yaml index 6bdcf7703..e74bba246 100644 --- a/.github/workflows/3-multi-runtime.yaml +++ b/.github/workflows/3-multi-runtime.yaml @@ -34,10 +34,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} @@ -56,7 +56,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/4-standalone.yaml b/.github/workflows/4-standalone.yaml index 92a9a7d82..15a1c8ff3 100644 --- a/.github/workflows/4-standalone.yaml +++ b/.github/workflows/4-standalone.yaml @@ -34,10 +34,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} @@ -56,7 +56,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/5-scheduler.yaml b/.github/workflows/5-scheduler.yaml index cb9234f20..02b526d55 100644 --- a/.github/workflows/5-scheduler.yaml +++ b/.github/workflows/5-scheduler.yaml @@ -34,10 +34,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} @@ -56,7 +56,7 @@ jobs: run: "./tools/github/run${{ env.TEST_SUITE }}Tests.sh" continue-on-error: true - id: logs - name: Upload Logs + name: Show results and Upload logs run: ./tools/github/checkAndUploadLogs.sh ${{ env.TEST_SUITE }} - name: Slack Notification run: > diff --git a/.github/workflows/6-performance.yaml b/.github/workflows/6-performance.yaml index cde8c5fbf..55e811b1d 100644 --- a/.github/workflows/6-performance.yaml +++ b/.github/workflows/6-performance.yaml @@ -34,10 +34,10 @@ env: SLACK_WEBHOOK: ${{secrets.SLACK_WEBHOOK}} # (optional) s3 log upload - AWS_BUCKET: ${{ secrets.AWS_BUCKET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} + LOG_BUCKET: ${{ secrets.LOG_BUCKET }} + LOG_ACCESS_KEY_ID: ${{ secrets.LOG_ACCESS_KEY_ID }} + LOG_SECRET_ACCESS_KEY: ${{ secrets.LOG_SECRET_ACCESS_KEY }} + LOG_REGION: ${{ secrets.LOG_REGION }} # github GH_BUILD: ${{ github.event_name }}-${{ github.sha }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 55473d04e..dbe6c7089 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -35,19 +35,19 @@ The build uploads the logs to an s3 bucket allowing to inspect them with a brows You need to create the bucket with the following commands: ``` -AWS_BUCKET=<name-of-your-bucket> -AWS_REGION=<the-region-you-use> -aws s3 mb s3://$AWS_BUCKET --region $AWS_REGION -aws s3 website s3://$AWS_BUCKET/ --index-document index.html -aws s3api put-bucket-acl --acl public-read --bucket $AWS_BUCKET +LOG_BUCKET=<name-of-your-bucket> +LOG_REGION=<the-region-you-use> +aws s3 mb s3://$LOG_BUCKET --region $LOG_REGION +aws s3 website s3://$LOG_BUCKET/ --index-document index.html +aws s3api put-bucket-acl --acl public-read --bucket $LOG_BUCKET ``` To enable upload to the created bucket you need to set the following secrets: -- `AWS_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. -- `AWS_ACCESS_KEY_ID`: your aws access key. -- `AWS_SECRET_ACCESS_KEY`: your aws secret key. -- `AWS_REGION`: important: the region where your bucket is. +- `LOG_BUCKET`: name of your bucket in s3 (just the name, without `s3://`); create it before. +- `LOG_ACCESS_KEY_ID`: your aws access key. +- `LOG_SECRET_ACCESS_KEY`: your aws secret key. +- `LOG_REGION`: important: the region where your bucket is. ## Slack notification diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala index 722997ac9..952598d19 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/s3/S3Aws.scala @@ -57,7 +57,7 @@ trait S3Aws extends FlatSpec { override protected def withFixture(test: NoArgTest) = { assume( - secretAccessKey != null, + secretAccessKey != null && secretAccessKey != "", "'AWS_SECRET_ACCESS_KEY' env not configured. Configure following " + "env variables for test to run. 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_REGION'") diff --git a/tools/github/checkAndUploadLogs.sh b/tools/github/checkAndUploadLogs.sh index 79f169720..da96610a7 100755 --- a/tools/github/checkAndUploadLogs.sh +++ b/tools/github/checkAndUploadLogs.sh @@ -16,11 +16,15 @@ # limitations under the License. # +# showing test results on the CI log +INDEX="tests/build/reports/tests/testCoverageLean/index.html" +test -f "$INDEX" && lynx -dump file://$PWD/$INDEX | grep . + # check variables -for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION +for i in LOG_BUCKET LOG_ACCESS_KEY_ID LOG_SECRET_ACCESS_KEY LOG_REGION do if test -z "${!i}" - then echo "Required Environment Variable Missing: $i" ; exit 1 + then echo "Required Environment Variable Missing: $i" ; exit 0 fi done @@ -42,7 +46,7 @@ TAGS="" [[ "$2" == "Unit" ]] && TAGS="db" LOG_DIR="$(date +%Y-%m-%d)/${LOG_NAME}-${GH_BUILD}-${GH_BRANCH}" -BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com" +BUCKET_URL="https://$LOG_BUCKET.s3.$LOG_REGION.amazonaws.com" echo "Logs: ${BUCKET_URL}/index.html#${LOG_DIR}/" echo "Reports: ${BUCKET_URL}/${LOG_DIR}/test-reports/reports/tests/testCoverageLean/index.html" diff --git a/tools/github/s3-upload.sh b/tools/github/s3-upload.sh index 38c685a04..5d2d0141b 100755 --- a/tools/github/s3-upload.sh +++ b/tools/github/s3-upload.sh @@ -17,7 +17,7 @@ # # check variables -for i in AWS_BUCKET AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +for i in LOG_BUCKET LOG_ACCESS_KEY_ID LOG_SECRET_ACCESS_KEY do if test -z "${!i}" then echo "Please set $i" ; exit 1 @@ -32,7 +32,7 @@ FROM="$1" TO="$2" BROWSER="https://raw.githubusercontent.com/qoomon/aws-s3-bucket-browser/master/index.html" -BUCKET_URL="https://$AWS_BUCKET.s3.$AWS_REGION.amazonaws.com/" +BUCKET_URL="https://$LOG_BUCKET.s3.$LOG_REGION.amazonaws.com/" # install rclone if ! which rclone @@ -41,13 +41,13 @@ fi RCLONE="rclone --config /dev/null \ --s3-provider AWS \ - --s3-region $AWS_REGION \ + --s3-region $LOG_REGION \ --s3-acl public-read \ - --s3-access-key-id $AWS_ACCESS_KEY_ID \ - --s3-secret-access-key $AWS_SECRET_ACCESS_KEY" + --s3-access-key-id $LOG_ACCESS_KEY_ID \ + --s3-secret-access-key $LOG_SECRET_ACCESS_KEY" curl -s "$BROWSER" |\ sed -e 's!bucketUrl: undefined!bucketUrl: "'$BUCKET_URL'"!' |\ - $RCLONE rcat ":s3:$AWS_BUCKET/index.html" + $RCLONE rcat ":s3:$LOG_BUCKET/index.html" -$RCLONE copyto "$FROM" ":s3:$AWS_BUCKET/$TO/" +$RCLONE copyto "$FROM" ":s3:$LOG_BUCKET/$TO/" diff --git a/tools/github/setup.sh b/tools/github/setup.sh index e72b04605..aefb21047 100755 --- a/tools/github/setup.sh +++ b/tools/github/setup.sh @@ -34,6 +34,9 @@ function retry() { fi } +# lynx utility to show test results on the job run +sudo apt-get -y install lynx + # setup docker to listen in port 4243 sudo systemctl stop docker sudo sed -i -e 's!/usr/bin/dockerd -H fd://!/usr/bin/dockerd -H tcp://0.0.0.0:4243 -H fd://!' /lib/systemd/system/docker.service