Repository: calcite-avatica Updated Branches: refs/heads/master 0c4de610d -> bf0f19ccd
[CALCITE-2682] Add ability to run tests in docker Project: http://git-wip-us.apache.org/repos/asf/calcite-avatica/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite-avatica/commit/bf0f19cc Tree: http://git-wip-us.apache.org/repos/asf/calcite-avatica/tree/bf0f19cc Diff: http://git-wip-us.apache.org/repos/asf/calcite-avatica/diff/bf0f19cc Branch: refs/heads/master Commit: bf0f19ccdcdce13fd26c09c2da89f54c1cdd8754 Parents: 0c4de61 Author: Francis Chuang <[email protected]> Authored: Sun Nov 18 21:48:22 2018 +1100 Committer: Francis Chuang <[email protected]> Committed: Sun Nov 18 21:48:22 2018 +1100 ---------------------------------------------------------------------- docker-compose.yml | 15 ++- docker.sh | 274 +++++++++++++++++++++++++++++++++++++++++++++++ release.sh | 263 --------------------------------------------- site/_docs/howto.md | 15 ++- 4 files changed, 300 insertions(+), 267 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/bf0f19cc/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index a2d9fd4..714e5ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: dry-run: image: maven:alpine working_dir: /src - command: sh -c "./release.sh dry-run" + command: sh -c "./docker.sh dry-run" volumes: - .:/src - maven-repo:/root/.m2 @@ -26,7 +26,7 @@ services: release: image: maven:alpine working_dir: /src - command: sh -c "./release.sh release" + command: sh -c "./docker.sh release" volumes: - .:/src - maven-repo:/root/.m2 @@ -34,10 +34,19 @@ services: clean: image: maven:alpine working_dir: /src - command: sh -c "./release.sh clean" + command: sh -c "./docker.sh clean" volumes: - .:/src - maven-repo:/root/.m2 + + test: + image: maven:alpine + working_dir: /src + command: sh -c "./docker.sh test" + volumes: + - .:/src + - maven-repo:/root/.m2 + volumes: maven-repo: http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/bf0f19cc/docker.sh ---------------------------------------------------------------------- diff --git a/docker.sh b/docker.sh new file mode 100755 index 0000000..56e8f2c --- /dev/null +++ b/docker.sh @@ -0,0 +1,274 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function terminate() { + printf "\n\nUser terminated build. Exiting...\n" + exit 1 +} + +trap terminate SIGINT + +KEYS=() + +init_glibc(){ + apk --no-cache add ca-certificates wget + echo "Installing glibc..." + wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + wget -q -O /tmp/glibc.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk + apk add /tmp/glibc.apk +} + +init_release(){ + apk --no-cache add git gnupg +} + +GPG_COMMAND="gpg2" + +get_gpg_keys(){ + GPG_KEYS=$($GPG_COMMAND --list-keys --with-colons --keyid-format LONG) + + KEY_NUM=1 + + KEY_DETAILS="" + + while read -r line; do + + IFS=':' read -ra PART <<< "$line" + + if [ ${PART[0]} == "pub" ]; then + + if [ -n "$KEY_DETAILS" ]; then + KEYS[$KEY_NUM]=$KEY_DETAILS + KEY_DETAILS="" + ((KEY_NUM++)) + + fi + + KEY_DETAILS=${PART[4]} + fi + + if [ ${PART[0]} == "uid" ]; then + KEY_DETAILS="$KEY_DETAILS - ${PART[9]}" + fi + + done <<< "$GPG_KEYS" + + if [ -n "$KEY_DETAILS" ]; then + KEYS[$KEY_NUM]=$KEY_DETAILS + fi +} + +mount_gpg_keys(){ + mkdir -p /.gnupg + + if [ -z "$(ls -A /.gnupg)" ]; then + echo "Please mount the contents of your .gnupg folder into /.gnupg. Exiting..." + exit 1 + fi + + mkdir -p /root/.gnupg + + cp -r /.gnupg/ /root/ + + chmod -R 700 /root/.gnupg/ + + rm -rf /root/.gnupg/*.lock +} + +SELECTED_GPG_KEY="" + +select_gpg_key(){ + + get_gpg_keys + + export GPG_TTY=/dev/console + + touch /root/.gnupg/gpg-agent.conf + echo 'default-cache-ttl 10000' >> /root/.gnupg/gpg-agent.conf + echo 'max-cache-ttl 10000' >> /root/.gnupg/gpg-agent.conf + + echo "Starting GPG agent..." + gpg-agent --daemon + + while $INVALID_KEY_SELECTED; do + + if [ "${#KEYS[@]}" -le 0 ]; then + echo "You do not have any GPG keys available. Exiting..." + exit 1 + fi + + echo "You have the following GPG keys:" + + for i in "${!KEYS[@]}"; do + echo "$i) ${KEYS[$i]}" + done + + read -p "Select your GPG key for signing: " KEY_INDEX + + SELECTED_GPG_KEY=$(sed 's/ -.*//' <<< ${KEYS[$KEY_INDEX]}) + + if [ -z $SELECTED_GPG_KEY ]; then + echo "Selected key is invalid, please try again." + continue + fi + + echo "Authenticating your GPG key..." + + echo "test" | $GPG_COMMAND --local-user $SELECTED_GPG_KEY --output /dev/null --sign - + + if [ $? != 0 ]; then + echo "Invalid GPG passphrase or GPG error. Please try again." + continue + fi + + echo "You have selected the following GPG key to sign the release:" + echo "${KEYS[$KEY_INDEX]}" + + INVALID_CONFIRMATION=true + + while $INVALID_CONFIRMATION; do + read -p "Is this correct? (y/n) " CONFIRM + + if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then + INVALID_KEY_SELECTED=false + INVALID_CONFIRMATION=false + elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then + INVALID_CONFIRMATION=false + fi + done + done +} + +RELEASE_VERSION="" +RC_NUMBER="" +DEV_VERSION="" +ASF_USERNAME="" +NAME="" + +get_build_configuration(){ + + while $NOT_CONFIRMED; do + read -p "Enter the version number to be released (example: 1.12.0): " RELEASE_VERSION + read -p "Enter the release candidate number (example: if you are releasing rc0, enter 0): " RC_NUMBER + read -p "Enter the development version number (example: if your release version is 1.12.0, enter 1.13.0): " DEV_VERSION + read -p "Enter your ASF username: " ASF_USERNAME + read -p "Enter your name (this will be used for git commits): " NAME + echo "Build configured as follows:" + echo "Release: $RELEASE_VERSION-rc$RC_NUMBER" + echo "Next development version: $DEV_VERSION-SNAPSHOT" + echo "ASF Username: $ASF_USERNAME" + echo "Name: $NAME" + + INVALID_CONFIRMATION=true + + while $INVALID_CONFIRMATION; do + read -p "Is this correct? (y/n) " CONFIRM + + if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then + NOT_CONFIRMED=false + INVALID_CONFIRMATION=false + elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then + INVALID_CONFIRMATION=false + fi + done + done +} + +ASF_PASSWORD="" + +set_git_credentials(){ + read -s -p "Enter your ASF password: " ASF_PASSWORD + + printf "\n" + + echo https://$ASF_USERNAME:[email protected] >> /root/.git-credentials + git config --global credential.helper 'store --file=/root/.git-credentials' + + git config --global user.name "$NAME" +} + +set_maven_credentials(){ + mkdir -p /root/.m2 + rm -f /root/.m2/settings.xml + rm -f /root/.m2/settings-security.xml + + ENCRYPTED_MAVEN_PASSWORD="$(mvn --encrypt-master-password $ASF_PASSWORD)" + + cat <<EOF >> /root/.m2/settings-security.xml +<settingsSecurity> + <master>$ENCRYPTED_MAVEN_PASSWORD</master> +</settingsSecurity> +EOF + + ENCRYPTED_ASF_PASSWORD="$(mvn --encrypt-password $ASF_PASSWORD)" + + cat <<EOF >> /root/.m2/settings.xml +<settings> + <servers> + <server> + <id>apache.snapshots.https</id> + <username>${ASF_USERNAME}</username> + <password>${ENCRYPTED_ASF_PASSWORD}</password> + </server> + <server> + <id>apache.releases.https</id> + <username>${ASF_USERNAME}</username> + <password>${ENCRYPTED_ASF_PASSWORD}</password> + </server> + </servers> +</settings> +EOF +} + +case $1 in + dry-run) + init_glibc + init_release + mount_gpg_keys + select_gpg_key + get_build_configuration + + mvn -Dmaven.artifact.threads=20 -DdryRun=true -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEV_VERSION-SNAPSHOT -Dtag="avatica-$RELEASE_VERSION-rc$RC_NUMBER" -Papache-release -Duser.name=$ASF_USERNAME release:prepare -Darguments="-DskipDockerCheck -Dgpg.keyname=$SELECTED_GPG_KEY" + ;; + + release) + init_glibc + init_release + mount_gpg_keys + select_gpg_key + get_build_configuration + set_git_credentials + set_maven_credentials + + mvn -Dmaven.artifact.threads=20 -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEV_VERSION-SNAPSHOT -Dtag="avatica-$RELEASE_VERSION-rc$RC_NUMBER" -Papache-release -Duser.name=$ASF_USERNAME release:prepare -Darguments=-Dgpg.keyname=$SELECTED_GPG_KEY + mvn -Dmaven.artifact.threads=20 -Papache-release -Duser.name=$ASF_USERNAME release:perform -Darguments="-DskipTests -Dgpg.keyname=$SELECTED_GPG_KEY" + ;; + + clean) + mvn release:clean + ;; + + test) + init_glibc + mvn clean verify -Dcheckstyle.skip -DskipDockerCheck + ;; + + *) + echo $"Usage: $0 {dry-run|release|clean|test}" + ;; + +esac \ No newline at end of file http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/bf0f19cc/release.sh ---------------------------------------------------------------------- diff --git a/release.sh b/release.sh deleted file mode 100755 index 119f3ad..0000000 --- a/release.sh +++ /dev/null @@ -1,263 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function terminate() { - printf "\n\nUser terminated build. Exiting...\n" - exit 1 -} - -trap terminate SIGINT - -KEYS=() - -init(){ - apk --no-cache add ca-certificates wget git gnupg - wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - wget -q -O /tmp/glibc.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add /tmp/glibc.apk -} - -GPG_COMMAND="gpg2" - -get_gpg_keys(){ - GPG_KEYS=$($GPG_COMMAND --list-keys --with-colons --keyid-format LONG) - - KEY_NUM=1 - - KEY_DETAILS="" - - while read -r line; do - - IFS=':' read -ra PART <<< "$line" - - if [ ${PART[0]} == "pub" ]; then - - if [ -n "$KEY_DETAILS" ]; then - KEYS[$KEY_NUM]=$KEY_DETAILS - KEY_DETAILS="" - ((KEY_NUM++)) - - fi - - KEY_DETAILS=${PART[4]} - fi - - if [ ${PART[0]} == "uid" ]; then - KEY_DETAILS="$KEY_DETAILS - ${PART[9]}" - fi - - done <<< "$GPG_KEYS" - - if [ -n "$KEY_DETAILS" ]; then - KEYS[$KEY_NUM]=$KEY_DETAILS - fi -} - -mount_gpg_keys(){ - mkdir -p /.gnupg - - if [ -z "$(ls -A /.gnupg)" ]; then - echo "Please mount the contents of your .gnupg folder into /.gnupg. Exiting..." - exit 1 - fi - - mkdir -p /root/.gnupg - - cp -r /.gnupg/ /root/ - - chmod -R 700 /root/.gnupg/ - - rm -rf /root/.gnupg/*.lock -} - -SELECTED_GPG_KEY="" - -select_gpg_key(){ - - get_gpg_keys - - export GPG_TTY=/dev/console - - touch /root/.gnupg/gpg-agent.conf - echo 'default-cache-ttl 10000' >> /root/.gnupg/gpg-agent.conf - echo 'max-cache-ttl 10000' >> /root/.gnupg/gpg-agent.conf - - echo "Starting GPG agent..." - gpg-agent --daemon - - while $INVALID_KEY_SELECTED; do - - if [ "${#KEYS[@]}" -le 0 ]; then - echo "You do not have any GPG keys available. Exiting..." - exit 1 - fi - - echo "You have the following GPG keys:" - - for i in "${!KEYS[@]}"; do - echo "$i) ${KEYS[$i]}" - done - - read -p "Select your GPG key for signing: " KEY_INDEX - - SELECTED_GPG_KEY=$(sed 's/ -.*//' <<< ${KEYS[$KEY_INDEX]}) - - if [ -z $SELECTED_GPG_KEY ]; then - echo "Selected key is invalid, please try again." - continue - fi - - echo "Authenticating your GPG key..." - - echo "test" | $GPG_COMMAND --local-user $SELECTED_GPG_KEY --output /dev/null --sign - - - if [ $? != 0 ]; then - echo "Invalid GPG passphrase or GPG error. Please try again." - continue - fi - - echo "You have selected the following GPG key to sign the release:" - echo "${KEYS[$KEY_INDEX]}" - - INVALID_CONFIRMATION=true - - while $INVALID_CONFIRMATION; do - read -p "Is this correct? (y/n) " CONFIRM - - if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then - INVALID_KEY_SELECTED=false - INVALID_CONFIRMATION=false - elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then - INVALID_CONFIRMATION=false - fi - done - done -} - -RELEASE_VERSION="" -RC_NUMBER="" -DEV_VERSION="" -ASF_USERNAME="" -NAME="" - -get_build_configuration(){ - - while $NOT_CONFIRMED; do - read -p "Enter the version number to be released (example: 1.12.0): " RELEASE_VERSION - read -p "Enter the release candidate number (example: if you are releasing rc0, enter 0): " RC_NUMBER - read -p "Enter the development version number (example: if your release version is 1.12.0, enter 1.13.0): " DEV_VERSION - read -p "Enter your ASF username: " ASF_USERNAME - read -p "Enter your name (this will be used for git commits): " NAME - echo "Build configured as follows:" - echo "Release: $RELEASE_VERSION-rc$RC_NUMBER" - echo "Next development version: $DEV_VERSION-SNAPSHOT" - echo "ASF Username: $ASF_USERNAME" - echo "Name: $NAME" - - INVALID_CONFIRMATION=true - - while $INVALID_CONFIRMATION; do - read -p "Is this correct? (y/n) " CONFIRM - - if [[ ($CONFIRM == "Y") || ($CONFIRM == "y") ]]; then - NOT_CONFIRMED=false - INVALID_CONFIRMATION=false - elif [[ ($CONFIRM == "N") || ($CONFIRM == "n") ]]; then - INVALID_CONFIRMATION=false - fi - done - done -} - -ASF_PASSWORD="" - -set_git_credentials(){ - read -s -p "Enter your ASF password: " ASF_PASSWORD - - printf "\n" - - echo https://$ASF_USERNAME:[email protected] >> /root/.git-credentials - git config --global credential.helper 'store --file=/root/.git-credentials' - - git config --global user.name "$NAME" -} - -set_maven_credentials(){ - mkdir -p /root/.m2 - rm -f /root/.m2/settings.xml - rm -f /root/.m2/settings-security.xml - - ENCRYPTED_MAVEN_PASSWORD="$(mvn --encrypt-master-password $ASF_PASSWORD)" - - cat <<EOF >> /root/.m2/settings-security.xml -<settingsSecurity> - <master>$ENCRYPTED_MAVEN_PASSWORD</master> -</settingsSecurity> -EOF - - ENCRYPTED_ASF_PASSWORD="$(mvn --encrypt-password $ASF_PASSWORD)" - - cat <<EOF >> /root/.m2/settings.xml -<settings> - <servers> - <server> - <id>apache.snapshots.https</id> - <username>${ASF_USERNAME}</username> - <password>${ENCRYPTED_ASF_PASSWORD}</password> - </server> - <server> - <id>apache.releases.https</id> - <username>${ASF_USERNAME}</username> - <password>${ENCRYPTED_ASF_PASSWORD}</password> - </server> - </servers> -</settings> -EOF -} - -case $1 in - dry-run) - init - mount_gpg_keys - select_gpg_key - get_build_configuration - - mvn -Dmaven.artifact.threads=20 -DdryRun=true -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEV_VERSION-SNAPSHOT -Dtag="avatica-$RELEASE_VERSION-rc$RC_NUMBER" -Papache-release -Duser.name=$ASF_USERNAME release:prepare -Darguments="-DskipDockerCheck -Dgpg.keyname=$SELECTED_GPG_KEY" - ;; - - release) - init - mount_gpg_keys - select_gpg_key - get_build_configuration - set_git_credentials - set_maven_credentials - - mvn -Dmaven.artifact.threads=20 -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEV_VERSION-SNAPSHOT -Dtag="avatica-$RELEASE_VERSION-rc$RC_NUMBER" -Papache-release -Duser.name=$ASF_USERNAME release:prepare -Darguments=-Dgpg.keyname=$SELECTED_GPG_KEY - mvn -Dmaven.artifact.threads=20 -Papache-release -Duser.name=$ASF_USERNAME release:perform -Darguments="-DskipTests -Dgpg.keyname=$SELECTED_GPG_KEY" - ;; - - clean) - init - mvn release:clean - ;; - - *) - echo $"Usage: $0 {dry-run|release|clean}" - ;; - -esac \ No newline at end of file http://git-wip-us.apache.org/repos/asf/calcite-avatica/blob/bf0f19cc/site/_docs/howto.md ---------------------------------------------------------------------- diff --git a/site/_docs/howto.md b/site/_docs/howto.md index 5733018..c832731 100644 --- a/site/_docs/howto.md +++ b/site/_docs/howto.md @@ -69,14 +69,27 @@ tests. The test suite will run by default when you build, unless you specify `-DskipTests`: +Note: During development, you should include `-DskipDockerCheck`, because the docker check checks to see if the dockerfile +includes the correct version, which will fail unless you are making a release of Avatica. + +### Running tests in your environment + {% highlight bash %} -$ ./mvnw clean verify -Dcheckstyle.skip +$ ./mvnw clean verify -Dcheckstyle.skip -DskipDockerCheck {% endhighlight %} By default, invoking the `verify` Maven lifecycle phase will also cause checkstyle rules to be run. It is expected that contributions pass the checkstyle rules; however, it is common to ignore these while working on a feature/bug and fix them at the end. +### To run tests in docker: + +* You will need to have [docker](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose/install/) installed. + +{% highlight bash %} +docker-compose run test +{% endhighlight %} + ## Contributing See the [developers guide]({{ site.baseurl }}/develop/#contributing).
