METRON-1847 Create reusable script with functions from prepare-commit (ottobackwards) closes apache/metron#1248
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/42068d95 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/42068d95 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/42068d95 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 42068d95192e7346bf0179399193a6afe01d9925 Parents: 3e73391 Author: ottobackwards <ottobackwa...@gmail.com> Authored: Thu Nov 8 22:14:38 2018 -0500 Committer: otto <o...@apache.org> Committed: Thu Nov 8 22:14:38 2018 -0500 ---------------------------------------------------------------------- .../committer-utils/metron-committer-common | 357 +++++++++++++++++++ dev-utilities/committer-utils/prepare-commit | 241 +------------ 2 files changed, 372 insertions(+), 226 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/42068d95/dev-utilities/committer-utils/metron-committer-common ---------------------------------------------------------------------- diff --git a/dev-utilities/committer-utils/metron-committer-common b/dev-utilities/committer-utils/metron-committer-common new file mode 100644 index 0000000..22e19a2 --- /dev/null +++ b/dev-utilities/committer-utils/metron-committer-common @@ -0,0 +1,357 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# common and unlikely to change vars +# + +# the upstream apache git repo for apache metron +METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git"; +# the upstream apache git repo for apache metron bro plugin kafka +BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git"; +# the common configuration file with the committer info +CONFIG_FILE=~/.metron-prepare-commit + +GITHUB_REMOTE="origin" +BASE_BRANCH=master + +# other var setup by these functions +PR= +WORK= +ORIGIN= +UPSTREAM= +PR_BRANCH= +USER= +EMAIL= +JIRA= +DESC= +APACHE_NAME= +APACHE_EMAIL= +GITHUB_NAME= + +# +# Initialize the variables from the default configuration file, if it exists +# +function init_configuration { + # does a config file already exist? + echo "$CONFIG_FILE" + if [ -f ${CONFIG_FILE} ]; then + #shellcheck source=/dev/null + source ${CONFIG_FILE} + echo " ...using settings from $CONFIG_FILE" + fi +} + + +# +# Initialize the committer variables if they are not provided through the configuration file. +# If it is not present, it will be written out for the next time +# +function init_committer_info { + # github account of committer (you) + if [ -z "$GITHUB_NAME" ]; then + read -p " your github username [$GITHUB_NAME]: " INPUT + [ -n "$INPUT" ] && GITHUB_NAME=${INPUT} + + # write setting to config file + echo "GITHUB_NAME=$GITHUB_NAME" >> ${CONFIG_FILE} + fi + + # apache id of committer (you) + if [ -z "$APACHE_NAME" ]; then + read -p " your apache userid [$APACHE_NAME]: " INPUT + [ -n "$INPUT" ] && APACHE_NAME=${INPUT} + + # write setting to config file + echo "APACHE_NAME=$APACHE_NAME" >> ${CONFIG_FILE} + fi + + # apache email addr of committer (you) + if [ -z "$APACHE_EMAIL" ]; then + APACHE_EMAIL=${APACHE_NAME}@apache.org + read -p " your apache email [$APACHE_EMAIL]: " INPUT + [ -n "$INPUT" ] && APACHE_EMAIL=${INPUT} + + # write setting to config file, so it is not needed next time + echo "APACHE_EMAIL=$APACHE_EMAIL" >> ${CONFIG_FILE} + fi +} + +# +# Provide the user with a choice of the metron or bro repositories +# +function choose_metron_or_bro_repo { + # which repo? metron or metron-bro-plugin-kafka + echo " [1] metron" + echo " [2] metron-bro-plugin-kafka" + read -p " which repo? [1]: " INPUT + case "${INPUT}" in + [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2) + INPUT="${BRO_PLUGIN_UPSTREAM}" ;; + [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'') + INPUT="${METRON_UPSTREAM}" ;; + *) + echo "Invalid repo, provided \"${INPUT}\". Please choose between metron or metron-bro-plugin-kafka" + exit 1 + ;; + esac + [ -n "$INPUT" ] && UPSTREAM=${INPUT} + + CHOSEN_REPO=$(basename ${UPSTREAM%%.git}) +} + +# +# Ask the user for the PR number +# +function read_pull_request { + # retrieve the pull request identifier + read -p " pull request: " PR + if [ -z "$PR" ]; then + echo "Error: missing pr" + exit 1 + fi + + # ensure that the pull request exists + PR_EXISTS=`curl -sI https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep Status: | sed 's/[^0-9]//g'` + if [ "$PR_EXISTS" != "200" ]; then + echo "Error: pull request #$PR does not exist" + exit 1 + fi +} + +# +# Setup the working directory. +# It is possible to override the default directory name by passing +# the desired directory name, using the available global variables +# +function setup_working_directory { + # working directory + if [ -z $1 ]; then + WORK=~/tmp/${CHOSEN_REPO}-pr${PR} + else + WORK=$1 + fi + + read -p " local working directory [$WORK]: " INPUT + [ -n "$INPUT" ] && WORK=${INPUT} + + # handle tilde expansion + WORK="${WORK/#\~/$HOME}" + + # warn the user if the working directory exists + if [ -d "$WORK" ]; then + read -p " directory exists [$WORK]. continue merge on existing repo? [yN] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + fi +} + +# +# If required will clone the desired repo. The default repo is metron, but an override +# may be passed as a parameter. +# If the directory already exists, it will ask the user for the name of the remote to use, with the +# default being "origin" +# +function setup_code { + # if working directory does not exist, checkout the base branch + if [ ! -d "$WORK" ]; then + + REPO_NAME="metron" + if [ -z $1 ]; then + REPO_NAME=$1 + fi + + # origin repository + ORIGIN="https://github.com/apache/${REPO_NAME}"; + read -p " origin repo [$ORIGIN]: " INPUT + [ -n "$INPUT" ] && ORIGIN=${INPUT} + + # what branch did the PR get submitted against? could be a feature branch + BASE_BRANCH=`curl -s https://api.github.com/repos/apache/${REPO_NAME}/pulls/${PR} | python -c 'import sys, json; print json.load(sys.stdin)["base"]["ref"]'` + read -p " base branch to merge into [$BASE_BRANCH]: " INPUT + [ -n "$INPUT" ] && BASE_BRANCH=${INPUT} + + # clone the repository and fetch updates + mkdir -p ${WORK} + git clone ${ORIGIN} ${WORK} + cd ${WORK} || exit "Failed to cd to ${WORK}" + + # setup the git user and email for your apache account + git config user.name "$APACHE_NAME" + git config user.email "$APACHE_EMAIL" + + # fetch any changes from upstream + git remote add upstream ${UPSTREAM} + if git fetch upstream "$BASE_BRANCH"; then + + if [ ${BASE_BRANCH} = "master" ]; then + # merge any changes from upstream + git checkout ${BASE_BRANCH} + git merge upstream/${BASE_BRANCH} + + else + # create a local branch from the remote feature branch + git checkout -B ${BASE_BRANCH} upstream/${BASE_BRANCH} + + fi + + else + # unable to fetch the base branch + exit $? + fi + + else + + # if the repo already exists, allow the user to provide the name of the Github remote + # this is needed to checkout the code for the PR + read -p " name of github remote [$GITHUB_REMOTE]: " INPUT + [ -n "$INPUT" ] && GITHUB_REMOTE=${INPUT} + + fi + + PR_BRANCH_REF="pull/$PR/head:pr-$PR" + PR_BRANCH="pr-$PR" + cd ${WORK} || exit "failed to move to ${WORK}" + git fetch ${GITHUB_REMOTE} ${PR_BRANCH_REF} + echo "" +} + +# +# Populates the contributor information from the PR information +# +function get_contributor_info { + # use github api to retrieve the contributor's login + USER=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep login | head -1 | awk -F":" '{print $2}' | sed 's/[^a-zA-Z.@_-]//g'` + read -p " github contributor's username [$USER]: " INPUT + [ -n "$INPUT" ] && USER=${INPUT} + + # validate the github contributor + if [ -z "$USER" ]; then + echo "Error: missing username" + exit 1 + fi + + # retrieve the contributor's email from the git commit history + EMAIL=`git log ${PR_BRANCH} | grep Author | head -1 | awk -F"<" '{print $2}' | sed 's/[<>]//g'` + read -p " github contributor's email [$EMAIL]: " INPUT + [ -n "$INPUT" ] && EMAIL=${INPUT} + + # validate email + if [ -z "$EMAIL" ] || [ "$EMAIL" = "null" ]; then + echo "Error: missing email" + exit 1 + fi +} + +# +# Populate the information for the JIRA associated with the PR +# +function get_jira_info { + # can we extract the JIRA from the PR title? + JIRA=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep title | head -1 | egrep -o -i 'METRON-[0-9]+' | awk '{print toupper($0)}'` + read -p " issue identifier in jira [$JIRA]: " INPUT + [ -n "$INPUT" ] && JIRA=${INPUT} + + # validate the JIRA issue + if [ -z "$JIRA" ]; then + echo "Error: missing jira" + exit 1 + fi + + # attempt to use the jira api to get a description of the jira + DESC=`curl -s https://issues.apache.org/jira/si/jira.issueviews:issue-xml/${JIRA}/${JIRA}.xml | grep "<summary>" | sed 's/^.*<summary>//' | sed 's/<.summary>.*$//'` + read -p " issue description [$DESC]: " INPUT + [ -n "$INPUT" ] && DESC=${INPUT} + + # validate description + if [ -z "$DESC" ]; then + echo "Error: missing description" + exit 1 + fi +} + +# +# Merge and Commit +# +function commit { + # commit message + AUTHOR="$USER <$EMAIL>" + if [ "$USER" == "$GITHUB_NAME" ]; then + MSG="$JIRA $DESC ($USER) closes apache/${CHOSEN_REPO}#$PR" + else + MSG="$JIRA $DESC ($USER via $GITHUB_NAME) closes apache/${CHOSEN_REPO}#$PR" + fi + read -p " commit message [$MSG]: " INPUT + [ -n "$INPUT" ] && MSG=${INPUT} + + # merge the contributor's branch and commit + echo "" + if git merge --squash "$PR_BRANCH"; then + git commit --author="$AUTHOR" -a -m "$MSG" + else + exit $? + fi +} + +# +# Displays the commit info ( the diff and log ) +# +function review_commit_info { + # review the commit + echo "" + echo "" + git diff --stat --color "upstream/$BASE_BRANCH..$BASE_BRANCH" + echo "" + echo "" + git log --oneline "$BASE_BRANCH" "^upstream/$BASE_BRANCH" +} + +# +# Runs the metron unit, integration and metron-interface tests +# +function run_tests { + echo "" + echo "" + read -p " run test suite? [yN] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + if [ "${UPSTREAM}" == "${METRON_UPSTREAM}" ]; then + mvn -q -T 2C -DskipTests clean install && + mvn -q -T 2C surefire:test@unit-tests && + mvn -q surefire:test@integration-tests && + mvn -q test --projects metron-interface/metron-config && + dev-utilities/build-utils/verify_licenses.sh + elif [ "${UPSTREAM}" == "${BRO_PLUGIN_UPSTREAM}" ]; then + echo "We don't currently support running metron-bro-plugin-kafka tests in this script" + fi + fi + +} + +# +# Gives the user instruction on next steps +# +function please_review_then { + echo "" + echo "Review commit carefully then run..." + echo " cd $WORK" + echo " git push upstream $BASE_BRANCH" + echo "" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/42068d95/dev-utilities/committer-utils/prepare-commit ---------------------------------------------------------------------- diff --git a/dev-utilities/committer-utils/prepare-commit b/dev-utilities/committer-utils/prepare-commit index db15aaa..c5dd884 100755 --- a/dev-utilities/committer-utils/prepare-commit +++ b/dev-utilities/committer-utils/prepare-commit @@ -16,242 +16,31 @@ # limitations under the License. # -# not likely to change -METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git"; -BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git"; -CONFIG_FILE=~/.metron-prepare-commit -GITHUB_REMOTE="origin" -BASE_BRANCH=master +LOCATION_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +#shellcheck source=./metron-committer-common +source ${LOCATION_DIR}/metron-committer-common -# does a config file already exist? -if [ -f $CONFIG_FILE ]; then - . $CONFIG_FILE - echo " ...using settings from $CONFIG_FILE" -fi +init_configuration -# which repo? metron or metron-bro-plugin-kafka -echo " [1] metron" -echo " [2] metron-bro-plugin-kafka" -read -p " which repo? [1]: " INPUT -case "${INPUT}" in - [Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo][Nn]-[Bb][Rr][Oo]-[Pp][Ll][Uu][Gg][Ii][Nn]-[Kk][Aa][Ff][Kk][Aa]|*metron-bro-plugin-kafka\.git|2) - INPUT="${BRO_PLUGIN_UPSTREAM}" ;; - [Mm][Ee][Tt][Rr][Oo][Nn]|*metron\.git|1|'') - INPUT="${METRON_UPSTREAM}" ;; - *) - echo "Invalid repo, provided \"${INPUT}\". Please choose between metron or metron-bro-plugin-kafka" - exit 1 - ;; -esac -[ -n "$INPUT" ] && UPSTREAM=$INPUT +choose_metron_or_bro_repo -CHOSEN_REPO=$(basename ${UPSTREAM%%.git}) +init_committer_info -# github account of committer (you) -if [ -z "$GITHUB_NAME" ]; then - read -p " your github username [$GITHUB_NAME]: " INPUT - [ -n "$INPUT" ] && GITHUB_NAME=$INPUT +read_pull_request - # write setting to config file - echo "GITHUB_NAME=$GITHUB_NAME" >> $CONFIG_FILE -fi +setup_working_directory ~/tmp/${CHOSEN_REPO}-pr${PR} -# apache id of committer (you) -if [ -z "$APACHE_NAME" ]; then - read -p " your apache userid [$APACHE_NAME]: " INPUT - [ -n "$INPUT" ] && APACHE_NAME=$INPUT +setup_code ${CHOSEN_REPO} - # write setting to config file - echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE -fi +get_contributor_info -# apache email addr of committer (you) -if [ -z "$APACHE_EMAIL" ]; then - APACHE_EMAIL=${APACHE_NAME}@apache.org - read -p " your apache email [$APACHE_EMAIL]: " INPUT - [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT +get_jira_info - # write setting to config file, so it is not needed next time - echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE -fi +commit -# retrieve the pull request identifier -read -p " pull request: " PR -if [ -z "$PR" ]; then - echo "Error: missing pr" - exit 1 -fi +review_commit_info -# ensure that the pull request exists -PR_EXISTS=`curl -sI https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | grep Status: | sed 's/[^0-9]//g'` -if [ "$PR_EXISTS" != "200" ]; then - echo "Error: pull request #$PR does not exist" - exit 1 -fi +run_tests -# working directory -WORK=~/tmp/${CHOSEN_REPO}-pr$PR -read -p " local working directory [$WORK]: " INPUT -[ -n "$INPUT" ] && WORK=$INPUT +please_review_then -# handle tilde expansion -WORK="${WORK/#\~/$HOME}" - -# warn the user if the working directory exists -if [ -d "$WORK" ]; then - read -p " directory exists [$WORK]. continue merge on existing repo? [yN] " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi -fi - -# if working directory does not exist, checkout the base branch -if [ ! -d "$WORK" ]; then - - # origin repository - ORIGIN="https://github.com/apache/${CHOSEN_REPO}"; - read -p " origin repo [$ORIGIN]: " INPUT - [ -n "$INPUT" ] && ORIGIN=$INPUT - - # what branch did the PR get submitted against? could be a feature branch - BASE_BRANCH=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | python -c 'import sys, json; print json.load(sys.stdin)["base"]["ref"]'` - read -p " base branch to merge into [$BASE_BRANCH]: " INPUT - [ -n "$INPUT" ] && BASE_BRANCH=$INPUT - - # clone the repository and fetch updates - mkdir -p $WORK - git clone $ORIGIN $WORK - cd $WORK - - # setup the git user and email for your apache account - git config user.name "$APACHE_NAME" - git config user.email $APACHE_EMAIL - - # fetch any changes from upstream - git remote add upstream $UPSTREAM - if git fetch upstream "$BASE_BRANCH"; then - - if [ $BASE_BRANCH = "master" ]; then - # merge any changes from upstream - git checkout $BASE_BRANCH - git merge upstream/$BASE_BRANCH - - else - # create a local branch from the remote feature branch - git checkout -B $BASE_BRANCH upstream/$BASE_BRANCH - - fi - - else - # unable to fetch the base branch - exit $? - fi - -else - - # if the repo already exists, allow the user to provide the name of the Github remote - # this is needed to checkout the code for the PR - read -p " name of github remote [$GITHUB_REMOTE]: " INPUT - [ -n "$INPUT" ] && GITHUB_REMOTE=$INPUT - -fi - -PR_BRANCH_REF="pull/$PR/head:pr-$PR" -PR_BRANCH="pr-$PR" -cd $WORK -git fetch $GITHUB_REMOTE $PR_BRANCH_REF -echo "" - -# use github api to retrieve the contributor's login -USER=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | grep login | head -1 | awk -F":" '{print $2}' | sed 's/[^a-zA-Z.@_-]//g'` -read -p " github contributor's username [$USER]: " INPUT -[ -n "$INPUT" ] && USER=$INPUT - -# validate the github contributor -if [ -z "$USER" ]; then - echo "Error: missing username" - exit 1 -fi - -# retrieve the contributor's email from the git commit history -EMAIL=`git log $PR_BRANCH | grep Author | head -1 | awk -F"<" '{print $2}' | sed 's/[<>]//g'` -read -p " github contributor's email [$EMAIL]: " INPUT -[ -n "$INPUT" ] && EMAIL=$INPUT - -# validate email -if [ -z "$EMAIL" ] || [ "$EMAIL" = "null" ]; then - echo "Error: missing email" - exit 1 -fi - -# can we extract the JIRA from the PR title? -JIRA=`curl -s https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/$PR | grep title | head -1 | egrep -o -i 'METRON-[0-9]+' | awk '{print toupper($0)}'` -read -p " issue identifier in jira [$JIRA]: " INPUT -[ -n "$INPUT" ] && JIRA=$INPUT - -# validate the JIRA issue -if [ -z "$JIRA" ]; then - echo "Error: missing jira" - exit 1 -fi - -# attempt to use the jira api to get a description of the jira -DESC=`curl -s https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml | grep "<summary>" | sed 's/^.*<summary>//' | sed 's/<.summary>.*$//'` -read -p " issue description [$DESC]: " INPUT -[ -n "$INPUT" ] && DESC=$INPUT - -# validate description -if [ -z "$DESC" ]; then - echo "Error: missing description" - exit 1 -fi - -# commit message -AUTHOR="$USER <$EMAIL>" -if [ "$USER" == "$GITHUB_NAME" ]; then - MSG="$JIRA $DESC ($USER) closes apache/${CHOSEN_REPO}#$PR" -else - MSG="$JIRA $DESC ($USER via $GITHUB_NAME) closes apache/${CHOSEN_REPO}#$PR" -fi -read -p " commit message [$MSG]: " INPUT -[ -n "$INPUT" ] && MSG=$INPUT - -# merge the contributor's branch and commit -echo "" -if git merge --squash "$PR_BRANCH"; then - git commit --author="$AUTHOR" -a -m "$MSG" -else - exit $? -fi - -# review the commit -echo "" -echo "" -git diff --stat --color "upstream/$BASE_BRANCH..$BASE_BRANCH" -echo "" -echo "" -git log --oneline "$BASE_BRANCH" "^upstream/$BASE_BRANCH" - -# run tests? -echo "" -echo "" -read -p " run test suite? [yN] " -n 1 -r -echo -if [[ $REPLY =~ ^[Yy]$ ]]; then - if [ "${UPSTREAM}" == "${METRON_UPSTREAM}" ]; then - mvn -q -T 2C -DskipTests clean install && - mvn -q -T 2C surefire:test@unit-tests && - mvn -q surefire:test@integration-tests && - mvn -q test --projects metron-interface/metron-config && - dev-utilities/build-utils/verify_licenses.sh - elif [ "${UPSTREAM}" == "${BRO_PLUGIN_UPSTREAM}" ]; then - echo "We don't currently support running metron-bro-plugin-kafka tests in this script" - fi -fi - -echo "" -echo "Review commit carefully then run..." -echo " cd $WORK" -echo " git push upstream $BASE_BRANCH" -echo ""
