http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/push-commits.py ---------------------------------------------------------------------- diff --git a/support/python3/push-commits.py b/support/python3/push-commits.py new file mode 100755 index 0000000..82a7004 --- /dev/null +++ b/support/python3/push-commits.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script is typically used by Mesos committers to push a locally applied +review chain to ASF git repo and mark the reviews as submitted on ASF +ReviewBoard. + +Example Usage: + +> git checkout master +> git pull origin +> ./support/python3/apply-reviews.py -c -r 1234 +> ./support/python3/push-commits.py +""" + +# TODO(vinod): Also post the commit message to the corresponding ASF JIRA +# tickets and resolve them if necessary. + +import argparse +import os +import re +import sys + +from subprocess import check_output + +REVIEWBOARD_URL = 'https://reviews.apache.org' + + +def get_reviews(revision_range): + """Return the list of reviews found in the commits in the revision range.""" + reviews = [] # List of (review id, commit log) tuples + + rev_list = check_output(['git', + 'rev-list', + '--reverse', + revision_range]).strip().split('\n') + for rev in rev_list: + commit_log = check_output(['git', + '--no-pager', + 'show', + '--no-color', + '--no-patch', + rev]).strip() + + pos = commit_log.find('Review: ') + if pos != -1: + pattern = re.compile('Review: ({url})$'.format( + url=os.path.join(REVIEWBOARD_URL, 'r', '[0-9]+'))) + match = pattern.search(commit_log.strip().strip('/')) + if match is None: + print("\nInvalid ReviewBoard URL: '{}'".format( + commit_log[pos:])) + sys.exit(1) + + url = match.group(1) + reviews.append((os.path.basename(url), commit_log)) + + return reviews + + +def close_reviews(reviews, options): + """Mark the given reviews as submitted on ReviewBoard.""" + for review_id, commit_log in reviews: + print('Closing review', review_id) + if not options['dry_run']: + check_output(['rbt', + 'close', + '--description', + commit_log, + review_id]) + + +def parse_options(): + """Return a dictionary of options parsed from command line arguments.""" + parser = argparse.ArgumentParser() + + parser.add_argument('-n', + '--dry-run', + action='store_true', + help='Perform a dry run.') + + args = parser.parse_args() + + options = {} + options['dry_run'] = args.dry_run + + return options + + +def main(): + """Main function to push the commits in this branch as review requests.""" + options = parse_options() + + current_branch_ref = check_output(['git', 'symbolic-ref', 'HEAD']).strip() + current_branch = current_branch_ref.replace('refs/heads/', '', 1) + + if current_branch != 'master': + print('Please run this script from master branch') + sys.exit(1) + + remote_tracking_branch = check_output(['git', + 'rev-parse', + '--abbrev-ref', + 'master@{upstream}']).strip() + + merge_base = check_output([ + 'git', + 'merge-base', + remote_tracking_branch, + 'master']).strip() + + if merge_base == current_branch_ref: + print('No new commits found to push') + sys.exit(1) + + reviews = get_reviews(merge_base + ".." + current_branch_ref) + + # Push the current branch to remote master. + remote = check_output(['git', + 'config', + '--get', + 'branch.master.remote']).strip() + + print('Pushing commits to', remote) + + if options['dry_run']: + check_output(['git', + 'push', + '--dry-run', + remote, + 'master:master']) + else: + check_output(['git', + 'push', + remote, + 'master:master']) + + # Now mark the reviews as submitted. + close_reviews(reviews, options) + +if __name__ == '__main__': + main()
http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/test-upgrade.py ---------------------------------------------------------------------- diff --git a/support/python3/test-upgrade.py b/support/python3/test-upgrade.py new file mode 100755 index 0000000..a1745bd --- /dev/null +++ b/support/python3/test-upgrade.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to test the upgrade path between two versions of Mesos.""" + +import argparse +import os +import subprocess +import sys +import tempfile +import time + +DEFAULT_PRINCIPAL = 'foo' +DEFAULT_SECRET = 'bar' + + +class Process(object): + """ + Helper class to keep track of process lifecycles. + + This class allows to start processes, capture their + output, and check their liveness during delays/sleep. + """ + + def __init__(self, args, environment=None): + """Initialize the Process.""" + outfile = tempfile.mktemp() + fout = open(outfile, 'w') + print('Run %s, output: %s' % (args, outfile)) + + # TODO(nnielsen): Enable glog verbose logging. + self.process = subprocess.Popen(args, + stdout=fout, + stderr=subprocess.STDOUT, + env=environment) + + def sleep(self, seconds): + """ + Poll the process for the specified number of seconds. + + If the process ends during that time, this method returns the process's + return value. If the process is still running after that time period, + this method returns `True`. + """ + poll_time = 0.1 + while seconds > 0: + seconds -= poll_time + time.sleep(poll_time) + poll = self.process.poll() + if poll != None: + return poll + return True + + def __del__(self): + """Kill the Process.""" + if self.process.poll() is None: + self.process.kill() + + +class Agent(Process): + """Class representing an agent process.""" + + def __init__(self, path, work_dir, credfile): + """Initialize a Mesos agent by running mesos-slave.sh.""" + Process.__init__(self, [os.path.join(path, 'bin', 'mesos-slave.sh'), + '--master=127.0.0.1:5050', + '--credential=' + credfile, + '--work_dir=' + work_dir, + '--resources=disk:2048;mem:2048;cpus:2']) + + +class Master(Process): + """Class representing a master process.""" + + def __init__(self, path, work_dir, credfile): + """Initialize a Mesos master by running mesos-master.sh.""" + Process.__init__(self, [os.path.join(path, 'bin', 'mesos-master.sh'), + '--ip=127.0.0.1', + '--work_dir=' + work_dir, + '--authenticate', + '--credentials=' + credfile, + '--roles=test']) + + +# TODO(greggomann): Add support for multiple frameworks. +class Framework(Process): + """Class representing a framework instance (the test-framework for now).""" + + def __init__(self, path): + """Initialize a framework.""" + # The test-framework can take these parameters as environment variables, + # but not as command-line parameters. + environment = { + # In Mesos 0.28.0, the `MESOS_BUILD_DIR` environment variable in the + # test framework was changed to `MESOS_HELPER_DIR`, and the '/src' + # subdirectory was added to the variable's path. Both are included + # here for backwards compatibility. + 'MESOS_BUILD_DIR': path, + 'MESOS_HELPER_DIR': os.path.join(path, 'src'), + # MESOS_AUTHENTICATE is deprecated in favor of + # MESOS_AUTHENTICATE_FRAMEWORKS, although 0.28.x still expects + # previous one, therefore adding both. + 'MESOS_AUTHENTICATE': '1', + 'MESOS_AUTHENTICATE_FRAMEWORKS': '1', + 'DEFAULT_PRINCIPAL': DEFAULT_PRINCIPAL, + 'DEFAULT_SECRET': DEFAULT_SECRET + } + + Process.__init__(self, [os.path.join(path, 'src', 'test-framework'), + '--master=127.0.0.1:5050'], environment) + + +def version(path): + """Get the Mesos version from the built executables.""" + mesos_master_path = os.path.join(path, 'bin', 'mesos-master.sh') + process = subprocess.Popen([mesos_master_path, '--version'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, _ = process.communicate() + return_code = process.returncode + if return_code != 0: + return False + + return output[:-1] + + +def create_master(master_version, build_path, work_dir, credfile): + """Create a master using a specific version.""" + print('##### Starting %s master #####' % master_version) + master = Master(build_path, work_dir, credfile) + if not master.sleep(0.5): + print('%s master exited prematurely' % master_version) + sys.exit(1) + return master + + +def create_agent(agent_version, build_path, work_dir, credfile): + """Create an agent using a specific version.""" + print('##### Starting %s agent #####' % agent_version) + agent = Agent(build_path, work_dir, credfile) + if not agent.sleep(0.5): + print('%s agent exited prematurely' % agent_version) + sys.exit(1) + return agent + + +def test_framework(framework_version, build_path): + """Run a version of the test framework on a specified version of Mesos.""" + print('##### Starting %s framework #####' % framework_version) + print('Waiting for %s framework to complete (10 sec max)...' % ( + framework_version)) + framework = Framework(build_path) + if framework.sleep(10) != 0: + print('%s framework failed' % framework_version) + sys.exit(1) + + +# TODO(nnielsen): Add support for zookeeper and failover of master. +# TODO(nnielsen): Add support for testing scheduler live upgrade/failover. +def main(): + """Main function to test the upgrade between two Mesos builds.""" + parser = argparse.ArgumentParser( + description='Test upgrade path between two mesos builds') + parser.add_argument('--prev', + type=str, + help='Build path to mesos version to upgrade from', + required=True) + + parser.add_argument('--next', + type=str, + help='Build path to mesos version to upgrade to', + required=True) + args = parser.parse_args() + + # Get the version strings from the built executables. + prev_version = version(args.prev) + next_version = version(args.__next__) + + if not prev_version or not next_version: + print('Could not get mesos version numbers') + sys.exit(1) + + # Write credentials to temporary file. + credfile = tempfile.mktemp() + with open(credfile, 'w') as fout: + fout.write(DEFAULT_PRINCIPAL + ' ' + DEFAULT_SECRET) + + # Create a work directory for the master. + master_work_dir = tempfile.mkdtemp() + + # Create a work directory for the agent. + agent_work_dir = tempfile.mkdtemp() + + print('Running upgrade test from %s to %s' % (prev_version, next_version)) + + print("""\ ++--------------+----------------+----------------+---------------+ +| Test case | Framework | Master | Agent | ++--------------+----------------+----------------+---------------+ +| #1 | %s\t| %s\t | %s\t | +| #2 | %s\t| %s\t | %s\t | +| #3 | %s\t| %s\t | %s\t | +| #4 | %s\t| %s\t | %s\t | ++--------------+----------------+----------------+---------------+ + +NOTE: live denotes that master process keeps running from previous case. + """ % (prev_version, prev_version, prev_version, + prev_version, next_version, prev_version, + prev_version, next_version, next_version, + next_version, next_version, next_version)) + + # Test case 1. + master = create_master(prev_version, args.prev, master_work_dir, credfile) + agent = create_agent(prev_version, args.prev, agent_work_dir, credfile) + test_framework(prev_version, args.prev) + + # Test case 2. + # NOTE: Need to stop and start the agent because standalone detector does + # not detect master failover. + agent.process.kill() + master.process.kill() + master = create_master(next_version, args.__next__, master_work_dir, + credfile) + agent = create_agent(prev_version, args.prev, agent_work_dir, credfile) + test_framework(prev_version, args.prev) + + # Test case 3. + agent.process.kill() + agent = create_agent(next_version, args.__next__, agent_work_dir, credfile) + test_framework(prev_version, args.prev) + + # Test case 4. + test_framework(next_version, args.__next__) + + # Tests passed. + sys.exit(0) + +if __name__ == '__main__': + main() http://git-wip-us.apache.org/repos/asf/mesos/blob/960df5c4/support/python3/verify-reviews.py ---------------------------------------------------------------------- diff --git a/support/python3/verify-reviews.py b/support/python3/verify-reviews.py new file mode 100755 index 0000000..2e92590 --- /dev/null +++ b/support/python3/verify-reviews.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script is used to build and test (verify) reviews that are posted +to ReviewBoard. The script is intended for use by automated "ReviewBots" +that are run on ASF infrastructure (or by anyone that wishes to donate +some compute power). For example, see 'support/jenkins/reviewbot.sh'. + +The script performs the following sequence: +* A query grabs review IDs from Reviewboard. +* In reverse order (most recent first), the script determines if the + review needs verification (if the review has been updated or changed + since the last run through this script). +* For each review that needs verification: + * The review is applied (via 'support/python3/apply-reviews.py'). + * Mesos is built and unit tests are run. + * The result is posted to ReviewBoard. +""" + +import atexit +import json +import os +import platform +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +from datetime import datetime + +REVIEWBOARD_URL = "https://reviews.apache.org" +REVIEW_SIZE = 1000000 # 1 MB in bytes. + +# TODO(vinod): Use 'argparse' module. +# Get the user and password from command line. +if len(sys.argv) < 3: + print("Usage: ./verify-reviews.py <user>" + "<password> [num-reviews] [query-params]") + sys.exit(1) + +USER = sys.argv[1] +PASSWORD = sys.argv[2] + +# Number of reviews to verify. +NUM_REVIEWS = -1 # All possible reviews. +if len(sys.argv) >= 4: + NUM_REVIEWS = int(sys.argv[3]) + +# Unless otherwise specified consider pending review requests to Mesos updated +# since 03/01/2014. +GROUP = "mesos" +LAST_UPDATED = "2014-03-01T00:00:00" +QUERY_PARAMS = "?to-groups=%s&status=pending&last-updated-from=%s" \ + % (GROUP, LAST_UPDATED) +if len(sys.argv) >= 5: + QUERY_PARAMS = sys.argv[4] + + +class ReviewError(Exception): + """Exception returned by post_review().""" + pass + + +def shell(command): + """Run a shell command.""" + print(command) + return subprocess.check_output( + command, stderr=subprocess.STDOUT, shell=True) + + +HEAD = shell("git rev-parse HEAD") + + +def api(url, data=None): + """Call the ReviewBoard API.""" + try: + auth_handler = urllib.request.HTTPBasicAuthHandler() + auth_handler.add_password( + realm="Web API", + uri="reviews.apache.org", + user=USER, + passwd=PASSWORD) + + opener = urllib.request.build_opener(auth_handler) + urllib.request.install_opener(opener) + + return json.loads(urllib.request.urlopen(url, data=data).read()) + except urllib.error.HTTPError as err: + print("Error handling URL %s: %s (%s)" % (url, err.reason, err.read())) + exit(1) + except urllib.error.URLError as err: + print("Error handling URL %s: %s" % (url, err.reason)) + exit(1) + + +def apply_review(review_id): + """Apply a review using the script apply-reviews.py.""" + print("Applying review %s" % review_id) + shell("python support/python3/apply-reviews.py -n -r %s" % review_id) + + +def apply_reviews(review_request, reviews): + """Apply multiple reviews at once.""" + # If there are no reviewers specified throw an error. + if not review_request["target_people"]: + raise ReviewError("No reviewers specified. Please find a reviewer by" + " asking on JIRA or the mailing list.") + + # If there is a circular dependency throw an error.` + if review_request["id"] in reviews: + raise ReviewError("Circular dependency detected for review %s." + "Please fix the 'depends_on' field." + % review_request["id"]) + else: + reviews.append(review_request["id"]) + + # First recursively apply the dependent reviews. + for review in review_request["depends_on"]: + review_url = review["href"] + print("Dependent review: %s " % review_url) + apply_reviews(api(review_url)["review_request"], reviews) + + # Now apply this review if not yet submitted. + if review_request["status"] != "submitted": + apply_review(review_request["id"]) + + +def post_review(review_request, message): + """Post a review on the review board.""" + print("Posting review: %s" % message) + + review_url = review_request["links"]["reviews"]["href"] + data = urllib.parse.urlencode({'body_top': message, 'public': 'true'}) + api(review_url, data) + + +@atexit.register +def cleanup(): + """Clean the git repository.""" + try: + shell("git clean -fd") + shell("git reset --hard %s" % HEAD) + except subprocess.CalledProcessError as err: + print("Failed command: %s\n\nError: %s" % (err.cmd, err.output)) + + +def verify_review(review_request): + """Verify a review.""" + print("Verifying review %s" % review_request["id"]) + build_output = "build_" + str(review_request["id"]) + + try: + # Recursively apply the review and its dependents. + reviews = [] + apply_reviews(review_request, reviews) + + reviews.reverse() # Reviews are applied in the reverse order. + + command = "" + if platform.system() == 'Windows': + command = "support\\windows-build.bat" + + # There is no equivalent to `tee` on Windows. + subprocess.check_call( + ['cmd', '/c', '%s 2>&1 > %s' % (command, build_output)]) + else: + # Launch docker build script. + + # TODO(jojy): Launch 'docker_build.sh' in subprocess so that + # verifications can be run in parallel for various configurations. + configuration = ("export " + "OS='ubuntu:14.04' " + "BUILDTOOL='autotools' " + "COMPILER='gcc' " + "CONFIGURATION='--verbose " + "--disable-libtool-wrappers' " + "ENVIRONMENT='GLOG_v=1 MESOS_VERBOSE=1'") + + command = "%s; ./support/docker-build.sh" % configuration + + # `tee` the output so that the console can log the whole build + # output. `pipefail` ensures that the exit status of the build + # command ispreserved even after tee'ing. + subprocess.check_call(['bash', '-c', + ('set -o pipefail; %s 2>&1 | tee %s') + % (command, build_output)]) + + # Success! + post_review( + review_request, + "Patch looks great!\n\n" \ + "Reviews applied: %s\n\n" \ + "Passed command: %s" % (reviews, command)) + except subprocess.CalledProcessError as err: + # If we are here because the docker build command failed, read the + # output from `build_output` file. For all other command failures read + # the output from `e.output`. + if os.path.exists(build_output): + output = open(build_output).read() + else: + output = err.output + + if platform.system() == 'Windows': + # We didn't output anything during the build (because `tee` + # doesn't exist), so we print the output to stdout upon error. + + # Pylint raises a no-member error on that line due to a bug + # fixed in pylint 1.7. + # TODO(ArmandGrillet): Remove this once pylint updated to >= 1.7. + # pylint: disable=no-member + sys.stdout.buffer.write(output) + + # Truncate the output when posting the review as it can be very large. + if len(output) > REVIEW_SIZE: + output = "...<truncated>...\n" + output[-REVIEW_SIZE:] + + output += "\nFull log: " + output += urllib.parse.urljoin(os.environ['BUILD_URL'], 'console') + + post_review( + review_request, + "Bad patch!\n\n" \ + "Reviews applied: %s\n\n" \ + "Failed command: %s\n\n" \ + "Error:\n%s" % (reviews, err.cmd, output)) + except ReviewError as err: + post_review( + review_request, + "Bad review!\n\n" \ + "Reviews applied: %s\n\n" \ + "Error:\n%s" % (reviews, err.args[0])) + + # Clean up. + cleanup() + + +def needs_verification(review_request): + """Return True if this review request needs to be verified.""" + print("Checking if review: %s needs verification" % review_request["id"]) + + # Skip if the review blocks another review. + if review_request["blocks"]: + print("Skipping blocking review %s" % review_request["id"]) + return False + + diffs_url = review_request["links"]["diffs"]["href"] + diffs = api(diffs_url) + + if not diffs["diffs"]: # No diffs attached! + print("Skipping review %s as it has no diffs" % review_request["id"]) + return False + + # Get the timestamp of the latest diff. + timestamp = diffs["diffs"][-1]["timestamp"] + rb_date_format = "%Y-%m-%dT%H:%M:%SZ" + diff_time = datetime.strptime(timestamp, rb_date_format) + print("Latest diff timestamp: %s" % diff_time) + + # Get the timestamp of the latest review from this script. + reviews_url = review_request["links"]["reviews"]["href"] + reviews = api(reviews_url + "?max-results=200") + review_time = None + for review in reversed(reviews["reviews"]): + if review["links"]["user"]["title"] == USER: + timestamp = review["timestamp"] + review_time = datetime.strptime(timestamp, rb_date_format) + print("Latest review timestamp: %s" % review_time) + break + + # TODO: Apply this check recursively up the dependency chain. + changes_url = review_request["links"]["changes"]["href"] + changes = api(changes_url) + dependency_time = None + for change in changes["changes"]: + if "depends_on" in change["fields_changed"]: + timestamp = change["timestamp"] + dependency_time = datetime.strptime(timestamp, rb_date_format) + print("Latest dependency change timestamp: %s" % dependency_time) + break + + # Needs verification if there is a new diff, or if the dependencies changed, + # after the last time it was verified. + return not review_time or review_time < diff_time or \ + (dependency_time and review_time < dependency_time) + + +def main(): + """Main function to verify the submitted reviews.""" + review_requests_url = \ + "%s/api/review-requests/%s" % (REVIEWBOARD_URL, QUERY_PARAMS) + + review_requests = api(review_requests_url) + num_reviews = 0 + for review_request in reversed(review_requests["review_requests"]): + if (NUM_REVIEWS == -1 or num_reviews < NUM_REVIEWS) and \ + needs_verification(review_request): + verify_review(review_request) + num_reviews += 1 + +if __name__ == '__main__': + main()