Repository: incubator-reef Updated Branches: refs/heads/master 1dd68f70e -> 4a9796792
[REEF-498] Create scripts for release versioning, packaging, checksum This pull request addressed the issue by * creating 'release.py' which create a release candidate and relevant files. * creating 'change_version.py' which changes versions in every pom.xml and relevant files. JIRA: [REEF-498](https://issues.apache.org/jira/browse/REEF-498) Pull Request: This closes #331 Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/4a979679 Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/4a979679 Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/4a979679 Branch: refs/heads/master Commit: 4a9796792e1d68ee9a0adc221626443ba7df7dee Parents: 1dd68f7 Author: dongjun-Lee <ga...@snu.ac.kr> Authored: Mon Aug 3 14:57:59 2015 +0900 Committer: Brian Cho <chobr...@apache.org> Committed: Tue Aug 25 17:06:48 2015 +0900 ---------------------------------------------------------------------- dev/change_version.py | 276 +++++++++++++++++++++++++++++++++++++++++++++ dev/release.py | 157 ++++++++++++++++++++++++++ 2 files changed, 433 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4a979679/dev/change_version.py ---------------------------------------------------------------------- diff --git a/dev/change_version.py b/dev/change_version.py new file mode 100644 index 0000000..7ffb5ed --- /dev/null +++ b/dev/change_version.py @@ -0,0 +1,276 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This script changes versions in every pom.xml and relevant files. + +(How to run) +python change_version <reef_home> <reef_version_for_pom.xml> -s <true or false> (optional) -p + +-s option changes value of 'IsSnapshot' in lang/cs/build.props. +If you use the option "-s false", bulid.props changes as, + <RemoveIncubating>true</RemoveIncubating> + <IsSnapshot>false</IsSnapshot> + <SnapshotNumber>00</SnapshotNumber> + +If you use "-s true", then the value of 'IsSnapshot' is changed to true. + +If you use "-p", then only the "pom.xml" files are changed. + +You can also see how to run the script with "python change_version.py -h" + +(Example) +python change_version ~/incubator_reef 0.12.0-incubating -s true +python change_version ~/incubator_reef 0.12.0-incubating -s false +python change_version ~/incubator_reef 0.12.0-incubating -p -s true +""" + + +import os +import re +import sys +import argparse + +""" +Get list of path for every file in a directory +""" +def get_filepaths(directory): + file_paths = [] + + for root, directories, files in os.walk(directory): + for filename in files: + filepath = os.path.join(root, filename) + file_paths.append(filepath) + return file_paths + +""" +Change REEF version to new_version in every pom.xml +""" +def change_pom(file, new_version): + changed_str = "" + ready_to_change = False + + f = open(file, 'r') + + while True: + line = f.readline() + if not line: + break + if "<groupId>org.apache.reef</groupId>" in line: + ready_to_change = True + if "<version>" in line and ready_to_change: + break + changed_str += line + + r = re.compile('<version>(.*?)</version>') + m = r.search(line) + old_version = m.group(1) + changed_str += line.replace(old_version, new_version) + + while True: + line = f.readline() + if not line: + break + changed_str += line + + f = open(file, 'w') + f.write(changed_str) + f.close() + +""" +Change JavaBridgeJarFileName in lang/cs/Org.Apache.REEF.Driver/Constants.cs +""" +def change_constants_cs(file, new_version): + changed_str = "" + + f = open(file, 'r') + while True: + line = f.readline() + if not line: + break + + if "JavaBridgeJarFileName" in line: + r = re.compile('"(.*?)"') + m = r.search(line) + old_version = m.group(1) + new_version = "reef-bridge-java-" + new_version + "-shaded.jar" + changed_str += line.replace(old_version, new_version) + else: + changed_str += line + + f = open(file, 'w') + f.write(changed_str) + f.close() + +""" +Change version in every AssemblyInfo.cs and lang/cs/Org.Apache.REEF.Bridge/AssemblyInfo.cpp +""" +def change_assembly_info_cs(file, new_version): + changed_str = "" + new_version = new_version.split("-")[0] + ".0" + + f = open(file, 'r') + r = re.compile('"(.*?)"') + + while True: + line = f.readline() + if not line: + break + if ("[assembly: AssemblyVersion(" in line and "*" not in line) or ("[assembly: AssemblyFileVersion(" in line) \ + or ("[assembly:AssemblyVersionAttribute(" in line and "*" not in line) \ + or ("[assembly:AssemblyFileVersion(" in line): + m = r.search(line) + old_version = m.group(1) + changed_str += line.replace(old_version, new_version) + else: + changed_str += line + + f = open(file, 'w') + f.write(changed_str) + f.close() + +""" +Read 'IsSnapshot' from lang/cs/build.props +""" +def read_is_snapshot(file): + f = open(file, 'r') + r = re.compile('<IsSnapshot>(.*?)</IsSnapshot>') + while True: + line = f.readline() + if not line: + break + if "<IsSnapshot>" in line and "</IsSnapshot>" in line: + m = r.search(line) + if(m.group(1)=="true"): + return True + else: + return False + +""" +Change lang/cs/build.props for the release branch +""" +def change_build_props(file, is_snapshot): + changed_str = "" + + f = open(file, 'r') + r1 = re.compile('<RemoveIncubating>(.*?)</RemoveIncubating>') + r2 = re.compile('<IsSnapshot>(.*?)</IsSnapshot>') + r3 = re.compile('<SnapshotNumber>(.*?)</SnapshotNumber>') + + while True: + line = f.readline() + if not line: + break + if "<RemoveIncubating>" and "</RemoveIncubating>" in line and is_snapshot=="false": + old_remove_incubating = r1.search(line).group(1) + changed_str += line.replace(old_remove_incubating, "true") + elif "<IsSnapshot>" in line and "</IsSnapshot>" in line: + old_is_snapshot = r2.search(line).group(1) + changed_str += line.replace(old_is_snapshot, is_snapshot) + elif "<SnapshotNumber>" in line and "</SnapshotNumber>" in line and is_snapshot=="false": + old_snapshot_number = r3.search(line).group(1) + changed_str += line.replace(old_snapshot_number, "00") + else: + changed_str += line + + f = open(file, 'w') + f.write(changed_str) + f.close() + + print file + +""" +Change the name of shaded.jar in run.cmd and lang/cs/Org.Apache.REEF.Client/Properties/Resources.xml +""" +def change_shaded_jar_name(file, new_version): + changed_str = "" + + f = open(file, 'r') + r1 = re.compile('reef-bridge-java-(.*?)-shaded.jar') + r2 = re.compile('reef-bridge-client-(.*?)-shaded.jar') + while True: + line = f.readline() + if not line: + break + m1 = r1.search(line) + m2 = r2.search(line) + if m1 is not None: + changed_str += line.replace(m1.group(1), new_version) + elif m2 is not None: + changed_str += line.replace(m2.group(1), new_version) + else: + changed_str += line + + f = open(file, 'w') + f.write(changed_str) + f.close() + +""" +Change version of every pom.xml, every AsssemblyInfo.cs, +Constants.cs, AssemblyInfo.cpp, run.cmd and Resources.xml +""" +def change_version(reef_home, new_version, pom_only): + if pom_only: + for fi in get_filepaths(reef_home): + if "pom.xml" in fi: + print fi + change_pom(fi, new_version) + + else: + for fi in get_filepaths(reef_home): + if "pom.xml" in fi: + print fi + change_pom(fi, new_version) + if "AssemblyInfo.cs" in fi: + print fi + change_assembly_info_cs(fi, new_version) + + change_assembly_info_cs(reef_home + "/lang/cs/Org.Apache.REEF.Bridge/AssemblyInfo.cpp", new_version) + print reef_home + "/lang/cs/Org.Apache.REEF.Bridge/AssemblyInfo.cpp" + + change_constants_cs(reef_home + "/lang/cs/Org.Apache.REEF.Driver/Constants.cs", new_version) + print reef_home + "/lang/cs/Org.Apache.REEF.Driver/Constants.cs" + + change_shaded_jar_name(reef_home + "/lang/cs/Org.Apache.REEF.Client/Properties/Resources.xml", new_version) + print reef_home + "/lang/cs/Org.Apache.REEF.Client/Properties/Resources.xml" + + change_shaded_jar_name(reef_home + "/lang/cs/Org.Apache.REEF.Client/run.cmd", new_version) + print reef_home + "/lang/cs/Org.Apache.REEF.Client/run.cmd" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Script for changing version of every pom.xml, " \ + + "every AssemblyInfo.cs, Constants.cs, and AssemblyInfo.cpp") + parser.add_argument("reef_home", type=str, help="REEF home") + parser.add_argument("reef_version", type=str, help="REEF version") + parser.add_argument("-s", "--isSnapshot", type=str, metavar="<true or false>", help="Change 'IsSnapshot' to true or false", required=True) + parser.add_argument("-p", "--pomonly", help="Change only poms", action="store_true") + args = parser.parse_args() + + reef_home = os.path.abspath(args.reef_home) + reef_version = args.reef_version + is_snapshot = args.isSnapshot + pom_only = args.pomonly + + if is_snapshot is not None and not pom_only: + change_build_props(reef_home + "/lang/cs/build.props", is_snapshot) + + if is_snapshot=="true": + reef_version += "-SNAPSHOT" + + change_version(reef_home, reef_version, pom_only) + http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4a979679/dev/release.py ---------------------------------------------------------------------- diff --git a/dev/release.py b/dev/release.py new file mode 100644 index 0000000..d0dbf69 --- /dev/null +++ b/dev/release.py @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +""" +This script creates a release candidate and relevant files. + +(How to run) +python release.py <reef_home> <reef_version> <rc candidate number> <public key id (8 hex numbers)> + +You can also see how to run the script with 'python release.py -h' + +(Examples) +python release.py ~/incubator-reef 0.12.0-incubating 1 E488F925 + +""" + + +import subprocess +import fnmatch +import tarfile +import hashlib +import sys +import os +import argparse + +""" +Get list from .gitignore +""" +def get_ignore_list(reef_home): + f = open(reef_home + "/.gitignore", "r") + if f is None: + return [] + + ignore_list = list() + + while True: + line = f.readline()[:-1] + if not line: + break + if not "#" in line: + ignore_list.insert(0, "*/" + line) + + return ignore_list + +""" +Make text for e-mail +""" +def get_mail_text(reef_version, rc_num): + file_name = "apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz" + + return_str = "" + return_str += "This is to call for a new vote for the source release of Apache REEF " \ + + reef_version + " (rc" + str(rc_num) + ").\n\n" + return_str += "The source tar ball, including signatures, digests, etc can be found at:\n" + "<yours>\n\n" + return_str += "The Git tag is release-" + reef_version + "-rc" + str(rc_num) + "\n" + return_str += "The Git commit ID is <Git Commit ID>\n\n\n" + + return_str += "Checksums of apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz:\n\n" + + md5 = open(file_name + ".md5").read().split(" ")[0] + return_str += "MD5: " + md5 + "\n" + + sha = open(file_name + ".sha512.txt").read().split(" ")[0] + return_str += "SHA: " + sha + "\n" + + return_str += "\nRelease artifacts are signed with a key found in the KEYS file available here:\n" + return_str += "\nhttps://dist.apache.org/repos/dist/release/incubator/reef/KEYS\n\n\n\n"; + + return_str += "<Issue Things>\n\n\n\n" + + return_str += "The vote will be open for 72 hours. Please download the release\n" \ + + "candidate, check the hashes/signature, build it and test it, and then\nplease vote:\n\n" \ + + "[ ] +1 Release this package as Apache REEF " + reef_version + "\n" \ + + "[ ] +0 no opinion\n" \ + + "[ ] -1 Do not release this package because ...\n\n" \ + + "Thanks!" + + return return_str + +""" +Function to exclude files in .gitignore when making tar.gz +Return true when a file is in .gitignore +""" +def exclude_git_ignore(file_name): + ignore_list = get_ignore_list(reef_home) + for e in ignore_list: + if fnmatch.fnmatch(file_name, e): + return True + + return fnmatch.fnmatch(file_name, "*/.git") or fnmatch.fnmatch(file_name, "*/.gitignore") or \ + fnmatch.fnmatch(file_name, "*/.gitattributes") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Script for release versioning, packaging, checksum") + + parser.add_argument("reef_home", type=str, help="REEF home") + parser.add_argument("reef_version", type=str, help="REEF version") + parser.add_argument("rc_num", type=int, help ="rc candidate number") + parser.add_argument("key_id", type=str, help = "public key id (8 hex numbers)") + args = parser.parse_args() + + reef_home = os.path.abspath(args.reef_home) + reef_version = args.reef_version + rc_num = args.rc_num + key_id = args.key_id + + build_result = subprocess.call("cd " + reef_home + " && " + "mvn apache-rat:check", shell=True) + + if build_result == 0: + file_name = "apache-reef-" + reef_version + "-rc" + str(rc_num) + ".tar.gz" + + # Make tar.gz + tar = tarfile.open(file_name, "w:gz") + tar.add(reef_home, arcname="apache-reef-"+reef_version , exclude=exclude_git_ignore) + tar.close() + + gpg_str = "gpg --armor -u " + str(key_id) + " --output " + file_name + ".asc " + "--detach-sig " + file_name + gpg_result = subprocess.call(gpg_str, shell=True) + + if gpg_result == 0: + md5 = hashlib.md5(open(file_name,'rb').read()).hexdigest() + sha = hashlib.sha512(open(file_name,'rb').read()).hexdigest() + + md5_file = open(file_name + ".md5", "w") + md5_file.write(md5 + " *" + file_name + "\n") + sha_file = open(file_name + ".sha512.txt", "w") + sha_file.write(sha + " *" + file_name + "\n") + md5_file.close() + sha_file.close() + + # Make a Text for an e-mail + print "\n==================================Result===================================" + print get_mail_text(reef_version, rc_num) + print "===========================================================================\n" + + + else: + print "gpg error" + + else: + print "build error - mvn apache-rat:check failed"
