Repository: ambari Updated Branches: refs/heads/branch-dev-patch-upgrade a25613025 -> 0f2c3375d
AMBARI-14946. Create python script for generating version definition file (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f2c3375 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f2c3375 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f2c3375 Branch: refs/heads/branch-dev-patch-upgrade Commit: 0f2c3375da8b3a5ab5739a5ae79db44bf5f91815 Parents: a256130 Author: Nate Cole <nc...@hortonworks.com> Authored: Fri Feb 5 14:22:28 2016 -0500 Committer: Nate Cole <nc...@hortonworks.com> Committed: Fri Feb 5 14:22:28 2016 -0500 ---------------------------------------------------------------------- .../src/main/resources/version_definition.xsd | 7 +- contrib/version-builder/example.sh | 44 +++ contrib/version-builder/version_builder.py | 354 +++++++++++++++++++ 3 files changed, 402 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0f2c3375/ambari-server/src/main/resources/version_definition.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd index 2efdd77..77b4203 100644 --- a/ambari-server/src/main/resources/version_definition.xsd +++ b/ambari-server/src/main/resources/version_definition.xsd @@ -19,7 +19,7 @@ <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> <xs:annotation> <xs:documentation> - This XSD is used to validate a repo definition file. You can verify the XML is valid + This XSD is used to validate a version definition file. You can verify the XML is valid by running (on Linux): xmllint --noout --load-trace --schema [path-to-this-file] [path-to-xml] </xs:documentation> @@ -31,9 +31,10 @@ <xs:element name="stack-id" type="xs:string" /> <xs:element name="version" type="xs:string" /> <xs:element name="build" type="xs:string" /> - <xs:element name="compatible-with" type="xs:string" minOccurs="0" maxOccurs="1" /> + <xs:element name="compatible-with" type="xs:string" minOccurs="0"/> <xs:element name="release-notes" type="xs:string" maxOccurs="1" /> <xs:element name="display" type="xs:string" minOccurs="0" /> + <xs:element name="package-version" type="xs:string" minOccurs="0" /> </xs:all> </xs:complexType> @@ -183,4 +184,4 @@ </xs:element> -</xs:schema> \ No newline at end of file +</xs:schema> http://git-wip-us.apache.org/repos/asf/ambari/blob/0f2c3375/contrib/version-builder/example.sh ---------------------------------------------------------------------- diff --git a/contrib/version-builder/example.sh b/contrib/version-builder/example.sh new file mode 100755 index 0000000..a93ddb6 --- /dev/null +++ b/contrib/version-builder/example.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +filename="version_241-12345.xml" + +python version_builder.py --file $filename --release-type PATCH +python version_builder.py --file $filename --release-stack HDP-2.3 +python version_builder.py --file $filename --release-version 2.4.1.1 +python version_builder.py --file $filename --release-build 12345 +python version_builder.py --file $filename --release-notes http://example.com +python version_builder.py --file $filename --release-display HDP-2.4.1.1-1234-patch +python version_builder.py --file $filename --release-compatible 2.4.[0-1].0 + +# call any number of times for each service in the repo +python version_builder.py --file $filename --manifest --manifest-id HDFS-271 --manifest-service HDFS --manifest-version 2.7.1.2.4 +python version_builder.py --file $filename --manifest --manifest-id HBASE-132 --manifest-service HBASE --manifest-version 1.3.2.4.3 + +#call any number of times for the target services to upgrade +python version_builder.py --file $filename --available --manifest-id HDFS-271 + +#call any number of times for repo per os +python version_builder.py --file $filename --repo --repo-os redhat6 --repo-id HDP-2.3 --repo-name HDP --repo-url http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0 +python version_builder.py --file $filename --repo --repo-os redhat6 --repo-id HDP-UTILS-1.1.0.20 --repo-name HDP-UTILS --repo-url http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6 + +python version_builder.py --file $filename --finalize --xsd ../../ambari-server/src/main/resources/version_definition.xsd + +# to upload this to running Ambari instance on localhost: +# curl -u admin:admin -H 'Content-Type: text/xml' -X POST -d @$filename http://localhost:8080/api/v1/version_definitions http://git-wip-us.apache.org/repos/asf/ambari/blob/0f2c3375/contrib/version-builder/version_builder.py ---------------------------------------------------------------------- diff --git a/contrib/version-builder/version_builder.py b/contrib/version-builder/version_builder.py new file mode 100644 index 0000000..6c20a47 --- /dev/null +++ b/contrib/version-builder/version_builder.py @@ -0,0 +1,354 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import optparse +import os +import subprocess +import sys +import xml.etree.ElementTree as ET + +def load_file(filename): + """ + Loads the specified XML file + """ + if os.path.exists(filename): + tree = ET.ElementTree() + tree.parse(filename) + root = tree.getroot() + else: + attribs = {} + attribs['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance" + attribs['xsi:noNamespaceSchemaLocation'] = "version_definition.xsd" + root = ET.Element("repository-version", attribs) + + ET.SubElement(root, "release") + ET.SubElement(root, "manifest") + ET.SubElement(root, "available-services") + ET.SubElement(root, "repository-info") + + return root + +def save_file(xml, filename): + """ + Saves the XML file + """ + p = subprocess.Popen(['xmllint', '--format', '--output', filename, '-'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate(input=ET.tostring(xml)) + +def check_xmllint(): + """ + Verifies utility xmllint is available + """ + try: + p = subprocess.Popen(['xmllint', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + (stdout, stderr) = p.communicate() + + if p.returncode != 0: + raise Exception("xmllint command does not appear to be available") + + except: + raise Exception("xmllint command does not appear to be available") + + +def validate_file(filename, xsdfile): + """ + Validates the XML file against the XSD + """ + args = ['xmllint', '--noout', '--load-trace', '--schema', xsdfile, filename] + + p = subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + + if p.returncode != 0: + raise Exception(stderr) + + if len(stdout) > 0: + print stdout + + if len(stderr) > 0: + print stderr + + +def update_simple(parent, name, value): + """ + Helper method to either update or create the element + """ + element = parent.find('./' + name) + + if element is None: + element = ET.SubElement(parent, name) + element.text = value + else: + element.text = value + +def process_release(xmlroot, options): + """ + Create elements of the 'release' parent + """ + release_element = xmlroot.find("./release") + + if release_element is None: + raise Exception("Element 'release' is not found") + + if options.release_type: + update_simple(release_element, "type", options.release_type) + + if options.release_stack: + update_simple(release_element, "stack-id", options.release_stack) + + if options.release_version: + update_simple(release_element, "version", options.release_version) + + if options.release_build: + update_simple(release_element, "build", options.release_build) + + if options.release_compatible: + update_simple(release_element, "compatible-with", options.release_compatible) + + if options.release_notes: + update_simple(release_element, "release-notes", options.release_notes) + + if options.release_display: + update_simple(release_element, "display", options.release_display) + + if options.release_package_version: + update_simple(release_element, "package-version", options.release_package_version) + +def process_manifest(xmlroot, options): + """ + Creates the manifest element + """ + if not options.manifest: + return + + manifest_element = xmlroot.find("./manifest") + + if manifest_element is None: + raise Exception("Element 'manifest' is not found") + + service_element = manifest_element.find("./service[@id='{0}']".format(options.manifest_id)) + + if service_element is None: + service_element = ET.SubElement(manifest_element, "service") + service_element.set('id', options.manifest_id) + + service_element.set('name', options.manifest_service) + service_element.set('version', options.manifest_version) + if options.manifest_version_id: + service_element.set('version-id', options.manifest_version_id) + +def process_available(xmlroot, options): + """ + Processes available service elements + """ + if not options.available: + return + + manifest_element = xmlroot.find("./manifest") + if manifest_element is None: + raise Exception("'manifest' element is not found") + + service_element = manifest_element.find("./service[@id='{0}']".format(options.manifest_id)) + if service_element is None: + raise Exception("Cannot add an available service for {0}; it's not on the manifest".format(options.manifest_id)) + + available_element = xmlroot.find("./available-services") + if available_element is None: + raise Exception("'available-services' is not found") + + service_element = available_element.find("./service[@idref='{0}']".format(options.manifest_id)) + + if service_element is not None: + available_element.remove(service_element) + + service_element = ET.SubElement(available_element, "service") + service_element.set('idref', options.manifest_id) + + if options.available_components: + components = options.available_components.split(',') + for component in components: + e = ET.SubElement(service_element, 'component') + e.text = component + + +def process_repo(xmlroot, options): + """ + Processes repository options. This method doesn't update or create individual elements, it + creates the entire repo structure + """ + if not options.repo: + return + + repo_parent = xmlroot.find("./repository-info") + if repo_parent is None: + raise Exception("'repository-info' element is not found") + + os_element = repo_parent.find("./os[@family='{0}']".format(options.repo_os)) + if os_element is None: + os_element = ET.SubElement(repo_parent, 'os') + os_element.set('family', options.repo_os) + + repo_element = os_element.find("./repo/[reponame='{0}']".format(options.repo_name)) + + if repo_element is not None: + os_element.remove(repo_element) + + repo_element = ET.SubElement(os_element, 'repo') + e = ET.SubElement(repo_element, 'baseurl') + e.text = options.repo_url + + e = ET.SubElement(repo_element, 'repoid') + e.text = options.repo_id + + e = ET.SubElement(repo_element, 'reponame') + e.text = options.repo_name + +def validate_manifest(parser, options): + """ + Validates manifest options from the command line + """ + if not options.manifest: + return + + template = "When specifying --manifest, {0} is also required" + + if not options.manifest_id: + parser.error(template.format("--manifest-id")) + + if not options.manifest_service: + parser.error(template.format("--manifest-service")) + + if not options.manifest_version: + parser.error(template.format("--manifest-version")) + +def validate_available(parser, options): + """ + Validates available service options from the command line + """ + if not options.available: + return + + if not options.manifest_id: + parser.error("When specifying --available, --manifest-id is also required") + +def validate_repo(parser, options): + """ + Validates repo options from the command line + """ + if not options.repo: + return + + template = "When specifying --repo, {0} is also required" + + if not options.repo_os: + parser.error(template.format("--repo-os")) + + if not options.repo_url: + parser.error(template.format("--repo-url")) + + if not options.repo_id: + parser.error(template.format("--repo-id")) + + if not options.repo_name: + parser.error(template.format("--repo-name")) + + +def main(argv): + parser = optparse.OptionParser( + epilog="OS utility 'xmllint' is required for this tool to function. It handles pretty-printing and XSD validation.") + + parser.add_option('--file', dest='filename', + help="The output XML file") + + parser.add_option('--finalize', action='store_true', dest='finalize', + help="Finalize and validate the XML file") + parser.add_option('--xsd', dest='xsd_file', + help="The XSD location when finalizing") + + parser.add_option('--release-type', type='choice', choices=['STANDARD', 'PATCH'], dest='release_type' , + help="Indicate the release type: i.e. STANDARD or PATCH") + parser.add_option('--release-stack', dest='release_stack', + help="The stack id: e.g. HDP-2.4") + parser.add_option('--release-version', dest='release_version', + help="The release version without build number: e.g. 2.4.0.1") + parser.add_option('--release-build', dest='release_build', + help="The release build number: e.g. 1234") + parser.add_option('--release-compatible', dest='release_compatible', + help="Regular Expression string to identify version compatibility for patches: e.g. 2.4.1.[0-9]") + parser.add_option('--release-notes', dest='release_notes', + help="A http link to the documentation notes") + parser.add_option('--release-display', dest='release_display', + help="The display name for this release") + parser.add_option('--release-package-version', dest='release_package_version', + help="Identifier to use when installing packages, generally a part of the package name") + + parser.add_option('--manifest', action='store_true', dest='manifest', + help="Add a manifest service with other options: --manifest-id, --manifest-service, --manifest-version, --manifest-version-id") + parser.add_option('--manifest-id', dest='manifest_id', + help="Unique ID for a service in a manifest. Required when specifying --manifest and --available") + parser.add_option('--manifest-service', dest='manifest_service') + parser.add_option('--manifest-version', dest='manifest_version') + parser.add_option('--manifest-version-id', dest='manifest_version_id') + + parser.add_option('--available', action='store_true', dest='available', + help="Add an available service with other options: --manifest-id, --available-components") + parser.add_option('--available-components', dest='available_components', + help="A CSV of service components that are intended to be upgraded via patch. \ + Omitting this implies the entire service should be upgraded") + + parser.add_option('--repo', action='store_true', dest='repo', + help="Add repository data with options: --repo-os, --repo-url, --repo-id, --repo-name") + parser.add_option('--repo-os', dest='repo_os', + help="The operating system type: i.e. redhat6, redhat7, debian7, ubuntu12, ubuntu14, suse11") + parser.add_option('--repo-url', dest='repo_url', + help="The base url for the repository data") + parser.add_option('--repo-id', dest='repo_id', help="The ID of the repo") + parser.add_option('--repo-name', dest='repo_name', help="The name of the repo") + + (options, args) = parser.parse_args() + + check_xmllint() + + # validate_filename + if not options.filename: + parser.error("--file option is required") + + validate_manifest(parser, options) + validate_available(parser, options) + validate_repo(parser, options) + + # validate_finalize + if options.finalize and not options.xsd_file: + parser.error("Must supply XSD (--xsd) when finalizing") + + # load file + root = load_file(options.filename) + + process_release(root, options) + process_manifest(root, options) + process_available(root, options) + process_repo(root, options) + + # save file + save_file(root, options.filename) + + if options.finalize: + validate_file(options.filename, options.xsd_file) + +if __name__ == "__main__": + main(sys.argv)