This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch Qinusty/235-manifest in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 38d5a4cf24956c4dc573add021f513a4975ff693 Author: Josh Smith <[email protected]> AuthorDate: Tue Jul 31 17:47:59 2018 +0100 cli.py: Add --build-manifest to build command This allows the user to opt in generating a build manifest containing all of the elements and sources used for their build alongside a buildstream version and build date/time. _manifest.py: New module to provide functionality for producing a manifest from a collection of elements. source.py: Add _get_element_index() to allow access to Source.__element_index. --- buildstream/_frontend/cli.py | 25 ++++++++- buildstream/_manifest.py | 124 +++++++++++++++++++++++++++++++++++++++++++ buildstream/source.py | 3 ++ 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 20624e2..b883956 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -3,6 +3,7 @@ import sys import click from .. import _yaml +from .. import _manifest from .._exceptions import BstError, LoadError, AppError from .._versions import BST_FORMAT_VERSION from .complete import main_bashcomplete, complete_path, CompleteUnhandled @@ -289,6 +290,15 @@ def init(app, project_name, format_version, element_path, force): ################################################################## # Build Command # ################################################################## +def _validate_manifest_path(ctx, param, value): + if not value: + return + if value.lower().endswith(".yaml") or value.lower().endswith(".yml"): + return os.path.abspath(value) + else: + raise click.BadParameter("Manifest files are outputted as YAML\n\t" + + "Please provide a path with a valid file extension (yml, yaml)") + @cli.command(short_help="Build elements in a pipeline") @click.option('--all', 'all_', default=False, is_flag=True, help="Build elements that would not be needed for the current build plan") @@ -305,10 +315,16 @@ def init(app, project_name, format_version, element_path, force): help="Allow tracking to cross junction boundaries") @click.option('--track-save', default=False, is_flag=True, help="Deprecated: This is ignored") [email protected]('--build-manifest', default=False, is_flag=True, + help="Produces a build manifest containing elements and sources.") [email protected]('--manifest-path', default=None, type=click.Path(readable=False), + help="Provides a path for a build manifest to be written to.", + callback=_validate_manifest_path) @click.argument('elements', nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions): +def build(app, elements, all_, track_, track_save, build_manifest, + manifest_path, track_all, track_except, track_cross_junctions): """Build elements in a pipeline""" if (track_except or track_cross_junctions) and not (track_ or track_all): @@ -329,6 +345,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac track_cross_junctions=track_cross_junctions, build_all=all_) + if build_manifest and not manifest_path: + manifest_path = os.path.join(app.project.directory, + "build_manifest.yaml") + if manifest_path: + _manifest.generate(app.context, app.stream.total_elements, + app._session_start, manifest_path) + ################################################################## # Fetch Command # diff --git a/buildstream/_manifest.py b/buildstream/_manifest.py new file mode 100644 index 0000000..a897866 --- /dev/null +++ b/buildstream/_manifest.py @@ -0,0 +1,124 @@ +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Josh Smith <[email protected]> + +from ruamel import yaml +from . import __version__ as bst_version +from . import _yaml +from ._message import MessageType, Message + +# generate(): +# +# Generates a manfiest file from the collection of elements. +# +# Args: +# context (Context): Application context +# elements (list of Element): Collection of elements to build the manifest from +# build_start_datetime (datetime): The start of the build assosciated with this manifest +# manifest_path (str): Absolute path to write the manifest file to. +# +def generate(context, elements, build_start_datetime, manifest_path): + with context.timed_activity("Building Manifest"): + manifest = _build(elements, build_start_datetime) + + _yaml.dump(manifest, manifest_path) + context.message(Message(None, MessageType.STATUS, + "Manifest saved to {}".format(manifest_path))) + +# _build(): +# +# Builds a manifest (dictionary) using the provided elements to be stored. +# +# Args: +# elements (list of Element): Collection of elements to build the manifest from +# build_start_datetime (datetime): The start of the build assosciated with this manifest +# +# Returns: +# (CommentedMap): A dictionary containing the entire +# manifest produced from the provided elements. +# +def _build(elements, build_start_datetime): + manifest = yaml.comments.CommentedMap() + + # Add BuildStream Version + manifest['BuildStream_Version'] = "{}".format(bst_version) + # Add Build Date + manifest['Build_Date'] = build_start_datetime.isoformat() + manifest['Elements'] = yaml.comments.CommentedMap() + + # Sort elements + elements = sorted(elements, key=lambda e: e.normal_name) + + # Add Elements + for elem in elements: + manifest['Elements'][elem.normal_name] = _build_element(elem) + + return manifest + +# _build_element(): +# +# Builds a manifest segment for an individual element. +# +# Args: +# element (Element): Element to extract information from +# +# Returns: +# (CommentedMap): A dictionary containing the information +# extracted from the provided element +# +def _build_element(element): + element_dict = yaml.comments.CommentedMap() + sources = yaml.comments.CommentedMap() + # Add Cache Key + cache_key = element._get_cache_key() + if cache_key: + element_dict["Cache_Key"] = cache_key + + # Add sources + for source in element.sources(): + src = _build_source(source) + if src: + source_desc = "{}({})".format(source._get_element_index(), type(source).__name__) + sources[source_desc] = src + if sources: + element_dict['Sources'] = sources + + + return element_dict + +# _build_source(): +# +# Builds a manifest segment for an individual source. +# +# Args: +# source (Source): Source to extract information from +# +# Returns: +# (CommentedMap): A dictionary containing the information +# extracted from the provided source +# +def _build_source(source): + src = yaml.comments.CommentedMap() + if hasattr(source, "url") and source.url: + src["url"] = source.url + if hasattr(source, "ref") and source.ref: + src["ref"] = source.ref + if hasattr(source, "path") and source.path: + src["path"] = source.path + + return src if src else None diff --git a/buildstream/source.py b/buildstream/source.py index 2f3f1c2..8f45ce0 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -786,6 +786,9 @@ class Source(Plugin): else: return None + def _get_element_index(self): + return self.__element_index + ############################################################# # Local Private Methods # #############################################################
