This is an automated email from the ASF dual-hosted git repository. root pushed a commit to branch tristan/multiple-caches in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f3447446606bf3be782f21de10b02d3f11ecc10c Author: Sam Thursfield <[email protected]> AuthorDate: Wed Nov 22 14:58:09 2017 +0000 Add support for multiple remote caches This extends the 'artifacts' configuration block such that a list of `url` mappings can be given instead of a single entry. For example: artifacts: - url: http://example.com/artifacts1 - url: ssh://[email protected]/artifacts2 The OSTreeCache class is updated to set up multiple remotes and query remote refs from all of them. There are no automated tests for this yet. Empty URLs ('') now raise an exception. They cause breakages internally if we allow them through, and they can only occur if the user or our tests are misconfiguring things somehow. We report failure to fetch from the cache by printing a message to stderr for now. This is because BuildStream's actual logging functionality can't be used during frontend init -- see issue #168. --- buildstream/_artifactcache/__init__.py | 2 +- buildstream/_artifactcache/artifactcache.py | 136 +++++++++++------- buildstream/_artifactcache/ostreecache.py | 214 +++++++++++++++++----------- buildstream/_context.py | 9 +- buildstream/_ostree.py | 2 +- buildstream/_pipeline.py | 32 ++--- buildstream/_project.py | 5 +- buildstream/data/userconfig.yaml | 8 -- buildstream/utils.py | 30 ++++ tests/frontend/pull.py | 33 +---- tests/frontend/push.py | 33 +---- tests/testutils/__init__.py | 2 +- tests/testutils/artifactshare.py | 34 +++++ 13 files changed, 317 insertions(+), 223 deletions(-) diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py index 991e848..7dc0ce3 100644 --- a/buildstream/_artifactcache/__init__.py +++ b/buildstream/_artifactcache/__init__.py @@ -18,4 +18,4 @@ # Authors: # Tristan Van Berkom <[email protected]> -from .artifactcache import ArtifactCache +from .artifactcache import ArtifactCache, artifact_cache_urls_from_config_node, configured_artifact_cache_urls diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index 1283b37..6eae24b 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -21,12 +21,72 @@ import os from collections import Mapping -from .._exceptions import ImplError +from .._exceptions import ImplError, LoadError, LoadErrorReason from .. import utils from .. import _yaml -# An ArtifactCache manages artifacts +def artifact_cache_url_from_spec(spec): + _yaml.node_validate(spec, ['url']) + url = _yaml.node_get(spec, str, 'url') + if len(url) == 0: + provenance = _yaml.node_get_provenance(spec) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: empty artifact cache URL".format(provenance)) + return url + + +# artifact_cache_urls_from_config_node() +# +# Parses the configuration of remote artifact caches from a config block. +# +# Args: +# config_node (dict): The config block, which may contain the 'artifacts' key +# +# Returns: +# A list of URLs pointing to remote artifact caches. +# +# Raises: +# LoadError, if the config block contains invalid keys. +# +def artifact_cache_urls_from_config_node(config_node): + urls = [] + + artifacts = config_node.get('artifacts', []) + if isinstance(artifacts, Mapping): + urls.append(artifact_cache_url_from_spec(artifacts)) + elif isinstance(artifacts, list): + for spec in artifacts: + urls.append(artifact_cache_url_from_spec(spec)) + else: + provenance = _yaml.node_get_provenance(config_node, key='artifacts') + raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, + "%s: 'artifacts' must be a single 'url:' mapping, or a list of mappings" % + (str(provenance))) + return urls + + +# configured_artifact_cache_urls(): +# +# Return the list of configured artifact remotes for a given project, in priority +# order. This takes into account the user and project configuration. +# +# Args: +# context (Context): The BuildStream context +# project (Project): The BuildStream project +# +# Returns: +# A list of URLs pointing to remote artifact caches. +# +def configured_artifact_cache_urls(context, project): + project_overrides = context._get_overrides(project.name) + project_extra_urls = artifact_cache_urls_from_config_node(project_overrides) + + return list(utils.deduplicate( + project_extra_urls + project.artifact_urls + context.artifact_urls)) + + +# An ArtifactCache manages artifacts. # # Args: # context (Context): The BuildStream context @@ -36,42 +96,24 @@ class ArtifactCache(): def __init__(self, context, project): self.context = context + self.project = project os.makedirs(context.artifactdir, exist_ok=True) self.extractdir = os.path.join(context.artifactdir, 'extract') self._local = False + self.urls = [] - project_overrides = context._get_overrides(project.name) - artifact_overrides = _yaml.node_get(project_overrides, Mapping, 'artifacts', default_value={}) - override_url = _yaml.node_get(artifact_overrides, str, 'url', default_value='') or None - - _yaml.node_validate(artifact_overrides, ['url']) - - if override_url: - self.url = override_url - elif project.artifact_url: - self.url = project.artifact_url - else: - self.url = context.artifact_url - - if self.url: - if self.url.startswith('/') or self.url.startswith('file://'): - self._local = True - - self.remote = utils.url_directory_name(self.url) - else: - self.remote = None - - self._offline = False - - # initialize_remote(): + # set_remotes(): # - # Initialize any remote artifact cache, if needed. This may require network - # access and could block for several seconds. + # Set the list of remote caches, which is initially empty. This will + # contact each remote cache. # - def initialize_remote(self): - pass + # Args: + # urls (list): List of artifact remote URLs, in priority order. + # on_failure (callable): Called if we fail to contact one of the caches. + def set_remotes(self, urls, on_failure=None): + self.urls = urls # contains(): # @@ -120,37 +162,28 @@ class ArtifactCache(): raise ImplError("Cache '{kind}' does not implement commit()" .format(kind=type(self).__name__)) - # set_offline() + # has_fetch_remotes(): # - # Do not attempt to pull or push artifacts. + # Check whether any remote repositories are available for fetching. # - def set_offline(self): - self._offline = True - - # can_fetch(): + # Returns: True if any remote repositories are configured, False otherwise # - # Check whether remote repository is available for fetching. - # - # Returns: True if remote repository is available, False otherwise - # - def can_fetch(self): - return (not self._offline or self._local) and \ - self.remote is not None + def has_fetch_remotes(self): + return (len(self.urls) > 0) - # can_push(): + # has_push_remotes(): # - # Check whether remote repository is available for pushing. + # Check whether any remote repositories are available for pushing. # - # Returns: True if remote repository is available, False otherwise + # Returns: True if any remote repository is configured, False otherwise # - def can_push(self): - return (not self._offline or self._local) and \ - self.url is not None + def has_push_remotes(self): + return (len(self.urls) > 0) # remote_contains_key(): # # Check whether the artifact for the specified Element is already available - # in the remote artifact cache. + # in any remote artifact cache. # # Args: # element (Element): The Element to check @@ -160,6 +193,3 @@ class ArtifactCache(): # def remote_contains(self, element, strength=None): return False - - def fetch_remote_refs(self): - pass diff --git a/buildstream/_artifactcache/ostreecache.py b/buildstream/_artifactcache/ostreecache.py index 78796dc..9987746 100644 --- a/buildstream/_artifactcache/ostreecache.py +++ b/buildstream/_artifactcache/ostreecache.py @@ -48,7 +48,7 @@ def buildref(element, key): return '{0}/{1}/{2}'.format(project.name, element_name, key) -# An ArtifactCache manages artifacts in an OSTree repository +# An OSTreeCache manages artifacts in an OSTree repository # # Args: # context (Context): The BuildStream context @@ -70,35 +70,19 @@ class OSTreeCache(ArtifactCache): self.repo = _ostree.ensure(ostreedir, False) self.push_url = None - self.pull_url = None - - self._remote_refs = None - - def initialize_remote(self): - if self.url is not None: - if self.url.startswith('ssh://'): - self.push_url = self.url - try: - # Contact the remote cache. - self.pull_url = initialize_push_connection(self.push_url) - except PushException as e: - raise ArtifactError("BuildStream did not connect succesfully " - "to the shared cache: {}".format(e)) - elif self.url.startswith('http://') or self.url.startswith('https://'): - self.push_url = None - self.pull_url = self.url - elif self._local: - self.push_url = self.url - self.pull_url = self.url - else: - raise ArtifactError("Unsupported URL scheme: {}".format(self.url)) + self.pull_urls = [] + self._remote_refs = {} + + def set_remotes(self, urls, on_failure=None): + self.urls = urls + + self._initialize_remotes(on_failure) - _ostree.configure_remote(self.repo, self.remote, self.pull_url) + def has_fetch_remotes(self): + return (len(self.pull_urls) > 0) - def can_push(self): - if self.enable_push: - return (not self._offline or self._local) and self.push_url is not None - return False + def has_push_remotes(self): + return (self.push_url is not None) # contains(): # @@ -134,7 +118,7 @@ class OSTreeCache(ArtifactCache): # Returns: True if the artifact is in the cache, False otherwise # def remote_contains_key(self, element, key): - if not self._remote_refs: + if len(self._remote_refs) == 0: return False ref = buildref(element, key) @@ -241,7 +225,7 @@ class OSTreeCache(ArtifactCache): # pull(): # - # Pull artifact from remote repository. + # Pull artifact from one of the configured remote repositories. # # Args: # element (Element): The Element whose artifact is to be fetched @@ -249,21 +233,13 @@ class OSTreeCache(ArtifactCache): # def pull(self, element, progress=None): - if self._offline and not self._local: - raise ArtifactError("Attempt to pull artifact while offline") - - if self.pull_url.startswith("/"): - remote = "file://" + self.pull_url - else: - remote = self.remote - + ref = buildref(element, element._get_cache_key()) weak_ref = buildref(element, element._get_cache_key(strength=_KeyStrength.WEAK)) try: - if self.remote_contains(element, strength=_KeyStrength.STRONG): + if ref in self._remote_refs: # fetch the artifact using the strong cache key - ref = buildref(element, element._get_cache_key()) - _ostree.fetch(self.repo, remote=remote, + _ostree.fetch(self.repo, remote=self._remote_refs[ref], ref=ref, progress=progress) # resolve ref to checksum @@ -271,9 +247,9 @@ class OSTreeCache(ArtifactCache): # update weak ref by pointing it to this newly fetched artifact _ostree.set_ref(self.repo, weak_ref, rev) - elif self.remote_contains(element): + elif weak_ref in self._remote_refs: # fetch the artifact using the weak cache key - _ostree.fetch(self.repo, remote=remote, + _ostree.fetch(self.repo, remote=self._remote_refs[weak_ref], ref=weak_ref, progress=progress) # resolve weak_ref to checksum @@ -292,35 +268,6 @@ class OSTreeCache(ArtifactCache): raise ArtifactError("Failed to pull artifact for element {}: {}" .format(element.name, e)) from e - # fetch_remote_refs(): - # - # Fetch list of artifacts from remote repository. - # - def fetch_remote_refs(self): - if self.pull_url.startswith("/"): - remote = "file://" + self.pull_url - elif self.remote is not None: - remote = self.remote - else: - raise ArtifactError("Attempt to fetch remote refs without any pull URL") - - def child_action(repo, remote, q): - try: - q.put((True, _ostree.list_remote_refs(self.repo, remote=remote))) - except OSTreeError as e: - q.put((False, e)) - - q = multiprocessing.Queue() - p = multiprocessing.Process(target=child_action, args=(self.repo, remote, q)) - p.start() - ret, res = q.get() - p.join() - - if ret: - self._remote_refs = res - else: - raise ArtifactError("Failed to fetch remote refs") from res - # push(): # # Push committed artifact to remote repository. @@ -336,17 +283,14 @@ class OSTreeCache(ArtifactCache): # (ArtifactError): if there was an error def push(self, element): - if self._offline and not self._local: - raise ArtifactError("Attempt to push artifact while offline") - if self.push_url is None: raise ArtifactError("The protocol in use does not support pushing.") ref = buildref(element, element._get_cache_key_from_artifact()) weak_ref = buildref(element, element._get_cache_key(strength=_KeyStrength.WEAK)) - if self.push_url.startswith("/"): + if self.push_url.startswith("file://"): # local repository - push_repo = _ostree.ensure(self.push_url, True) + push_repo = _ostree.ensure(self.push_url[7:], True) _ostree.fetch(push_repo, remote=self.repo.get_path().get_uri(), ref=ref) _ostree.fetch(push_repo, remote=self.repo.get_path().get_uri(), ref=weak_ref) @@ -377,9 +321,117 @@ class OSTreeCache(ArtifactCache): return pushed - # set_offline(): + # _initialize_remote(): + # + # Do protocol-specific initialization necessary to use a given OSTree + # remote. + # + # The SSH protocol that we use only supports pushing so initializing these + # involves contacting the remote to find out the corresponding pull URL. + # + # Args: + # url (str): URL of the remote + # + # Returns: + # (str, str): the pull URL and push URL for the remote + # + # Raises: + # ArtifactError: if there was an error + def _initialize_remote(self, url): + if url.startswith('ssh://'): + try: + push_url = url + pull_url = initialize_push_connection(url) + except PushException as e: + raise ArtifactError("BuildStream did not connect successfully " + "to the shared cache {}: {}".format(url, e)) + elif url.startswith('/'): + push_url = pull_url = 'file://' + url + elif url.startswith('file://'): + push_url = pull_url = url + elif url.startswith('http://') or url.startswith('https://'): + push_url = None + pull_url = url + else: + raise ArtifactError("Unsupported URL: {}".format(url)) + + return push_url, pull_url + + # _ensure_remote(): # - # Do not attempt to pull or push artifacts. + # Ensure that our OSTree repo has a remote configured for the given URL. + # Note that SSH access to remotes is not handled by libostree itself. # - def set_offline(self): - self._offline = True + # Args: + # repo (OSTree.Repo): an OSTree repository + # pull_url (str): the URL where libostree can pull from the remote + # + # Returns: + # (str): the name of the remote, which can be passed to various other + # operations implemented by the _ostree module. + # + # Raises: + # OSTreeError: if there was a problem reported by libostree + def _ensure_remote(self, repo, pull_url): + remote_name = utils.url_directory_name(pull_url) + _ostree.configure_remote(repo, remote_name, pull_url) + return remote_name + + def _initialize_remotes(self, on_failure=None): + self.push_url = None + self.pull_urls = [] + self._remote_refs = {} + + # Callback to initialize one remote in a 'multiprocessing' subprocess. + # + # We cannot do this in the main process because of the way the tasks + # run by the main scheduler calls into libostree using + # fork()-without-exec() subprocesses. OSTree fetch operations in + # subprocesses hang if fetch operations were previously done in the + # main process. + # + def child_action(url, q): + try: + push_url, pull_url = self._initialize_remote(url) + remote = self._ensure_remote(self.repo, pull_url) + remote_refs = _ostree.list_remote_refs(self.repo, remote=remote) + q.put((None, push_url, pull_url, remote_refs)) + except Exception as e: + # Exceptions aren't automatically propagated by + # multiprocessing, so we catch everything here. Note that + # GLib.Error subclasses can't be returned (as they don't + # 'pickle') and so they'll be ignored. + q.put((e, None, None, None)) + + # Kick off all the initialization jobs one by one. + # + # Note that we cannot use multiprocessing.Pool here because it's not + # possible to pickle local functions such as child_action(). + # + q = multiprocessing.Queue() + for url in self.urls: + p = multiprocessing.Process(target=child_action, args=(url, q)) + p.start() + exception, push_url, pull_url, remote_refs = q.get() + p.join() + + if exception and on_failure: + on_failure(url, exception) + elif exception: + raise ArtifactError() from exception + else: + # Use first pushable remote we find for pushing. + if push_url and not self.push_url: + self.push_url = push_url + + # Use all pullable remotes, in priority order + if pull_url and pull_url not in self.pull_urls: + self.pull_urls.append(pull_url) + + # Update our overall map of remote refs with any refs that are + # present in the new remote and were not already found in + # higher priority ones. + remote = self._ensure_remote(self.repo, pull_url) + for ref in remote_refs: + if ref not in self._remote_refs: + self._remote_refs[ref] = remote diff --git a/buildstream/_context.py b/buildstream/_context.py index 6d0a19b..78ec9f8 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -29,6 +29,7 @@ from . import _yaml from ._exceptions import LoadError, LoadErrorReason, BstError from ._message import Message, MessageType from ._profile import Topics, profile_start, profile_end +from ._artifactcache import artifact_cache_urls_from_config_node # Context() @@ -61,8 +62,8 @@ class Context(): # The local binary artifact cache directory self.artifactdir = None - # The URL from which to push and pull prebuilt artifacts - self.artifact_url = None + # The URLs from which to push and pull prebuilt artifacts + self.artifact_urls = [] # The directory to store build logs self.logdir = None @@ -161,9 +162,7 @@ class Context(): setattr(self, dir, path) # Load artifact share configuration - artifacts = _yaml.node_get(defaults, Mapping, 'artifacts') - _yaml.node_validate(artifacts, ['url']) - self.artifact_url = _yaml.node_get(artifacts, str, 'url', default_value='') or None + self.artifact_urls = artifact_cache_urls_from_config_node(defaults) # Load logging config logging = _yaml.node_get(defaults, Mapping, 'logging') diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py index c9f0389..1655563 100644 --- a/buildstream/_ostree.py +++ b/buildstream/_ostree.py @@ -357,4 +357,4 @@ def list_remote_refs(repo, remote="origin"): _, refs = repo.remote_list_refs(remote) return refs except GLib.GError as e: - raise OSTreeError("Failed to fetch remote refs from '{}': {}".format(remote, e.message)) from e + raise OSTreeError(message=e.message) from e diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py index 5afb42f..5f0c7e4 100644 --- a/buildstream/_pipeline.py +++ b/buildstream/_pipeline.py @@ -40,6 +40,7 @@ from . import Scope from . import _site from . import utils from ._platform import Platform +from ._artifactcache import configured_artifact_cache_urls from ._scheduler import SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue @@ -145,8 +146,8 @@ class Pipeline(): self.initialize_workspaces() - if use_remote_cache and self.artifacts.can_fetch(): - self.fetch_remote_refs() + if use_remote_cache: + self.initialize_remote_caches() self.resolve_cache_keys(inconsistent) @@ -165,14 +166,13 @@ class Pipeline(): self.project._set_workspace(element, source, workspace) - def fetch_remote_refs(self): - with self.timed_activity("Fetching remote refs", silent_nested=True): - try: - self.artifacts.initialize_remote() - self.artifacts.fetch_remote_refs() - except ArtifactError: - self.message(MessageType.WARN, "Failed to fetch remote refs") - self.artifacts.set_offline() + def initialize_remote_caches(self): + def remote_failed(url, error): + self.message(MessageType.WARN, "Failed to fetch remote refs from {}: {}\n".format(url, error)) + + with self.timed_activity("Initializing remote caches", silent_nested=True): + artifact_urls = configured_artifact_cache_urls(self.context, self.project) + self.artifacts.set_remotes(artifact_urls, on_failure=remote_failed) def resolve_cache_keys(self, inconsistent): if inconsistent: @@ -438,12 +438,12 @@ class Pipeline(): if track_plan: track = TrackQueue(save=save) queues.append(track) - if self.artifacts.can_fetch(): + if self.artifacts.has_fetch_remotes(): pull = PullQueue() queues.append(pull) queues.append(fetch) queues.append(build) - if self.artifacts.can_push(): + if self.artifacts.has_push_remotes(): push = PushQueue() queues.append(push) @@ -681,8 +681,8 @@ class Pipeline(): # def pull(self, scheduler, elements): - if not self.artifacts.can_fetch(): - raise PipelineError("Not configured for pulling artifacts") + if not self.artifacts.has_fetch_remotes(): + raise PipelineError("Not artifact caches available for pulling artifacts") plan = elements self.assert_consistent(plan) @@ -719,8 +719,8 @@ class Pipeline(): # def push(self, scheduler, elements): - if not self.artifacts.can_push(): - raise PipelineError("Not configured for pushing artifacts") + if not self.artifacts.has_push_remotes(): + raise PipelineError("No artifact caches available for pushing artifacts") plan = elements self.assert_consistent(plan) diff --git a/buildstream/_project.py b/buildstream/_project.py index 50f5d9c..35873fb 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -28,6 +28,7 @@ from . import _yaml from ._profile import Topics, profile_start, profile_end from ._exceptions import LoadError, LoadErrorReason from ._options import OptionPool +from ._artifactcache import artifact_cache_urls_from_config_node # The base BuildStream format version @@ -172,9 +173,7 @@ class Project(): # # Load artifacts pull/push configuration for this project - artifacts = _yaml.node_get(config, Mapping, 'artifacts', default_value={}) - _yaml.node_validate(artifacts, ['url']) - self.artifact_url = _yaml.node_get(artifacts, str, 'url', default_value='') or None + self.artifact_urls = artifact_cache_urls_from_config_node(config) # Workspace configurations self._workspaces = self._load_workspace_config() diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml index f43989d..edd74ef 100644 --- a/buildstream/data/userconfig.yaml +++ b/buildstream/data/userconfig.yaml @@ -48,14 +48,6 @@ scheduler: # on-error: quit -# -# Artifacts -# -artifacts: - - # A url from which to push and pull prebuilt artifacts. - # Some protocols only support pushing. - url: '' # # Logging diff --git a/buildstream/utils.py b/buildstream/utils.py index 2b6c451..8cde752 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -34,6 +34,7 @@ import stat import string import subprocess import tempfile +import itertools from contextlib import contextmanager import pkg_resources @@ -983,3 +984,32 @@ def _glob2re(pat): else: res = res + re.escape(c) return res + '\Z(?ms)' + + +# deduplicate() +# +# Remove duplicate entries in a list or other iterable. +# +# Copied verbatim from the unique_everseen() example at +# https://docs.python.org/3/library/itertools.html#itertools-recipes +# +# Args: +# iterable (iterable): What to deduplicate +# key (callable): Optional function to map from list entry to value +# +# Returns: +# (generator): Generator that produces a deduplicated version of 'iterable' +# +def deduplicate(iterable, key=None): + seen = set() + seen_add = seen.add + if key is None: + for element in itertools.filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 5f06e55..18a4b46 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -1,7 +1,7 @@ import os import shutil import pytest -from tests.testutils import cli, create_artifact_share +from tests.testutils import cli, create_artifact_share, configure_remote_caches from tests.testutils.site import IS_LINUX from buildstream import _yaml @@ -29,9 +29,9 @@ def assert_shared(cli, share, project, element_name): @pytest.mark.parametrize( 'override_url, project_url, user_url', [ - pytest.param('', '', 'share.repo', id='user-config'), - pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'), - pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'), + pytest.param(None, None, 'share.repo', id='user-config'), + pytest.param(None, 'share.repo', None, id='project-config'), + pytest.param('share.repo', None, None, id='project-override-in-user-config'), ]) @pytest.mark.datafiles(DATA_DIR) def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url): @@ -49,29 +49,8 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url): project_url = share.repo if project_url == 'share.repo' else project_url user_url = share.repo if user_url == 'share.repo' else user_url - # Configure artifact share - cli.configure({ - 'artifacts': { - 'url': user_url, - }, - 'projects': { - 'test': { - 'artifacts': { - 'url': override_url, - } - } - } - }) - - if project_url: - project_conf_file = str(datafiles.join('project.conf')) - project_config = _yaml.load(project_conf_file) - project_config.update({ - 'artifacts': { - 'url': project_url, - } - }) - _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file) + project_conf_file = str(datafiles.join('project.conf')) + configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url) # Now try bst push result = cli.run(project=project, args=['push', 'import-bin.bst']) diff --git a/tests/frontend/push.py b/tests/frontend/push.py index b5eddf8..9d897a8 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -1,6 +1,6 @@ import os import pytest -from tests.testutils import cli, create_artifact_share +from tests.testutils import cli, create_artifact_share, configure_remote_caches from tests.testutils.site import IS_LINUX from buildstream import _yaml @@ -28,9 +28,9 @@ def assert_shared(cli, share, project, element_name): @pytest.mark.parametrize( 'override_url, project_url, user_url', [ - pytest.param('', '', 'share.repo', id='user-config'), - pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'), - pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'), + pytest.param(None, None, 'share.repo', id='user-config'), + pytest.param(None, 'share.repo', None, id='project-config'), + pytest.param('share.repo', None, None, id='project-override-in-user-config'), ]) @pytest.mark.datafiles(DATA_DIR) def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url): @@ -48,29 +48,8 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url): project_url = share.repo if project_url == 'share.repo' else project_url user_url = share.repo if user_url == 'share.repo' else user_url - # Configure artifact share - cli.configure({ - 'artifacts': { - 'url': user_url, - }, - 'projects': { - 'test': { - 'artifacts': { - 'url': override_url, - } - } - } - }) - - if project_url: - project_conf_file = str(datafiles.join('project.conf')) - project_config = _yaml.load(project_conf_file) - project_config.update({ - 'artifacts': { - 'url': project_url, - } - }) - _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file) + project_conf_file = str(datafiles.join('project.conf')) + configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url) # Now try bst push result = cli.run(project=project, args=['push', 'target.bst']) diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py index 9fc450a..f0eb171 100644 --- a/tests/testutils/__init__.py +++ b/tests/testutils/__init__.py @@ -1,3 +1,3 @@ from .runcli import cli from .repo import create_repo, ALL_REPO_KINDS -from .artifactshare import create_artifact_share +from .artifactshare import create_artifact_share, configure_remote_caches diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index 907ed76..ebf38f3 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -3,6 +3,8 @@ import pytest import subprocess import os +from buildstream import _yaml + from .site import HAVE_OSTREE_CLI @@ -106,3 +108,35 @@ class ArtifactShare(): def create_artifact_share(directory): return ArtifactShare(directory) + + +# Write out cache configuration into the user config and project config files. +# +# User config is set through a helper on the 'cli' object, while the +# project.conf file is updated manually using the _yaml module. +# +def configure_remote_caches(cli, project_conf_file, override_url, project_url=None, user_url=None): + user_config = {} + if user_url is not None: + user_config['artifacts'] = { + 'url': user_url + } + + if override_url is not None: + user_config['projects'] = { + 'test': { + 'artifacts': { + 'url': override_url, + } + } + } + cli.configure(user_config) + + if project_url is not None: + project_config = _yaml.load(project_conf_file) + project_config.update({ + 'artifacts': { + 'url': project_url, + } + }) + _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
