This is an automated email from the ASF dual-hosted git repository. tvb pushed a commit to branch jennis/deps_for_push in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8e654015c2eccd62d538ae6d39e78524913775b2 Author: James Ennis <[email protected]> AuthorDate: Thu Aug 29 12:32:21 2019 +0100 cli.py: Allow pull to handle artifact refs This patch adds support for the handling of artifact refs in bst artifact pull. A test for this has also been added. --- src/buildstream/_frontend/cli.py | 10 +++++----- src/buildstream/_project.py | 5 ----- src/buildstream/_stream.py | 6 ++++-- tests/frontend/buildcheckout.py | 2 +- tests/frontend/pull.py | 42 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 40d73e3..80b7479 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1132,10 +1132,10 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, help='The dependency artifacts to pull (default: none)') @click.option('--remote', '-r', default=None, help="The URL of the remote cache (defaults to the first configured cache)") [email protected]('elements', nargs=-1, [email protected]('artifacts', nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def artifact_pull(app, elements, deps, remote): +def artifact_pull(app, artifacts, deps, remote): """Pull a built artifact from the configured remote artifact cache. Specifying no elements will result in pulling the default targets @@ -1159,12 +1159,12 @@ def artifact_pull(app, elements, deps, remote): with app.initialized(session_name="Pull"): ignore_junction_targets = False - if not elements: - elements = app.project.get_default_targets() + if not artifacts: + artifacts = app.project.get_default_targets() # Junction elements cannot be pulled, exclude them from default targets ignore_junction_targets = True - app.stream.pull(elements, selection=deps, remote=remote, + app.stream.pull(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets) diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 00beebf..7260d96 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -482,13 +482,8 @@ class Project(): # 2. The ArtifactCache.contains() method expects an Element # and a key, not a ref. # - artifactdir = self._context.artifactdir artifacts = [] for ref in targets: - if not os.path.exists(os.path.join(artifactdir, ref)): - raise LoadError("{}\nis not present in the artifact cache ({})".format(ref, artifactdir), - LoadErrorReason.MISSING_FILE) - artifacts.append(ArtifactElement._new_from_artifact_ref(ref, self._context, task)) ArtifactElement._clear_artifact_refs_cache() diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index bea43f8..9abbb88 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -416,7 +416,8 @@ class Stream(): selection=selection, ignore_junction_targets=ignore_junction_targets, use_artifact_config=use_config, - artifact_remote_url=remote) + artifact_remote_url=remote, + load_refs=True) if not self._artifacts.has_fetch_remotes(): raise StreamError("No artifact caches available for pulling artifacts") @@ -563,7 +564,8 @@ class Stream(): self._export_artifact(tar, location, compression, target, hardlinks, virdir) except AttributeError as e: raise ArtifactError("Artifact reference '{}' seems to be invalid. " - "Note that an Element name can also be used.".format(artifact))from e + "Note that an Element name can also be used." + .format(artifact._element.get_artifact_name()))from e else: try: with target._prepare_sandbox(scope=scope, directory=None, diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 98b179b..6281217 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -358,7 +358,7 @@ def test_build_checkout_invalid_ref(datafiles, cli): checkout_args = ['artifact', 'checkout', '--deps', 'none', '--tar', checkout, non_existent_artifact] result = cli.run(project=project, args=checkout_args) - assert "{}\nis not present in the artifact cache".format(non_existent_artifact) in result.stderr + assert "Artifact reference '{}' seems to be invalid".format(non_existent_artifact) in result.stderr @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index fd49ff1..f978258 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -558,3 +558,45 @@ def test_pull_access_rights(cli, tmpdir, datafiles): st = os.lstat(os.path.join(checkout, 'usr/share/big-file')) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0644 + + +# Tests `bst artifact pull $artifact_ref` [email protected](DATA_DIR) +def test_pull_artifact(cli, tmpdir, datafiles): + project = str(datafiles) + element = 'target.bst' + + # Configure a local cache + local_cache = os.path.join(str(tmpdir), 'cache') + cli.configure({'cachedir': local_cache}) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + + result = cli.run(project=project, args=['build', element]) + result.assert_success() + + # Assert that the *artifact* is cached locally + cache_key = cli.get_element_key(project, element) + artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key) + assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + + # Assert that the target is shared (note that assert shared will use the artifact name) + assert_shared(cli, share, project, element) + + # Now we've pushed, remove the local cache + shutil.rmtree(os.path.join(local_cache, 'artifacts')) + + # Assert that nothing is cached locally anymore + assert not os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + + # Now try bst artifact pull + result = cli.run(project=project, args=['artifact', 'pull', artifact_ref]) + result.assert_success() + + # And assert that it's again in the local cache, without having built + assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
