sophie-h commented on code in PR #62:
URL:
https://github.com/apache/buildstream-plugins/pull/62#discussion_r1528959924
##########
src/buildstream_plugins/sources/cargo.py:
##########
@@ -367,17 +644,31 @@ def track(self, *, previous_sources_dir):
for package in lock["package"]:
if "source" not in package:
continue
- new_ref += [{"name": package["name"], "version":
str(package["version"]), "sha": package.get("checksum")}]
+
+ ref = {}
+
+ if package["source"][:4] == "git+":
Review Comment:
Added a check
##########
src/buildstream_plugins/sources/cargo.py:
##########
@@ -294,6 +308,166 @@ def _get_mirror_file(self, sha=None):
return os.path.join(self._get_mirror_dir(), sha or self.sha)
+# Locks on repositories for write access
+REPO_LOCKS = {}
+
+
+# CrateGit()
+#
+# Use a SourceFetcher class to be the per crate helper.
+#
+# This one is for crates fetched from git repositories.
+#
+# Args:
+# cargo (Cargo): The main Source implementation
+# name (str): The name of the crate to depend on
+# version (str): The version of the crate to depend on
+# repo (str): Repository URL
+# commit (str): Sha of the git commit
+class CrateGit(SourceFetcher):
+ def __init__(self, cargo, name, version, repo, commit):
+ super().__init__()
+
+ self.cargo = cargo
+ self.name = name
+ self.version = str(version)
+ self.repo = repo
+ self.commit = commit
+ # TODO: Is this right?
+ self.mark_download_url(self.repo)
Review Comment:
Should I implement this kind of mirror code?
##########
src/buildstream_plugins/sources/cargo.py:
##########
@@ -294,6 +310,268 @@ def _get_mirror_file(self, sha=None):
return os.path.join(self._get_mirror_dir(), sha or self.sha)
+# Locks on repositories for write access
+REPO_LOCKS = {} # type: dict[str, threading.Lock]
+
+
+# CrateGit()
+#
+# Use a SourceFetcher class to be the per crate helper.
+#
+# This one is for crates fetched from git repositories.
+#
+# Args:
+# cargo (Cargo): The main Source implementation
+# name (str): The name of the crate to depend on
+# version (str): The version of the crate to depend on
+# repo (str): Repository URL
+# commit (str): Sha of the git commit
+class CrateGit(SourceFetcher):
+ def __init__(self, cargo, name, version, repo, commit):
+ super().__init__()
+
+ self.cargo = cargo
+ self.name = name
+ self.version = str(version)
+ self.repo = repo
+ self.commit = commit
+ # TODO: Is this right?
+ self.mark_download_url(self.repo)
+
+ ########################################################
+ # SourceFetcher API method implementations #
+ ########################################################
+
+ def fetch(self, alias_override=None, **kwargs):
+ lock = REPO_LOCKS.setdefault(self._get_mirror_dir(), threading.Lock())
+
+ with lock, self._mirror_repo() as repo,
self.cargo.timed_activity(f"Fetching from {self.repo}"):
+ # TODO: Auth not supported
+ client, path = dulwich.client.get_transport_and_path(self.repo)
+ remote_refs = client.fetch(
+ path,
+ repo,
+ determine_wants=lambda refs, depth=None:
[self.commit.encode()],
+ depth=1,
+ )
+
+ ########################################################
+ # Helper APIs for the Cargo Source to use #
+ ########################################################
+
+ def ref_node(self):
+ return {"kind": "git", "name": self.name, "version": self.version,
"repo": self.repo, "commit": self.commit}
+
+ # stage()
+ #
+ # A delegate method to do the work for a single git repo
+ # in Source.stage().
+ #
+ # Args:
+ # (directory): The vendor subdirectory to stage to
+ #
+ def stage(self, directory):
+ self.cargo.status(f"Checking out {self.commit}")
+
+ crate_target_dir = os.path.join(directory,
f"{self.name}-{self.version}")
+ tmp_dir = os.path.join(directory, f"{self.name}-{self.version}-tmp")
+
+ try:
+ os.mkdir(tmp_dir)
+ except FileExistsError:
+ raise SourceError(
+ f"Cargo doens't support vendoring for identical crate version
from different sources {self.name} {self.version}"
+ )
+
+ with Repo(self._get_mirror_dir(), bare=True) as mirror:
+ with Repo.init(tmp_dir) as dest:
+ dest.object_store.add_object(mirror[self.commit.encode()])
+ dest.refs[b"HEAD"] = self.commit.encode()
+ dest.update_shallow([self.commit.encode()], [])
+
+ with Repo(tmp_dir, object_store=mirror.object_store) as dest:
+ dest.reset_index()
+
+ # Workspace handling
+ #
+ # When new workspace features are added it is worth checking if
+ #
<https://github.com/flatpak/flatpak-builder-tools/blob/HEAD/cargo/flatpak-cargo-generator.py>
+ # has implemented them already. This implementation is inspired by the
mentioned source.
+
+ with open(os.path.join(tmp_dir, "Cargo.toml"), "rb") as f:
+ root_toml = tomllib.load(f)
+
+ crates = {}
+
+ if "workspace" in root_toml and "memers":
+ # Find wanted crate inside workspace
+ for member in root_toml["workspace"].get("members", []):
+ for crate_toml_path in glob.glob(os.path.join(tmp_dir, member,
"Cargo.toml")):
+ crate_path =
os.path.normpath(os.path.dirname(crate_toml_path))
+
+ with open(crate_toml_path, "rb") as f:
+ crate_toml = tomllib.load(f)
+ crates[crate_toml["package"]["name"]] = {
+ "config": crate_toml,
+ "path": crate_path,
+ }
+
+ crate = crates[self.name]
+ # Apply information inherited from workspace Cargo.toml
+ config_inherit_workspace(crate["config"], root_toml["workspace"])
+
+ with open(os.path.join(crates[self.name]["path"], "Cargo.toml"),
"bw") as f:
+ tomli_w.dump(crate["config"], f)
+
+ shutil.move(crate["path"], crate_target_dir)
+ else:
+ # No workspaces involed, just reploy complete dir as is
+ shutil.move(tmp_dir, crate_target_dir)
+
+ # Write .cargo-checksum.json required by cargo vendoring
+ with open(os.path.join(crate_target_dir, ".cargo-checksum.json"), "w")
as f:
+ json.dump({"files": {}, "package": None}, f)
+
+ shutil.rmtree(tmp_dir, ignore_errors=True)
+
+ # is_cached()
+ #
+ # Get whether we have a local cached version of the git commit
+ #
+ # Returns:
+ # (bool): Whether we are cached or not
+ #
+ def is_cached(self):
+ with Repo(self._get_mirror_dir(), bare=True) as repo:
+ return self.commit.encode() in repo
+
+ # is_resolved()
+ #
+ # Get whether the current git repo is resolved
+ #
+ # Returns:
+ # (bool): Always true since we always have a commit
+ #
+ def is_resolved(self):
+ return True
+
+ ########################################################
+ # Private helpers #
+ ########################################################
+
+ # _get_mirror_dir()
+ #
+ # Gets the local mirror directory for this upstream git repository
+ #
+ def _get_mirror_dir(self):
+ if self.repo.endswith(".git"):
+ norm_url = self.repo[:-4]
+ else:
+ norm_url = self.repo
+
+ return os.path.join(
+ self.cargo.get_mirror_directory(),
+ utils.url_directory_name(norm_url) + ".git",
+ )
+
+ # _get_url()
+ #
+ # Gets the git URL to download this crate from
+ #
+ # Args:
+ # alias (str|None): The URL alias to apply, if any
+ #
+ # Returns:
+ # (str): The URL for this crate
+ #
+ def _get_url(self, alias=None):
+ return self.cargo.translate_url(self.cargo.repo, alias_override=alias)
+
+ # _mirror_repo()
+ #
+ # Returns the mirror repo, initialized if it doesn not exist yet
+ #
+ # Returns:
+ # (Repo): The mirror repo crate
+ #
+ def _mirror_repo(self):
+ try:
+ return Repo.init_bare(self._get_mirror_dir(), mkdir=True)
+ except FileExistsError:
+ return Repo(self._get_mirror_dir(), bare=True)
+
+
+# config_inherit_workspace()
+#
+# Adds inherited values to config
+#
+# Args:
+# config (dict): Crate config
+# workspace_config (dict): Workspace config
+def config_inherit_workspace(config, workspace_config):
+ workspace_deps = workspace_config.get("dependencies")
+ if workspace_deps is not None:
+ dependencies = []
+ for key in ["dependencies", "dev-dependencies", "build-dependencies"]:
+ if key in config:
+ dependencies.append(config[key])
+
+ if "traget" in config:
+ for target in config["target"].values():
+ if "dependencies" in target:
+ dependencies.append(target["dependencies"])
+
+ for deps in dependencies:
+ inherit_deps(deps, workspace_deps)
+
+ workspace_package = workspace_config.get("package")
+ if workspace_package is not None:
+ inherit_package(config["package"], workspace_package)
+
+
+# inherit_package()
+#
+# Adds inherited [package] entries to config
+#
+# Args:
+# items (dict): Package items
+# workspace_items (dict): Workspace items
+def inherit_package(items, workspace_items):
+ for key, value in items.items():
+ if isinstance(value, dict) and "workspace" in value:
+ workspace_value = workspace_items.get(key)
+ if workspace_value is None:
+ raise SourceError("Can't inherit package information from
workspace: Value missing from workspace")
Review Comment:
This message is more for debugging the plugin since the crate wouldn't have
originally been able to build with this. But I tried to clarify it.
##########
src/buildstream_plugins/sources/cargo.py:
##########
@@ -294,6 +310,268 @@ def _get_mirror_file(self, sha=None):
return os.path.join(self._get_mirror_dir(), sha or self.sha)
+# Locks on repositories for write access
+REPO_LOCKS = {} # type: dict[str, threading.Lock]
+
+
+# CrateGit()
+#
+# Use a SourceFetcher class to be the per crate helper.
+#
+# This one is for crates fetched from git repositories.
+#
+# Args:
+# cargo (Cargo): The main Source implementation
+# name (str): The name of the crate to depend on
+# version (str): The version of the crate to depend on
+# repo (str): Repository URL
+# commit (str): Sha of the git commit
+class CrateGit(SourceFetcher):
+ def __init__(self, cargo, name, version, repo, commit):
+ super().__init__()
+
+ self.cargo = cargo
+ self.name = name
+ self.version = str(version)
+ self.repo = repo
+ self.commit = commit
+ # TODO: Is this right?
+ self.mark_download_url(self.repo)
+
+ ########################################################
+ # SourceFetcher API method implementations #
+ ########################################################
+
+ def fetch(self, alias_override=None, **kwargs):
+ lock = REPO_LOCKS.setdefault(self._get_mirror_dir(), threading.Lock())
+
+ with lock, self._mirror_repo() as repo,
self.cargo.timed_activity(f"Fetching from {self.repo}"):
+ # TODO: Auth not supported
+ client, path = dulwich.client.get_transport_and_path(self.repo)
+ remote_refs = client.fetch(
+ path,
+ repo,
+ determine_wants=lambda refs, depth=None:
[self.commit.encode()],
+ depth=1,
+ )
+
+ ########################################################
+ # Helper APIs for the Cargo Source to use #
+ ########################################################
+
+ def ref_node(self):
+ return {"kind": "git", "name": self.name, "version": self.version,
"repo": self.repo, "commit": self.commit}
+
+ # stage()
+ #
+ # A delegate method to do the work for a single git repo
+ # in Source.stage().
+ #
+ # Args:
+ # (directory): The vendor subdirectory to stage to
+ #
+ def stage(self, directory):
+ self.cargo.status(f"Checking out {self.commit}")
+
+ crate_target_dir = os.path.join(directory,
f"{self.name}-{self.version}")
+ tmp_dir = os.path.join(directory, f"{self.name}-{self.version}-tmp")
+
+ try:
+ os.mkdir(tmp_dir)
+ except FileExistsError:
+ raise SourceError(
+ f"Cargo doens't support vendoring for identical crate version
from different sources {self.name} {self.version}"
+ )
+
+ with Repo(self._get_mirror_dir(), bare=True) as mirror:
+ with Repo.init(tmp_dir) as dest:
+ dest.object_store.add_object(mirror[self.commit.encode()])
+ dest.refs[b"HEAD"] = self.commit.encode()
+ dest.update_shallow([self.commit.encode()], [])
+
+ with Repo(tmp_dir, object_store=mirror.object_store) as dest:
+ dest.reset_index()
+
+ # Workspace handling
+ #
+ # When new workspace features are added it is worth checking if
+ #
<https://github.com/flatpak/flatpak-builder-tools/blob/HEAD/cargo/flatpak-cargo-generator.py>
+ # has implemented them already. This implementation is inspired by the
mentioned source.
+
+ with open(os.path.join(tmp_dir, "Cargo.toml"), "rb") as f:
+ root_toml = tomllib.load(f)
+
+ crates = {}
+
+ if "workspace" in root_toml and "memers":
+ # Find wanted crate inside workspace
+ for member in root_toml["workspace"].get("members", []):
+ for crate_toml_path in glob.glob(os.path.join(tmp_dir, member,
"Cargo.toml")):
+ crate_path =
os.path.normpath(os.path.dirname(crate_toml_path))
+
+ with open(crate_toml_path, "rb") as f:
+ crate_toml = tomllib.load(f)
+ crates[crate_toml["package"]["name"]] = {
+ "config": crate_toml,
+ "path": crate_path,
+ }
+
+ crate = crates[self.name]
+ # Apply information inherited from workspace Cargo.toml
+ config_inherit_workspace(crate["config"], root_toml["workspace"])
+
+ with open(os.path.join(crates[self.name]["path"], "Cargo.toml"),
"bw") as f:
+ tomli_w.dump(crate["config"], f)
+
+ shutil.move(crate["path"], crate_target_dir)
+ else:
+ # No workspaces involed, just reploy complete dir as is
+ shutil.move(tmp_dir, crate_target_dir)
+
+ # Write .cargo-checksum.json required by cargo vendoring
+ with open(os.path.join(crate_target_dir, ".cargo-checksum.json"), "w")
as f:
+ json.dump({"files": {}, "package": None}, f)
+
+ shutil.rmtree(tmp_dir, ignore_errors=True)
+
+ # is_cached()
+ #
+ # Get whether we have a local cached version of the git commit
+ #
+ # Returns:
+ # (bool): Whether we are cached or not
+ #
+ def is_cached(self):
+ with Repo(self._get_mirror_dir(), bare=True) as repo:
+ return self.commit.encode() in repo
+
+ # is_resolved()
+ #
+ # Get whether the current git repo is resolved
+ #
+ # Returns:
+ # (bool): Always true since we always have a commit
+ #
+ def is_resolved(self):
+ return True
+
+ ########################################################
+ # Private helpers #
+ ########################################################
+
+ # _get_mirror_dir()
+ #
+ # Gets the local mirror directory for this upstream git repository
+ #
+ def _get_mirror_dir(self):
+ if self.repo.endswith(".git"):
+ norm_url = self.repo[:-4]
+ else:
+ norm_url = self.repo
+
+ return os.path.join(
+ self.cargo.get_mirror_directory(),
+ utils.url_directory_name(norm_url) + ".git",
+ )
+
+ # _get_url()
+ #
+ # Gets the git URL to download this crate from
+ #
+ # Args:
+ # alias (str|None): The URL alias to apply, if any
+ #
+ # Returns:
+ # (str): The URL for this crate
+ #
+ def _get_url(self, alias=None):
+ return self.cargo.translate_url(self.cargo.repo, alias_override=alias)
+
+ # _mirror_repo()
+ #
+ # Returns the mirror repo, initialized if it doesn not exist yet
+ #
+ # Returns:
+ # (Repo): The mirror repo crate
+ #
+ def _mirror_repo(self):
+ try:
+ return Repo.init_bare(self._get_mirror_dir(), mkdir=True)
+ except FileExistsError:
+ return Repo(self._get_mirror_dir(), bare=True)
+
+
+# config_inherit_workspace()
+#
+# Adds inherited values to config
+#
+# Args:
+# config (dict): Crate config
+# workspace_config (dict): Workspace config
+def config_inherit_workspace(config, workspace_config):
+ workspace_deps = workspace_config.get("dependencies")
+ if workspace_deps is not None:
+ dependencies = []
+ for key in ["dependencies", "dev-dependencies", "build-dependencies"]:
+ if key in config:
+ dependencies.append(config[key])
+
+ if "traget" in config:
+ for target in config["target"].values():
+ if "dependencies" in target:
+ dependencies.append(target["dependencies"])
+
+ for deps in dependencies:
+ inherit_deps(deps, workspace_deps)
+
+ workspace_package = workspace_config.get("package")
+ if workspace_package is not None:
+ inherit_package(config["package"], workspace_package)
+
+
+# inherit_package()
+#
+# Adds inherited [package] entries to config
Review Comment:
I added a bunch of text plus links to the docs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
