Copilot commented on code in PR #13456:
URL: https://github.com/apache/gluten/pull/13456#discussion_r4513605121
##########
gluten-ut/src-iceberg-upstream/test/python/test_baseline.py:
##########
@@ -397,5 +622,111 @@ def
test_download_redirect_does_not_forward_github_token_to_artifact_storage(sel
self.assertIsNone(redirected.get_header("Authorization"))
+class InputFingerprintTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name)
+ self.git = ["git", "-C", str(self.root)]
+ subprocess.run(self.git + ["init", "-q"], check=True)
+ self.original = Path.cwd()
+ os.chdir(self.root)
+ self.addCleanup(os.chdir, self.original)
+
+ def tree(self):
+ subprocess.run(self.git + ["add", "."], check=True)
+ return subprocess.check_output(self.git + ["write-tree"],
text=True).strip()
+
+ def write(self, name, data):
+ path = self.root / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(data)
+
+ def test_only_known_non_runtime_changes_preserve_the_fingerprint(self):
+ self.write("pom.xml", "production dependency versions")
+ first = self.tree()
+ fingerprint = baseline.git_inputs(first)
+ for name in (
+ "docs/developers/Guide.md",
+ "README.md",
+ "AGENTS.md",
+ ".github/ISSUE_TEMPLATE/bug.yml",
+ ".devcontainer/devcontainer.json",
+ ".github/workflows/unrelated.yml",
+ ):
+ self.write(name, "unrelated change")
+ with self.subTest(name=name):
+ self.assertNotEqual(first, self.tree())
+ self.assertEqual(fingerprint, baseline.git_inputs(self.tree()))
+ for name in (
+ "pom.xml",
+ "backends-velox/pom.xml",
+ "gluten-core/src/main/Runtime.scala",
+ "gluten-ui/src/main/Listener.scala",
+ "gluten-iceberg/src/main/Scan.java",
+ "gluten-substrait/src/main/plan.proto",
+ "gluten-arrow/pom.xml",
+ "backends-velox/src/test/resources/schema.md",
+ "shims/spark35/pom.xml",
+ "cpp/velox/Scan.cc",
+ "ep/build-velox/src/velox.patch",
+ ".gitmodules",
+ "dev/vcpkg/vcpkg.json",
+ "dev/builddeps-veloxbe.sh",
+ "build/mvn",
+ ".mvn/maven.config",
+ "new-module/src/main/Dependency.java",
+ ".github/workflows/util/build.sh",
+ ".github/workflows/velox_native_build.yml",
+ ".github/workflows/velox_backend_x86.yml",
+ "backends-velox/src-iceberg-upstream/test/java/Observer.java",
+ ):
+ self.write(name, "runtime change")
+ changed = baseline.git_inputs(self.tree())
+ with self.subTest(name=name):
+ self.assertNotEqual(fingerprint, changed)
+ fingerprint = changed
+
+ def test_renames_deletions_modes_symlinks_and_submodules_are_inputs(self):
+ self.write("build/mvn", "build script")
+ previous = baseline.git_inputs(self.tree())
+ (self.root / "build/mvn").chmod(0o755)
+ changed = baseline.git_inputs(self.tree())
+ self.assertNotEqual(previous, changed)
+ previous = changed
+ (self.root / "build/mvn").rename(self.root / "build/renamed")
+ changed = baseline.git_inputs(self.tree())
+ self.assertNotEqual(previous, changed)
+ previous = changed
+ (self.root / "build/renamed").unlink()
+ self.assertNotEqual(previous, baseline.git_inputs(self.tree()))
+ for kind, mode in (("blob", "120000"), ("commit", "160000")):
+ entry = {"path": "dependency", "mode": mode, "type": kind, "sha":
"a" * 40}
+ previous = baseline.input_fingerprint([entry])
+ entry["sha"] = "b" * 40
+ self.assertNotEqual(previous, baseline.input_fingerprint([entry]))
+
+ def test_local_and_api_trees_use_the_same_fingerprint(self):
+ self.write("build/mvn", "script")
+ (self.root / "linked-input").symlink_to("build/mvn")
+ self.write("docs/ignored.md", "doc")
+ tree = self.tree()
+ entries = []
+ for record in subprocess.check_output(
+ self.git + ["ls-tree", "-rz", tree]
+ ).split(b"\0"):
+ if record:
+ metadata, path = record.decode().split("\t", 1)
+ mode, kind, sha = metadata.split()
+ entries.append(dict(path=path, mode=mode, type=kind, sha=sha))
+ # GitHub's recursive response also contains intermediate trees.
+ entries.append(dict(path="build", mode="040000", type="tree", sha="c"
* 40))
Review Comment:
This test constructs entries using `git ls-tree` without `-r`, which mirrors
the same non-recursive behavior currently in `baseline.git_inputs()`. Once
`git_inputs()` is corrected to be recursive, this test should also use a
recursive tree listing so it continues to validate that local-tree and
GitHub-API-tree fingerprints match over the same file set.
##########
.github/workflows/util/iceberg-upstream/baseline.py:
##########
@@ -46,11 +46,65 @@
".github/workflows/iceberg_spark_ut.yml",
".github/workflows/iceberg_upstream_revision.yml",
".github/workflows/velox_native_build.yml",
+ ".github/workflows/velox_backend_x86.yml",
".github/workflows/util/iceberg-upstream",
"backends-velox/pom.xml",
"backends-velox/src-iceberg-upstream",
"gluten-ut/src-iceberg-upstream",
)
+# Keep every tracked production, dependency, build and test input, including
+# new/unknown paths. Only these known non-runtime inputs are excluded. In
+# particular, do not exclude *.md/*.rst globally: they can be test resources.
+NON_RUNTIME_PATHS = (
+ "docs/",
+ ".devcontainer/",
+ ".idea/",
+ ".github/ISSUE_TEMPLATE/",
+ ".github/PULL_REQUEST_TEMPLATE.md",
+ ".github/skills/",
+ ".asf.yaml",
+ "AGENTS.md",
+ "CONTRIBUTING.md",
+ "README.md",
+ "mkdocs.yml",
+)
+
+
+def input_fingerprint(entries):
+ """Hash Git paths, modes, types and object IDs; include gitlinks and
symlinks."""
+ digest = hashlib.sha256()
+ for entry in sorted(entries, key=lambda item: item["path"]):
+ path = entry["path"]
+ if entry["type"] == "tree" or any(
+ path.startswith(name) if name.endswith("/") else path == name
+ for name in NON_RUNTIME_PATHS
+ ):
+ continue
+ # Other workflows do not execute in this comparison. Their supporting
+ # scripts remain inputs, as do all invoked workflow definitions.
+ if (
+ path.startswith(".github/workflows/")
+ and path.endswith((".yml", ".yaml"))
+ and path not in HARNESS_PATHS
+ and "/" not in path[len(".github/workflows/") :]
+ ):
+ continue
+ for field in ("path", "mode", "type", "sha"):
+ digest.update(entry[field].encode() + b"\0")
+ return digest.hexdigest()
+
+
+def git_inputs(revision):
+ entries = []
+ tree = git_tree(revision)
+ listing = subprocess.check_output(["git", "ls-tree", "-rz", tree])
+ for record in listing.split(b"\0"):
+ if not record:
+ continue
+ metadata, path = record.decode().split("\t", 1)
+ mode, kind, sha = metadata.split()
+ entries.append({"path": path, "mode": mode, "type": kind, "sha": sha})
+ return input_fingerprint(entries)
Review Comment:
`git_inputs()` currently calls `git ls-tree` without `-r`, so it only lists
the repository root entries (and many of those are `type == 'tree'` which
`input_fingerprint()` skips). This makes the computed inputs fingerprint ignore
most tracked files under subdirectories, which can incorrectly treat
runtime-impacting changes as compatible. Use a recursive listing (e.g., add
`-r` to `git ls-tree`) so blobs/submodules across the full tree contribute to
the fingerprint.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]