danny0405 commented on code in PR #20676:
URL: https://github.com/apache/hudi/pull/20676#discussion_r4624028197
##########
hudi-integ-test/spark-pipelines/keyed_flows.py:
##########
@@ -42,33 +42,48 @@ def client(mode, remote, root):
spark = SparkSession.builder.remote(remote).getOrCreate()
try:
require(spark.conf.get("spark.hudi.test.run.id") == root.name,
"Connected to the wrong test server")
+ table_type = (root / "table-type").read_text()
+ properties = f"'type'='{table_type}', {PROPERTIES}"
if mode == "prepare":
spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts
BIGINT, part STRING, value STRING, "
- f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY
(part) TBLPROPERTIES ({PROPERTIES})").collect()
+ f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY
(part) TBLPROPERTIES ({properties})").collect()
else:
expected = json.loads((root / "expected.json").read_text())
for table in TABLES:
actual = [list(row) for row in spark.table(f"sdp_test.{table}")
.select("id", "ts", "part",
"value").orderBy("id").collect()]
require(actual == expected, (table, actual, expected))
- print(f"PASS {mode} {table}: {actual}")
+ print(f"PASS {mode} {table_type} {table}: {actual}")
+ if table_type == "mor" and mode == "verify-2":
+ location = root / "warehouse" / "sdp_test.db" / table
+ require(any(location.rglob("*.log.*")), f"No MOR log files
in {location}")
+ base_rows = [list(row) for row in spark.read.format("hudi")
+ .option("hoodie.datasource.query.type",
"read_optimized")
+ .load(location.as_uri()).select("id", "ts",
"part", "value")
+ .orderBy("id").collect()]
+ require(base_rows != expected, "Update must require log
merging, not only base-file reads")
Review Comment:
Fixed in db87ba1474e. The scenario now saves the exact stage-1 expected rows
and requires the MOR read-optimized output to equal them, with the table and
actual/expected base rows in the failure message. The snapshot is still checked
against the exact stage-2 rows. Re-ran the full MOR matrix with
PYTHONOPTIMIZE=1: both new and existing targets retained the exact stage-1 base
rows while their snapshots contained the newer update.
##########
hudi-integ-test/spark-pipelines/keyed_flows.py:
##########
@@ -89,14 +104,22 @@ def scenario(h):
(h.root / "expected.json").write_text(json.dumps(expected))
h.run_pipeline(path, f"stage-{i}")
h.client(Path(__file__).resolve(), f"verify-{i}")
+ def completed_commits():
+ return {table: sorted(p.name for p in
+ (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" /
"timeline").glob("*")
+ if p.suffix in (".commit", ".deltacommit", ".replacecommit"))
for table in TABLES}
+
+ before_restart = completed_commits()
+ require(all(before_restart.values()), "Expected completed commits before
restart")
h.run_pipeline(path, "restart")
+ require(completed_commits() == before_restart, "Unchanged restart wrote an
extra commit")
Review Comment:
Fixed in db87ba1474e. Captured after_restart = completed_commits() and
included both before_restart and after_restart dictionaries in the require
message, so failures identify each table and its before/after commit filenames.
Re-ran the full COW and MOR CLI/Connect matrices with PYTHONOPTIMIZE=1; both
passed, including unchanged restart and all five rejected configurations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]