danny0405 commented on code in PR #20676:
URL: https://github.com/apache/hudi/pull/20676#discussion_r4624028197


##########
hudi-integ-test/spark-pipelines/keyed_flows.py:
##########
@@ -42,33 +42,48 @@ def client(mode, remote, root):
     spark = SparkSession.builder.remote(remote).getOrCreate()
     try:
         require(spark.conf.get("spark.hudi.test.run.id") == root.name, 
"Connected to the wrong test server")
+        table_type = (root / "table-type").read_text()
+        properties = f"'type'='{table_type}', {PROPERTIES}"
         if mode == "prepare":
             spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts 
BIGINT, part STRING, value STRING, "
-                      f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({PROPERTIES})").collect()
+                      f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({properties})").collect()
         else:
             expected = json.loads((root / "expected.json").read_text())
             for table in TABLES:
                 actual = [list(row) for row in spark.table(f"sdp_test.{table}")
                           .select("id", "ts", "part", 
"value").orderBy("id").collect()]
                 require(actual == expected, (table, actual, expected))
-                print(f"PASS {mode} {table}: {actual}")
+                print(f"PASS {mode} {table_type} {table}: {actual}")
+                if table_type == "mor" and mode == "verify-2":
+                    location = root / "warehouse" / "sdp_test.db" / table
+                    require(any(location.rglob("*.log.*")), f"No MOR log files 
in {location}")
+                    base_rows = [list(row) for row in spark.read.format("hudi")
+                                 .option("hoodie.datasource.query.type", 
"read_optimized")
+                                 .load(location.as_uri()).select("id", "ts", 
"part", "value")
+                                 .orderBy("id").collect()]
+                    require(base_rows != expected, "Update must require log 
merging, not only base-file reads")

Review Comment:
   Fixed in db87ba1474e. The scenario now saves the exact stage-1 expected rows 
and requires the MOR read-optimized output to equal them, with the table and 
actual/expected base rows in the failure message. The snapshot is still checked 
against the exact stage-2 rows. Re-ran the full MOR matrix with 
PYTHONOPTIMIZE=1: both new and existing targets retained the exact stage-1 base 
rows while their snapshots contained the newer update.



##########
hudi-integ-test/spark-pipelines/keyed_flows.py:
##########
@@ -89,14 +104,22 @@ def scenario(h):
         (h.root / "expected.json").write_text(json.dumps(expected))
         h.run_pipeline(path, f"stage-{i}")
         h.client(Path(__file__).resolve(), f"verify-{i}")
+    def completed_commits():
+        return {table: sorted(p.name for p in
+                (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / 
"timeline").glob("*")
+                if p.suffix in (".commit", ".deltacommit", ".replacecommit")) 
for table in TABLES}
+
+    before_restart = completed_commits()
+    require(all(before_restart.values()), "Expected completed commits before 
restart")
     h.run_pipeline(path, "restart")
+    require(completed_commits() == before_restart, "Unchanged restart wrote an 
extra commit")

Review Comment:
   Fixed in db87ba1474e. Captured after_restart = completed_commits() and 
included both before_restart and after_restart dictionaries in the require 
message, so failures identify each table and its before/after commit filenames. 
Re-ran the full COW and MOR CLI/Connect matrices with PYTHONOPTIMIZE=1; both 
passed, including unchanged restart and all five rejected configurations.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to