Awesome06 commented on issue #55272:
URL: https://github.com/apache/arrow/issues/55272#issuecomment-6721704491
@pitrou Yes. These are measured on disk using PyArrow 26.0 and DuckDB 2.0
(not AI-generated numbers).
The datasets were generated via DuckDB's official TPC-H/TPC-DS engines
(`dbgen`/`dsdgen`) at SF 1.0, written to physical parquet files via
`pq.write_table()` with `data_page_version="1.0"` and `"2.0"`, and measured via
`os.path.getsize()`.
<details>
<summary><b>Self-contained Python reproduction script</b> (requires:
<code>pip install pyarrow duckdb</code>)</summary>
```python
import os
import tempfile
import duckdb
import pyarrow.parquet as pq
def compare(table, name, codecs=("snappy", "zstd")):
results = []
with tempfile.TemporaryDirectory() as tmpdir:
for codec in codecs:
p_v1 = os.path.join(tmpdir, f"{name}_{codec}_v1.parquet")
p_v2 = os.path.join(tmpdir, f"{name}_{codec}_v2.parquet")
pq.write_table(table, p_v1, data_page_version="1.0",
compression=codec)
pq.write_table(table, p_v2, data_page_version="2.0",
compression=codec)
s1 = os.path.getsize(p_v1)
s2 = os.path.getsize(p_v2)
delta = s2 - s1
pct = (delta / s1) * 100.0
results.append((codec, s1, s2, delta, pct))
return results
def main():
con = duckdb.connect()
print("Generating TPC-H SF 1.0 (lineitem)...")
con.execute("CALL dbgen(sf=1.0)")
tbl_lineitem = con.sql("SELECT * FROM lineitem").to_arrow_table()
res_lineitem = compare(tbl_lineitem, "tpch_lineitem")
print("Generating SSB SF 1.0 (lineorder)...")
con.execute("""
CREATE TABLE lineorder AS
SELECT
l_orderkey AS lo_orderkey, l_linenumber AS lo_linenumber, o_custkey
AS lo_custkey,
l_partkey AS lo_partkey, l_suppkey AS lo_suppkey, o_orderdate AS
lo_orderdate,
o_orderpriority AS lo_orderpriority, o_shippriority AS
lo_shippriority,
l_quantity AS lo_quantity, l_extendedprice AS lo_extendedprice,
o_totalprice AS lo_ordtotalprice, l_discount AS lo_discount,
(l_extendedprice * (1 - l_discount))::INT64 AS lo_revenue,
(l_extendedprice * (1 - l_tax))::INT64 AS lo_supplycost,
l_tax AS lo_tax, l_commitdate AS lo_commitdate, l_shipmode AS
lo_shipmode
FROM lineitem JOIN orders ON l_orderkey = o_orderkey;
""")
tbl_lineorder = con.sql("SELECT * FROM lineorder").to_arrow_table()
res_lineorder = compare(tbl_lineorder, "ssb_lineorder")
print("Generating TPC-DS SF 1.0 (store_sales, catalog_sales)...")
con_ds = duckdb.connect()
con_ds.execute("CALL dsdgen(sf=1.0)")
tbl_store_sales = con_ds.sql("SELECT * FROM
store_sales").to_arrow_table()
tbl_catalog_sales = con_ds.sql("SELECT * FROM
catalog_sales").to_arrow_table()
res_store_sales = compare(tbl_store_sales, "tpcds_store_sales")
res_catalog_sales = compare(tbl_catalog_sales, "tpcds_catalog_sales")
print(f"\n{'Benchmark (SF 1.0)':<25} {'Codec':<8} {'V1 Bytes':<14} {'V2
Bytes':<14} {'Delta Bytes':<12} {'Delta (%)':<10}")
print("-" * 85)
for name, res_list in [
("TPC-H lineitem", res_lineitem),
("SSB lineorder", res_lineorder),
("TPC-DS store_sales", res_store_sales),
("TPC-DS catalog_sales", res_catalog_sales),
]:
for codec, s1, s2, delta, pct in res_list:
print(f"{name:<25} {codec:<8} {s1:<14,d} {s2:<14,d}
{delta:<+12,d} {pct:>+8.3f}%")
if __name__ == "__main__":
main()
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
