Dandandan commented on issue #20324:
URL: https://github.com/apache/datafusion/issues/20324#issuecomment-3941465761
I have another approach here, which solves the issue of TPC-H by integrating
it with the fileopener.
It looks like my intuition was right, the improvements are biggest on
clickbench_partitioned + filter pushdown (compared to main with enabled filter
pushdown).
```
--------------------
Benchmark clickbench_partitioned.json
--------------------
┏━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Query ┃ main ┃
parquet-morsel-driven-execution-237164415184908839 ┃ Change ┃
┡━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ QQuery 0 │ 1.04 ms │ 1.00
ms │ no change │
│ QQuery 1 │ 14.11 ms │ 16.10
ms │ 1.14x slower │
│ QQuery 2 │ 52.48 ms │ 46.45
ms │ +1.13x faster │
│ QQuery 3 │ 50.95 ms │ 46.23
ms │ +1.10x faster │
│ QQuery 4 │ 388.88 ms │ 364.90
ms │ +1.07x faster │
│ QQuery 5 │ 547.58 ms │ 447.44
ms │ +1.22x faster │
│ QQuery 6 │ 3.12 ms │ 6.26
ms │ 2.01x slower │
│ QQuery 7 │ 15.97 ms │ 17.84
ms │ 1.12x slower │
│ QQuery 8 │ 522.32 ms │ 470.94
ms │ +1.11x faster │
│ QQuery 9 │ 686.47 ms │ 622.81
ms │ +1.10x faster │
│ QQuery 10 │ 119.62 ms │ 108.15
ms │ +1.11x faster │
│ QQuery 11 │ 137.10 ms │ 122.23
ms │ +1.12x faster │
│ QQuery 12 │ 514.64 ms │ 421.11
ms │ +1.22x faster │
│ QQuery 13 │ 699.62 ms │ 601.43
ms │ +1.16x faster │
│ QQuery 14 │ 487.00 ms │ 430.26
ms │ +1.13x faster │
│ QQuery 15 │ 457.71 ms │ 435.63
ms │ no change │
│ QQuery 16 │ 1050.45 ms │ 978.28
ms │ +1.07x faster │
│ QQuery 17 │ 1010.59 ms │ 959.54
ms │ +1.05x faster │
│ QQuery 18 │ 2154.63 ms │ 2224.28
ms │ no change │
│ QQuery 19 │ 42.00 ms │ 37.71
ms │ +1.11x faster │
│ QQuery 20 │ 974.61 ms │ 733.71
ms │ +1.33x faster │
│ QQuery 21 │ 1022.29 ms │ 848.43
ms │ +1.20x faster │
│ QQuery 22 │ 1629.23 ms │ 1486.54
ms │ +1.10x faster │
│ QQuery 23 │ 5254.51 ms │ 5107.54
ms │ no change │
│ QQuery 24 │ 92.77 ms │ 52.92
ms │ +1.75x faster │
│ QQuery 25 │ 209.27 ms │ 164.90
ms │ +1.27x faster │
│ QQuery 26 │ 94.20 ms │ 53.32
ms │ +1.77x faster │
│ QQuery 27 │ 1172.79 ms │ 965.89
ms │ +1.21x faster │
│ QQuery 28 │ 10360.30 ms │ 9105.67
ms │ +1.14x faster │
│ QQuery 29 │ 366.58 ms │ 313.69
ms │ +1.17x faster │
│ QQuery 30 │ 476.59 ms │ 417.03
ms │ +1.14x faster │
│ QQuery 31 │ 483.61 ms │ 411.29
ms │ +1.18x faster │
│ QQuery 32 │ 2403.66 ms │ 2390.82
ms │ no change │
│ QQuery 33 │ 2234.37 ms │ 2245.70
ms │ no change │
│ QQuery 34 │ 2431.62 ms │ 2603.20
ms │ 1.07x slower │
│ QQuery 35 │ 750.36 ms │ 695.45
ms │ +1.08x faster │
│ QQuery 36 │ 107.58 ms │ 60.84
ms │ +1.77x faster │
│ QQuery 37 │ 43.81 ms │ 33.13
ms │ +1.32x faster │
│ QQuery 38 │ 73.31 ms │ 39.49
ms │ +1.86x faster │
│ QQuery 39 │ 196.98 ms │ 112.23
ms │ +1.76x faster │
│ QQuery 40 │ 19.56 ms │ 11.80
ms │ +1.66x faster │
│ QQuery 41 │ 16.27 ms │ 11.99
ms │ +1.36x faster │
│ QQuery 42 │ 14.26 ms │ 9.85
ms │ +1.45x faster │
└───────────┴─────────────┴────────────────────────────────────────────────────┴───────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
┃ Benchmark Summary ┃
┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
│ Total Time (main) │
39384.81ms │
│ Total Time (parquet-morsel-driven-execution-237164415184908839) │
36233.99ms │
│ Average Time (main) │
915.93ms │
│ Average Time (parquet-morsel-driven-execution-237164415184908839) │
842.65ms │
│ Queries Faster │
33 │
│ Queries Slower │
4 │
│ Queries with No Change │
6 │
│ Queries with Failure │
0 │
└───────────────────────────────────────────────────────────────────┴────────────┘
--------------------
Benchmark clickbench_pushdown.json
--------------------
┏━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Query ┃ main ┃
parquet-morsel-driven-execution-237164415184908839 ┃ Change ┃
┡━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ QQuery 0 │ 1.22 ms │ 0.99
ms │ +1.23x faster │
│ QQuery 1 │ 16.97 ms │ 19.87
ms │ 1.17x slower │
│ QQuery 2 │ 56.67 ms │ 50.67
ms │ +1.12x faster │
│ QQuery 3 │ 54.60 ms │ 48.81
ms │ +1.12x faster │
│ QQuery 4 │ 438.98 ms │ 375.83
ms │ +1.17x faster │
│ QQuery 5 │ 590.83 ms │ 470.20
ms │ +1.26x faster │
│ QQuery 6 │ 6.06 ms │ 6.56
ms │ 1.08x slower │
│ QQuery 7 │ 22.96 ms │ 22.14
ms │ no change │
│ QQuery 8 │ 567.93 ms │ 488.96
ms │ +1.16x faster │
│ QQuery 9 │ 779.84 ms │ 655.94
ms │ +1.19x faster │
│ QQuery 10 │ 177.41 ms │ 146.74
ms │ +1.21x faster │
│ QQuery 11 │ 199.19 ms │ 160.92
ms │ +1.24x faster │
│ QQuery 12 │ 601.62 ms │ 474.74
ms │ +1.27x faster │
│ QQuery 13 │ 810.95 ms │ 669.54
ms │ +1.21x faster │
│ QQuery 14 │ 571.92 ms │ 472.43
ms │ +1.21x faster │
│ QQuery 15 │ 512.93 ms │ 431.51
ms │ +1.19x faster │
│ QQuery 16 │ 1154.54 ms │ 996.90
ms │ +1.16x faster │
│ QQuery 17 │ 1129.89 ms │ 969.80
ms │ +1.17x faster │
│ QQuery 18 │ 2749.89 ms │ 2139.90
ms │ +1.29x faster │
│ QQuery 19 │ 71.38 ms │ 40.51
ms │ +1.76x faster │
│ QQuery 20 │ 1039.04 ms │ 743.52
ms │ +1.40x faster │
│ QQuery 21 │ 1095.05 ms │ 843.70
ms │ +1.30x faster │
│ QQuery 22 │ 1963.60 ms │ 1419.34
ms │ +1.38x faster │
│ QQuery 23 │ 541.91 ms │ 168.55
ms │ +3.22x faster │
│ QQuery 24 │ 154.14 ms │ 52.50
ms │ +2.94x faster │
│ QQuery 25 │ 294.96 ms │ 217.76
ms │ +1.35x faster │
│ QQuery 26 │ 178.71 ms │ 72.34
ms │ +2.47x faster │
│ QQuery 27 │ 1472.81 ms │ 1007.74
ms │ +1.46x faster │
│ QQuery 28 │ 12007.56 ms │ 9154.18
ms │ +1.31x faster │
│ QQuery 29 │ 383.20 ms │ 316.07
ms │ +1.21x faster │
│ QQuery 30 │ 529.90 ms │ 425.70
ms │ +1.24x faster │
│ QQuery 31 │ 521.25 ms │ 422.83
ms │ +1.23x faster │
│ QQuery 32 │ 3094.35 ms │ 2331.41
ms │ +1.33x faster │
│ QQuery 33 │ 3140.57 ms │ 2439.02
ms │ +1.29x faster │
│ QQuery 34 │ 4863.90 ms │ 3157.70
ms │ +1.54x faster │
│ QQuery 35 │ 969.41 ms │ 718.54
ms │ +1.35x faster │
│ QQuery 36 │ 96.56 ms │ 60.88
ms │ +1.59x faster │
│ QQuery 37 │ 58.77 ms │ 38.08
ms │ +1.54x faster │
│ QQuery 38 │ 56.74 ms │ 33.54
ms │ +1.69x faster │
│ QQuery 39 │ 178.06 ms │ 113.26
ms │ +1.57x faster │
│ QQuery 40 │ 31.91 ms │ 19.46
ms │ +1.64x faster │
│ QQuery 41 │ 28.49 ms │ 18.32
ms │ +1.55x faster │
│ QQuery 42 │ 20.52 ms │ 12.38
ms │ +1.66x faster │
└───────────┴─────────────┴────────────────────────────────────────────────────┴───────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
┃ Benchmark Summary ┃
┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
│ Total Time (main) │
43237.21ms │
│ Total Time (parquet-morsel-driven-execution-237164415184908839) │
32429.76ms │
│ Average Time (main) │
1005.52ms │
│ Average Time (parquet-morsel-driven-execution-237164415184908839) │
754.18ms │
│ Queries Faster │
40 │
│ Queries Slower │
2 │
│ Queries with No Change │
1 │
│ Queries with Failure │
0 │
└───────────────────────────────────────────────────────────────────┴────────────┘
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
