cbomgit commented on issue #13090:
URL: https://github.com/apache/hudi/issues/13090#issuecomment-2779164719
Additional details:
I confirmed my approx record size by running a test with just a SIMPLE
non-bloom index and ingesting a weeks worth of data. 128 MB files pack around
1.2M records per file, yielding around 95 bytes per record:
```
val df =
spark.read.parquet("s3://path/to/table/region=NA/year=2025/month=03/day=18/*.parquet")
val countsDf = df.groupBy("_hoodie_file_name").agg(
countDistinct("_hoodie_record_key").as("count")
)
val percentiles = List(0.5, 0.75, 0.90, 0.95)
countsDf.agg(
avg("count").as("avg"),
min("count").as("min"),
max("count").as("max"),
percentile_approx($"count", typedLit(percentiles),
lit(10000)).as("percentiles")
).select(
$"avg",
$"min",
$"max",
$"percentiles".getItem(0).as("p50"),
$"percentiles".getItem(1).as("p75"),
$"percentiles".getItem(2).as("p90"),
$"percentiles".getItem(3).as("p95")
).show(false)
val recordCountInFile =
spark.read.parquet("s3://path/to/table/region=NA/year=2025/month=03/day=18/file_id.parquet").count
(121.0 * 1000 * 1000) / recordCountInFile
df: org.apache.spark.sql.DataFrame = [_hoodie_commit_time: string,
_hoodie_commit_seqno: string ... 32 more fields]
countsDf: org.apache.spark.sql.DataFrame = [_hoodie_file_name: string,
count: bigint]
percentiles: List[Double] = List(0.5, 0.75, 0.9, 0.95)
+------------------+------+-------+-------+-------+-------+-------+
|avg |min |max |p50 |p75 |p90 |p95 |
+------------------+------+-------+-------+-------+-------+-------+
|1263895.4036393713|224265|1333819|1266274|1267078|1267956|1268659|
+------------------+------+-------+-------+-------+-------+-------+
recordCountInFile: Long = 1265650 // records per file
res64: Double = 95.60304981629993 // record size
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
