gabotechs commented on code in PR #87: URL: https://github.com/apache/datafusion-iceberg/pull/87#discussion_r4625689468
########## crates/test-tools/src/taxi.rs: ########## @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A synthetic table shaped like the NYC TLC Yellow Taxi trip records. +//! +//! The values are made up but reproducible. They're drawn from `rand_pcg`'s +//! `Pcg64Mcg`, seeded with the day, whose output `rand` documents as portable +//! across platforms. The formulas only add, multiply, divide and round, which +//! IEEE 754 specifies exactly, so they don't depend on the platform's math +//! library either. + +use std::sync::Arc; + +use datafusion::arrow::array::{ + Date32Array, Float64Array, Int32Array, Int64Array, RecordBatch, + TimestampMicrosecondArray, +}; +use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, TimeUnit}; +use datafusion::error::Result; +use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties}; +use iceberg::TableIdent; +use iceberg::spec::{NestedField, PrimitiveType, Schema, Transform, Type}; +use rand::distr::Distribution; +use rand::distr::weighted::WeightedIndex; +use rand::{RngExt, SeedableRng}; +use rand_pcg::Pcg64Mcg; + +use crate::error::df_err; +use crate::{FixtureCatalog, FixtureTable}; + +/// The number of days of trips, each of which is one partition and one file. +pub const TAXI_DAYS: usize = 7; +/// The number of trips picked up on each day. +pub const TAXI_TRIPS_PER_DAY: usize = 25_000; +/// The first pickup date, 2024-01-08, in days since the Unix epoch. +pub const TAXI_FIRST_DAY: i32 = 19_730; + +const MICROS_PER_SECOND: i64 = 1_000_000; +const SECONDS_PER_DAY: i64 = 86_400; + +/// Creates table `default.taxi` of trips, partitioned by `pickup_date`, with +/// one snapshot that adds one file per day, in date order. +/// +/// The table is the only one in its own [`FixtureCatalog`], and keeps its +/// files until dropped. Each file is a single row group with column chunk +/// statistics but without page indexes. +pub async fn create_taxi_table() -> Result<FixtureTable> { + let fixtures = FixtureCatalog::try_new().await?; + let ident = TableIdent::from_strs(["default", "taxi"]).map_err(df_err)?; + let mut table = fixtures + .table(ident, taxi_schema()) + .partition_by("pickup_date", Transform::Identity) + .create() + .await?; + let mut append = table.append().writer_properties( + WriterProperties::builder() + .set_max_row_group_row_count(Some(TAXI_TRIPS_PER_DAY)) + .set_statistics_enabled(EnabledStatistics::Chunk) + .set_offset_index_disabled(true) + .build(), + ); + for day in 0..TAXI_DAYS { + append = append.file(taxi_trips(day)?); + } + append.commit().await?; + Ok(table) +} + +/// The taxi table's schema: 13 optional columns with field ids 1 to 13, in +/// order. +pub fn taxi_schema() -> Schema { + let columns = [ + ("vendor_id", PrimitiveType::Int), + ("pickup_at", PrimitiveType::Timestamp), + ("dropoff_at", PrimitiveType::Timestamp), + ("passenger_count", PrimitiveType::Long), + ("trip_distance", PrimitiveType::Double), + ("pickup_location_id", PrimitiveType::Int), + ("dropoff_location_id", PrimitiveType::Int), + ("payment_type", PrimitiveType::Long), + ("fare_amount", PrimitiveType::Double), + ("tip_amount", PrimitiveType::Double), + ("tolls_amount", PrimitiveType::Double), + ("total_amount", PrimitiveType::Double), + ("pickup_date", PrimitiveType::Date), + ]; + Schema::builder() + .with_fields(columns.into_iter().zip(1..).map(|((name, ty), id)| { + NestedField::optional(id, name, Type::Primitive(ty)).into() + })) + .build() + .expect("the taxi schema is valid") +} + +/// How often each passenger count occurs, in percent. `None` is a missing +/// count. +const PASSENGER_COUNTS: [(Option<i64>, u32); 7] = [ + (None, 5), + (Some(1), 70), + (Some(2), 13), + (Some(3), 5), + (Some(4), 3), + (Some(5), 2), + (Some(0), 2), +]; + +/// How often each payment type occurs among trips with a passenger count, in +/// percent. Trips without one have payment type 0, as in the real records. +const PAYMENT_TYPES: [(i64, u32); 4] = [(1, 78), (2, 18), (3, 3), (4, 1)]; + +/// The trips picked up on the `day`th day, ordered by pickup time. +/// +/// Pickup times are distinct within a day, so ordering by them is total. +/// About 5% of trips have no passenger count; those have payment type 0, as +/// in the real records. +pub fn taxi_trips(day: usize) -> Result<RecordBatch> { Review Comment: If we start using this harness for testing the write path that indeed could be useful, I think we can do it then if we find the need, otherwise we might complicate this code a bit without the current tests benefitting from it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
