This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20680-948706b059e41b24e3918e6362dba38f788dd0f8 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 6a0c435db649fce65348958f535f1039a9d35c10 Author: voonhous <[email protected]> AuthorDate: Wed Nov 18 06:20:47 2026 +0000 feat(trino): add a direct HoodieSchema to Trino type mapper (#20680) * feat(trino): add a direct HoodieSchema to Trino type mapper The connector reaches its column types through the Hive type strings in the metastore, so HoodieSchema and Trino types are never related in one place. Add TrinoTypeMapper, which maps each HoodieSchemaType to one Trino type and rejects the ones Trino cannot hold. Nothing in the connector calls it yet; moving the column types onto it changes what Trino reports for UUID, TIME, TIMESTAMP and VECTOR columns and is a separate step. Closes #20670 * fix(trino): address review on TrinoTypeMapper - Reject the lone-null union ["null"] with NOT_SUPPORTED by checking isComplexUnion() before unwrapping, instead of letting getNonNullType() throw IllegalArgumentException. - Guard the schema argument with requireNonNull. - Rename the unwrapped local from type to nonNullSchema. - Javadoc: a synced TIMESTAMP is bigint through the metastore unless hoodie.datasource.hive_sync.support_timestamp is on. - Test BLOB and VARIANT against their full row types, which also covers a nested row, and add the ["null"] rejection case. --- .../io/trino/plugin/hudi/util/TrinoTypeMapper.java | 143 +++++++++++++++++++++ .../plugin/hudi/util/TestTrinoTypeMapper.java | 127 ++++++++++++++++++ 2 files changed, 270 insertions(+) diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/TrinoTypeMapper.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/TrinoTypeMapper.java new file mode 100644 index 000000000000..21980349b385 --- /dev/null +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/TrinoTypeMapper.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.util; + +import io.trino.spi.TrinoException; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.MapType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeOperators; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchema.TimePrecision; +import org.apache.hudi.common.schema.HoodieSchemaType; + +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.Decimals.MAX_PRECISION; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.TimeType.createTimeType; +import static io.trino.spi.type.TimestampType.createTimestampType; +import static io.trino.spi.type.TimestampWithTimeZoneType.createTimestampWithTimeZoneType; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.UuidType.UUID; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.util.Objects.requireNonNull; + +/** + * Maps {@link HoodieSchema} to Trino types directly, one Trino type per {@link HoodieSchemaType}. + * <p> + * The connector does not use this mapping yet: its column types still come from the Hive type strings in the + * metastore. That path gives different types for the rows marked below, because it goes through what Hive can + * declare. Moving the connector onto this mapping changes those column types and is a separate, gated step. + * <table> + * <caption>HoodieSchema to Trino</caption> + * <tr><th>HoodieSchema</th><th>Trino</th><th>Type identity</th></tr> + * <tr><td>BOOLEAN, INT, LONG, FLOAT, DOUBLE</td><td>boolean, integer, bigint, real, double</td><td>kept</td></tr> + * <tr><td>STRING, BYTES</td><td>varchar, varbinary</td><td>kept</td></tr> + * <tr><td>DECIMAL(p, s)</td><td>decimal(p, s), up to precision 38</td><td>kept</td></tr> + * <tr><td>DATE</td><td>date</td><td>kept</td></tr> + * <tr><td>TIME millis, micros</td><td>time(3), time(6)</td><td>kept; integer or bigint through the metastore</td></tr> + * <tr><td>TIMESTAMP millis, micros, UTC-adjusted</td><td>timestamp(3), timestamp(6) with time zone</td> + * <td>kept; bigint through the metastore, timestamp(9) with hoodie.datasource.hive_sync.support_timestamp</td></tr> + * <tr><td>TIMESTAMP millis, micros, local</td><td>timestamp(3), timestamp(6)</td> + * <td>kept; bigint through the metastore, timestamp(9) with hoodie.datasource.hive_sync.support_timestamp</td></tr> + * <tr><td>UUID</td><td>uuid</td><td>kept; varbinary through the metastore</td></tr> + * <tr><td>ARRAY, MAP, RECORD</td><td>array, map with varchar keys, row</td><td>kept</td></tr> + * <tr><td>ENUM</td><td>varchar</td><td>lost: Trino has no enum type</td></tr> + * <tr><td>FIXED</td><td>varbinary</td><td>lost: Trino has no fixed-length binary type</td></tr> + * <tr><td>VECTOR</td><td>array of real, double or tinyint</td><td>lost: the dimension; varbinary through the metastore</td></tr> + * <tr><td>BLOB, VARIANT</td><td>row of their fields</td><td>lost: Trino sees a plain row</td></tr> + * </table> + * hoodie.datasource.hive_sync.support_timestamp is off by default and on for Spark SQL writes. + * <p> + * A nullable type maps as its non-null type; Trino carries nullability on the column. Other unions and NULL have no + * Trino type and are rejected. + */ +public final class TrinoTypeMapper +{ + private final TypeOperators typeOperators; + + public TrinoTypeMapper(TypeOperators typeOperators) + { + this.typeOperators = requireNonNull(typeOperators, "typeOperators is null"); + } + + public Type toTypeSystem(HoodieSchema schema) + { + requireNonNull(schema, "schema is null"); + // Covers ["null"], which getNonNullType() cannot reduce, and every union wider than ["null", T] + if (schema.isComplexUnion()) { + throw unsupported(schema, "Trino has no union type"); + } + HoodieSchema nonNullSchema = schema.getNonNullType(); + return switch (nonNullSchema.getType()) { + case BOOLEAN -> BOOLEAN; + case INT -> INTEGER; + case LONG -> BIGINT; + case FLOAT -> REAL; + case DOUBLE -> DOUBLE; + case STRING, ENUM -> VARCHAR; + case BYTES, FIXED -> VARBINARY; + case UUID -> UUID; + case DATE -> DATE; + case DECIMAL -> { + HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) nonNullSchema; + if (decimal.getPrecision() > MAX_PRECISION) { + throw unsupported(nonNullSchema, "Trino decimals stop at precision " + MAX_PRECISION); + } + yield createDecimalType(decimal.getPrecision(), decimal.getScale()); + } + case TIME -> createTimeType(trinoPrecision(((HoodieSchema.Time) nonNullSchema).getPrecision())); + case TIMESTAMP -> { + HoodieSchema.Timestamp timestamp = (HoodieSchema.Timestamp) nonNullSchema; + int precision = trinoPrecision(timestamp.getPrecision()); + // A UTC-adjusted timestamp is an instant; a local one is a wall-clock time + yield timestamp.isUtcAdjusted() ? createTimestampWithTimeZoneType(precision) : createTimestampType(precision); + } + case ARRAY -> new ArrayType(toTypeSystem(nonNullSchema.getElementType())); + case MAP -> new MapType(VARCHAR, toTypeSystem(nonNullSchema.getValueType()), typeOperators); + case RECORD, BLOB, VARIANT -> RowType.from(nonNullSchema.getFields().stream() + .map(field -> RowType.field(field.name(), toTypeSystem(field.schema()))) + .toList()); + case VECTOR -> new ArrayType(switch (((HoodieSchema.Vector) nonNullSchema).getVectorElementType()) { + case FLOAT -> REAL; + case DOUBLE -> DOUBLE; + case INT8 -> TINYINT; + }); + // Unreachable: complex unions are rejected above and getNonNullType() unwraps ["null", T] + case UNION -> throw unsupported(nonNullSchema, "Trino has no union type"); + case NULL -> throw unsupported(nonNullSchema, "Trino has no null type"); + }; + } + + private static int trinoPrecision(TimePrecision precision) + { + return switch (precision) { + case MILLIS -> 3; + case MICROS -> 6; + }; + } + + private static TrinoException unsupported(HoodieSchema schema, String reason) + { + return new TrinoException(NOT_SUPPORTED, "No Trino type for Hudi type %s (%s): %s".formatted(schema.getType(), schema, reason)); + } +} diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTrinoTypeMapper.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTrinoTypeMapper.java new file mode 100644 index 000000000000..1f67b77bb67e --- /dev/null +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestTrinoTypeMapper.java @@ -0,0 +1,127 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.util; + +import io.trino.spi.TrinoException; +import io.trino.spi.type.ArrayType; +import io.trino.spi.type.MapType; +import io.trino.spi.type.RowType; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeOperators; +import org.apache.hudi.common.schema.HoodieSchema; +import org.apache.hudi.common.schema.HoodieSchemaField; +import org.apache.hudi.common.schema.HoodieSchemaType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.stream.Stream; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.RowType.field; +import static io.trino.spi.type.TimeType.TIME_MICROS; +import static io.trino.spi.type.TimeType.TIME_MILLIS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; +import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MICROS; +import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.UuidType.UUID; +import static io.trino.spi.type.VarbinaryType.VARBINARY; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +final class TestTrinoTypeMapper +{ + private static final TypeOperators TYPE_OPERATORS = new TypeOperators(); + private static final TrinoTypeMapper MAPPER = new TrinoTypeMapper(TYPE_OPERATORS); + + @ParameterizedTest + @MethodSource("mappings") + void testToTypeSystem(HoodieSchema schema, Type expected) + { + assertThat(MAPPER.toTypeSystem(schema)).isEqualTo(expected); + // Trino carries nullability on the column, not in the type + assertThat(MAPPER.toTypeSystem(HoodieSchema.createNullable(schema))).isEqualTo(expected); + } + + static Stream<Arguments> mappings() + { + HoodieSchema struct = HoodieSchema.createRecord("point", null, null, List.of( + HoodieSchemaField.of("name", HoodieSchema.create(HoodieSchemaType.STRING), null, null), + HoodieSchemaField.of("at", HoodieSchema.createNullable(HoodieSchema.createLocalTimestampMicros()), null, HoodieSchema.NULL_VALUE))); + return Stream.of( + Arguments.of(HoodieSchema.create(HoodieSchemaType.BOOLEAN), BOOLEAN), + Arguments.of(HoodieSchema.create(HoodieSchemaType.INT), INTEGER), + Arguments.of(HoodieSchema.create(HoodieSchemaType.LONG), BIGINT), + Arguments.of(HoodieSchema.create(HoodieSchemaType.FLOAT), REAL), + Arguments.of(HoodieSchema.create(HoodieSchemaType.DOUBLE), DOUBLE), + Arguments.of(HoodieSchema.create(HoodieSchemaType.STRING), VARCHAR), + Arguments.of(HoodieSchema.create(HoodieSchemaType.BYTES), VARBINARY), + Arguments.of(HoodieSchema.createEnum("color", null, null, List.of("RED", "GREEN")), VARCHAR), + Arguments.of(HoodieSchema.createFixed("md5", null, null, 16), VARBINARY), + Arguments.of(HoodieSchema.createUUID(), UUID), + Arguments.of(HoodieSchema.createDate(), DATE), + Arguments.of(HoodieSchema.createDecimal(10, 2), createDecimalType(10, 2)), + Arguments.of(HoodieSchema.createDecimal("amount", null, null, 20, 4, 9), createDecimalType(20, 4)), + Arguments.of(HoodieSchema.createTimeMillis(), TIME_MILLIS), + Arguments.of(HoodieSchema.createTimeMicros(), TIME_MICROS), + Arguments.of(HoodieSchema.createTimestampMillis(), TIMESTAMP_TZ_MILLIS), + Arguments.of(HoodieSchema.createTimestampMicros(), TIMESTAMP_TZ_MICROS), + Arguments.of(HoodieSchema.createLocalTimestampMillis(), TIMESTAMP_MILLIS), + Arguments.of(HoodieSchema.createLocalTimestampMicros(), TIMESTAMP_MICROS), + Arguments.of(HoodieSchema.createArray(HoodieSchema.createNullable(HoodieSchema.create(HoodieSchemaType.LONG))), new ArrayType(BIGINT)), + Arguments.of(HoodieSchema.createMap(HoodieSchema.createDate()), new MapType(VARCHAR, DATE, TYPE_OPERATORS)), + Arguments.of(struct, RowType.from(List.of(field("name", VARCHAR), field("at", TIMESTAMP_MICROS)))), + Arguments.of(HoodieSchema.createVector(4), new ArrayType(REAL)), + Arguments.of(HoodieSchema.createVector(4, HoodieSchema.Vector.VectorElementType.DOUBLE), new ArrayType(DOUBLE)), + Arguments.of(HoodieSchema.createVector(4, HoodieSchema.Vector.VectorElementType.INT8), new ArrayType(TINYINT)), + // BLOB's reference is a nested record, so this also covers a row inside a row + Arguments.of(HoodieSchema.createBlob(), RowType.from(List.of( + field("type", VARCHAR), + field("data", VARBINARY), + field("reference", RowType.from(List.of( + field("external_path", VARCHAR), + field("offset", BIGINT), + field("length", BIGINT), + field("managed", BOOLEAN))))))), + Arguments.of(HoodieSchema.createVariant(), RowType.from(List.of(field("metadata", VARBINARY), field("value", VARBINARY))))); + } + + @Test + void testTypesWithoutATrinoType() + { + assertThatThrownBy(() -> MAPPER.toTypeSystem(HoodieSchema.createDecimal(39, 2))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Trino decimals stop at precision 38"); + assertThatThrownBy(() -> MAPPER.toTypeSystem(HoodieSchema.createUnion(HoodieSchema.create(HoodieSchemaType.INT), HoodieSchema.create(HoodieSchemaType.STRING)))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Trino has no union type"); + assertThatThrownBy(() -> MAPPER.toTypeSystem(HoodieSchema.create(HoodieSchemaType.NULL))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Trino has no null type"); + assertThatThrownBy(() -> MAPPER.toTypeSystem(HoodieSchema.createUnion(HoodieSchema.create(HoodieSchemaType.NULL)))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Trino has no union type"); + } +}
