fmorillo7694 commented on code in PR #206: URL: https://github.com/apache/flink-connector-aws/pull/206#discussion_r4125208350
########## flink-connector-aws-e2e-tests/flink-catalog-aws-glue-e2e-tests/src/test/java/org/apache/flink/table/catalog/glue/test/GlueCatalogEndToEndITCase.java: ########## @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.test; + +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.types.Row; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.GlueClientBuilder; +import software.amazon.awssdk.services.glue.model.Column; +import software.amazon.awssdk.services.glue.model.EntityNotFoundException; +import software.amazon.awssdk.services.glue.model.GetTableRequest; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +/** + * End-to-end test for the Glue catalog against <b>real AWS Glue</b>. + * + * <p>This test exercises the complete user path: {@code CREATE CATALOG ... WITH ('type'='glue')} + * discovers {@code GlueCatalogFactory} via SPI, the factory builds its own {@code GlueClient} from + * the default credential/region chain, and all catalog operations flow through Flink SQL DDL and + * the planner down to the real Glue service. + * + * <p>Following the convention of the other AWS end-to-end tests in this repository, the test is + * gated on explicit credentials and skips cleanly when they are absent: + * + * <ul> + * <li>{@code IT_CASE_GLUE_CATALOG_ACCESS_KEY} - AWS access key id (required) + * <li>{@code IT_CASE_GLUE_CATALOG_SECRET_KEY} - AWS secret access key (required) + * <li>{@code IT_CASE_GLUE_CATALOG_SESSION_TOKEN} - session token (optional, for temporary + * credentials) + * <li>{@code IT_CASE_GLUE_CATALOG_REGION} - region (optional, defaults to {@code us-east-1}) + * </ul> + * + * <p>Credentials are handed to the factory-built client through the AWS SDK system properties + * ({@code aws.accessKeyId}, {@code aws.secretAccessKey}, {@code aws.sessionToken}, {@code + * aws.region}), which the default provider chain resolves - no test-only code path exists in the + * factory. + * + * <p>Unlike the moto-based integration tests in the catalog module, this test also covers the two + * behaviours only real Glue exhibits: column-name lowercasing with case restoration through the + * catalog, and {@code dropDatabase} (whose emptiness check requires the UDF API that moto does not + * implement). + * + * <p>Run with: {@code mvn verify -Prun-aws-end-to-end-tests} with the environment variables above. + */ +@Tag("requires-aws-credentials") +class GlueCatalogEndToEndITCase { Review Comment: Added `testPartitionedTableEndToEnd` (partition key declared first, `DESCRIBE` per-column types, `ADD`/`DROP`/`SHOW PARTITIONS` against real Glue) — green against real Glue. There is also a new `GlueCatalogKinesisEndToEndITCase` that stores Kinesis table definitions in real Glue and runs a streaming job through them against a Localstack Kinesis (records in, filtered records out), proving what the catalog persists is a working table definition. ########## flink-catalog-aws/flink-catalog-aws-glue/src/test/java/org/apache/flink/table/catalog/glue/operator/GlueTableOperationsTest.java: ########## @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.operator; + +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.apache.flink.table.catalog.glue.util.GlueTestClientFactory; +import org.apache.flink.table.catalog.glue.util.RealGlueCleanupExtension; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.CreateTableRequest; +import software.amazon.awssdk.services.glue.model.InvalidInputException; +import software.amazon.awssdk.services.glue.model.OperationTimeoutException; +import software.amazon.awssdk.services.glue.model.ResourceNumberLimitExceededException; +import software.amazon.awssdk.services.glue.model.Table; +import software.amazon.awssdk.services.glue.model.TableInput; + +import java.util.List; + +import static org.assertj.core.api.Assumptions.assumeThat; + +/** + * Unit tests for the GlueTableOperations class. These tests verify that table operations such as + * create, drop, get, and list are correctly executed against the AWS Glue service. + */ +@ExtendWith(RealGlueCleanupExtension.class) +public class GlueTableOperationsTest { + + private static final String CATALOG_NAME = "testcatalog"; + private String databaseName; + private static final String TABLE_NAME = "testtable"; + + private GlueClient glueClient; + private GlueTableOperator glueTableOperations; + + @BeforeEach + void setUp() { + glueClient = GlueTestClientFactory.createClient(); + glueTableOperations = new GlueTableOperator(glueClient, CATALOG_NAME); + databaseName = GlueTestClientFactory.uniqueName("testdb"); + // Real Glue rejects table operations in a non-existent database (the in-memory + // fake is lenient), so the test database must actually exist. + glueClient.createDatabase(b -> b.databaseInput(db -> db.name(databaseName))); + } + + @Test + void testTableExists() { + // Create a test table + TableInput tableInput = TableInput.builder().name(TABLE_NAME).build(); + glueClient.createTable( + CreateTableRequest.builder() + .databaseName(databaseName) + .tableInput(tableInput) + .build()); + + Assertions.assertTrue(glueTableOperations.glueTableExists(databaseName, TABLE_NAME)); + } + + @Test + void testTableExistsWhenNotFound() { + Assertions.assertFalse(glueTableOperations.glueTableExists(databaseName, TABLE_NAME)); + } + + @Test + void testListTables() { + // Create test tables + TableInput table1 = TableInput.builder().name("table1").build(); + TableInput table2 = TableInput.builder().name("table2").build(); + + glueClient.createTable( + CreateTableRequest.builder().databaseName(databaseName).tableInput(table1).build()); + glueClient.createTable( + CreateTableRequest.builder().databaseName(databaseName).tableInput(table2).build()); + + List<String> result = glueTableOperations.listTables(databaseName); + Assertions.assertEquals(2, result.size()); + Assertions.assertTrue(result.contains("table1")); + Assertions.assertTrue(result.contains("table2")); + } + + @Test + void testListTablesWithInvalidInput() { + fakeClient() + .setNextException(InvalidInputException.builder().message("Invalid input").build()); + Assertions.assertThrows( + CatalogException.class, () -> glueTableOperations.listTables(databaseName)); + } + + @Test + void testCreateTable() { + TableInput tableInput = TableInput.builder().name(TABLE_NAME).build(); + + Assertions.assertDoesNotThrow( + () -> glueTableOperations.createTable(databaseName, tableInput)); + Assertions.assertTrue(glueTableOperations.glueTableExists(databaseName, TABLE_NAME)); + } + + @Test + void testCreateTableWithUppercaseLetters() { + TableInput tableInput = TableInput.builder().name("TestTable").build(); + + // Uppercase letters should now be accepted with case preservation Review Comment: Done — the test now goes through `buildTableInput` and asserts the storage name is lowercase, the declared name is preserved, the table resolves by any spelling, and `listTables` returns the declared case. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFlinkSchemaProperties.java: ########## @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; + +/** + * Persists the parts of a Flink schema that AWS Glue columns cannot represent - computed columns, + * metadata columns, watermarks, and the primary key - as Glue table parameters, and restores them + * when a table is read back. + * + * <p>Physical columns are stored as native Glue columns (so other engines can read the table), + * while everything else round-trips through {@code flink.schema.*} parameters. Without this, {@code + * CREATE TABLE} statements with watermarks or primary keys would silently lose them on read back, + * and computed or metadata columns would be corrupted into physical columns. + */ +@Internal +public final class GlueFlinkSchemaProperties { + + /** Prefix shared by all schema-fidelity parameters written by this class. */ + public static final String SCHEMA_PARAMETER_PREFIX = "flink.schema."; + + private static final String COLUMN_PREFIX = SCHEMA_PARAMETER_PREFIX + "column."; + private static final String COMPUTED_NAME_SUFFIX = ".computed.name"; + private static final String COMPUTED_EXPR_SUFFIX = ".computed.expr"; + private static final String METADATA_NAME_SUFFIX = ".metadata.name"; + private static final String METADATA_DATA_TYPE_SUFFIX = ".metadata.data-type"; + private static final String METADATA_KEY_SUFFIX = ".metadata.key"; + private static final String METADATA_VIRTUAL_SUFFIX = ".metadata.virtual"; + private static final String PHYSICAL_DATA_TYPE_SUFFIX = ".data-type"; + + private static final String WATERMARK_PREFIX = SCHEMA_PARAMETER_PREFIX + "watermark."; + private static final String WATERMARK_ROWTIME_SUFFIX = ".rowtime"; + private static final String WATERMARK_STRATEGY_EXPR_SUFFIX = ".strategy.expr"; + + private static final String PRIMARY_KEY_NAME = SCHEMA_PARAMETER_PREFIX + "primary-key.name"; + private static final String PRIMARY_KEY_COLUMNS = + SCHEMA_PARAMETER_PREFIX + "primary-key.columns"; + + /** + * Comma-joined names of physical columns declared NOT NULL. Glue type strings carry no + * nullability, so without this the restored primary key would fail schema resolution ("Invalid + * primary key: column is nullable"). + */ + private static final String NOT_NULL_COLUMNS = SCHEMA_PARAMETER_PREFIX + "not-null-columns"; + + private static final GlueTypeConverter TYPE_CONVERTER = new GlueTypeConverter(); + + private GlueFlinkSchemaProperties() {} + + /** + * Serializes the non-physical parts of the given schema into the target parameter map. + * + * @param resolvedSchema the resolved Flink schema + * @param targetParameters the mutable Glue table parameter map to write into + * @throws CatalogException if an expression cannot be serialized to SQL + */ + public static void serializeNonPhysicalSchema( + ResolvedSchema resolvedSchema, Map<String, String> targetParameters) { + List<Column> columns = resolvedSchema.getColumns(); + List<String> notNullColumns = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + if (column instanceof Column.PhysicalColumn) { + if (!column.getDataType().getLogicalType().isNullable()) { + notNullColumns.add(column.getName()); + } + // Glue's Hive-style type strings are lossy for some types (for example + // TIMESTAMP(3) has no Glue representation and would come back as + // TIMESTAMP(6)). When the Glue round-trip does not reproduce the declared + // type, record the original so the read path can restore it exactly. + LogicalType declared = column.getDataType().getLogicalType(); + LogicalType glueRoundTrip = + TYPE_CONVERTER + .toFlinkDataType( + TYPE_CONVERTER.toGlueDataType(column.getDataType())) + .getLogicalType(); + if (!declared.copy(true).equals(glueRoundTrip.copy(true))) { + targetParameters.put( + COLUMN_PREFIX + i + PHYSICAL_DATA_TYPE_SUFFIX, Review Comment: Exactly — with a partition key declared first, the `TIMESTAMP(3)` recorded for position 1 landed on the INT column. Fixed by keying the parameters by name (thread on `GlueTableUtils:159`); `testPartitionedTableDeclaredOrderAndTypesSurviveRoundTrip` asserts `ts` is `TIMESTAMP(3)`, `id` is `INT NOT NULL`, etc., after the round-trip. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTypeConverter.java: ########## @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.catalog.glue.exception.UnsupportedDataTypeMappingException; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.RowType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class for converting Flink types to Glue types and vice versa. Supports the conversion of + * common primitive, array, map, and struct types. + */ +public class GlueTypeConverter { + + /** Logger for tracking Glue type conversions. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTypeConverter.class); + + /** Regular expressions for handling specific Glue types. */ + private static final Pattern DECIMAL_PATTERN = Pattern.compile("decimal\\((\\d+),(\\d+)\\)"); + + private static final Pattern ARRAY_PATTERN = Pattern.compile("array<(.+)>"); + private static final Pattern MAP_PATTERN = Pattern.compile("map<(.+),(.+)>"); + private static final Pattern STRUCT_PATTERN = Pattern.compile("struct<(.+)>"); + + /** + * Converts a Flink DataType to its corresponding Glue type as a string. + * + * @param flinkType The Flink DataType to be converted. + * @return The Glue type as a string. + */ + public String toGlueDataType(DataType flinkType) { + LogicalType logicalType = flinkType.getLogicalType(); + LogicalTypeRoot typeRoot = logicalType.getTypeRoot(); + + // Handle various Flink types and map them to corresponding Glue types + switch (typeRoot) { + case CHAR: + case VARCHAR: + return "string"; + case BOOLEAN: + return "boolean"; + case BINARY: + case VARBINARY: + return "binary"; + case DECIMAL: + DecimalType decimalType = (DecimalType) logicalType; + return String.format( + "decimal(%d,%d)", decimalType.getPrecision(), decimalType.getScale()); + case TINYINT: + return "tinyint"; + case SMALLINT: + return "smallint"; + case INTEGER: + return "int"; + case BIGINT: + return "bigint"; + case FLOAT: + return "float"; + case DOUBLE: + return "double"; + case DATE: + return "date"; + case TIME_WITHOUT_TIME_ZONE: + return "string"; // Glue doesn't have a direct time type, use string + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return "timestamp"; + case ARRAY: + ArrayType arrayType = (ArrayType) logicalType; + return "array<" + toGlueDataType(DataTypes.of(arrayType.getElementType())) + ">"; + case MAP: + MapType mapType = (MapType) logicalType; + return String.format( + "map<%s,%s>", + toGlueDataType(DataTypes.of(mapType.getKeyType())), + toGlueDataType(DataTypes.of(mapType.getValueType()))); + case ROW: + RowType rowType = (RowType) logicalType; + StringBuilder structBuilder = new StringBuilder("struct<"); + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (i > 0) { + structBuilder.append(","); + } + // Keep original field name for nested structs + structBuilder + .append(rowType.getFieldNames().get(i)) + .append(":") + .append(toGlueDataType(DataTypes.of(rowType.getChildren().get(i)))); + } + structBuilder.append(">"); + return structBuilder.toString(); + default: + throw new UnsupportedDataTypeMappingException( + "Flink type not supported by Glue Catalog: " + logicalType.getTypeRoot()); + } + } + + /** + * Converts a Glue type (as a string) to the corresponding Flink DataType. + * + * @param glueType The Glue type as a string. + * @return The corresponding Flink DataType. + * @throws IllegalArgumentException if the Glue type is invalid or unknown. + */ + public DataType toFlinkDataType(String glueType) { + if (glueType == null || glueType.trim().isEmpty()) { + throw new IllegalArgumentException("Glue type cannot be null or empty"); + } + + // Trim but don't lowercase - we'll handle case-insensitivity per type + String trimmedGlueType = glueType.trim(); + + // Handle DECIMAL type - using lowercase for pattern matching + Matcher decimalMatcher = DECIMAL_PATTERN.matcher(trimmedGlueType.toLowerCase()); + if (decimalMatcher.matches()) { + int precision = Integer.parseInt(decimalMatcher.group(1)); + int scale = Integer.parseInt(decimalMatcher.group(2)); + return DataTypes.DECIMAL(precision, scale); + } + + // Handle ARRAY type - using lowercase for pattern matching but preserving content + Matcher arrayMatcher = ARRAY_PATTERN.matcher(trimmedGlueType); + if (arrayMatcher.matches()) { + // Extract from original string to preserve case in content + int contentStart = trimmedGlueType.indexOf('<') + 1; + int contentEnd = trimmedGlueType.lastIndexOf('>'); + String elementType = trimmedGlueType.substring(contentStart, contentEnd); + return DataTypes.ARRAY(toFlinkDataType(elementType)); + } + + // Handle MAP type - using lowercase for pattern matching but preserving content + Matcher mapMatcher = MAP_PATTERN.matcher(trimmedGlueType); + if (mapMatcher.matches()) { + // Extract from original string to preserve case in content + int contentStart = trimmedGlueType.indexOf('<') + 1; + int contentEnd = trimmedGlueType.lastIndexOf('>'); + String mapContent = trimmedGlueType.substring(contentStart, contentEnd); + + // Split key and value types + int commaPos = findMapTypeSeparator(mapContent); + if (commaPos < 0) { + throw new IllegalArgumentException("Invalid map type format: " + glueType); + } + + String keyType = mapContent.substring(0, commaPos).trim(); + String valueType = mapContent.substring(commaPos + 1).trim(); + + return DataTypes.MAP(toFlinkDataType(keyType), toFlinkDataType(valueType)); + } + + // Handle STRUCT type - using lowercase for pattern matching but preserving content + Matcher structMatcher = STRUCT_PATTERN.matcher(trimmedGlueType); + if (structMatcher.matches()) { + // Extract from original string to preserve case in field names + int contentStart = trimmedGlueType.indexOf('<') + 1; + int contentEnd = trimmedGlueType.lastIndexOf('>'); + String structContent = trimmedGlueType.substring(contentStart, contentEnd); + + return parseStructType(structContent); + } + + // Handle primitive types (case insensitive) + switch (trimmedGlueType.toLowerCase()) { Review Comment: Confirmed against the Glue Data Catalog type reference: boolean, tinyint, smallint, int, bigint, float, double, decimal, string, char, varchar, binary, date, timestamp, array, map, struct, union. Added `integer` (Athena DDL alias), bare `decimal` (Hive default `DECIMAL(10,0)`), parameterised `char(n)`/`varchar(n)`, and an explicit "Hive union types have no Flink equivalent" rejection for `uniontype<…>`. The parameterised test enumerates them all. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTypeConverter.java: ########## @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.catalog.glue.exception.UnsupportedDataTypeMappingException; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.LogicalTypeRoot; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.RowType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class for converting Flink types to Glue types and vice versa. Supports the conversion of + * common primitive, array, map, and struct types. + */ +public class GlueTypeConverter { + + /** Logger for tracking Glue type conversions. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTypeConverter.class); + + /** Regular expressions for handling specific Glue types. */ + private static final Pattern DECIMAL_PATTERN = Pattern.compile("decimal\\((\\d+),(\\d+)\\)"); + + private static final Pattern ARRAY_PATTERN = Pattern.compile("array<(.+)>"); + private static final Pattern MAP_PATTERN = Pattern.compile("map<(.+),(.+)>"); + private static final Pattern STRUCT_PATTERN = Pattern.compile("struct<(.+)>"); + + /** + * Converts a Flink DataType to its corresponding Glue type as a string. + * + * @param flinkType The Flink DataType to be converted. + * @return The Glue type as a string. + */ + public String toGlueDataType(DataType flinkType) { + LogicalType logicalType = flinkType.getLogicalType(); + LogicalTypeRoot typeRoot = logicalType.getTypeRoot(); + + // Handle various Flink types and map them to corresponding Glue types + switch (typeRoot) { + case CHAR: + case VARCHAR: + return "string"; + case BOOLEAN: + return "boolean"; + case BINARY: + case VARBINARY: + return "binary"; + case DECIMAL: + DecimalType decimalType = (DecimalType) logicalType; + return String.format( + "decimal(%d,%d)", decimalType.getPrecision(), decimalType.getScale()); + case TINYINT: + return "tinyint"; + case SMALLINT: + return "smallint"; + case INTEGER: + return "int"; + case BIGINT: + return "bigint"; + case FLOAT: + return "float"; + case DOUBLE: + return "double"; + case DATE: + return "date"; + case TIME_WITHOUT_TIME_ZONE: + return "string"; // Glue doesn't have a direct time type, use string + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return "timestamp"; + case ARRAY: + ArrayType arrayType = (ArrayType) logicalType; + return "array<" + toGlueDataType(DataTypes.of(arrayType.getElementType())) + ">"; + case MAP: + MapType mapType = (MapType) logicalType; + return String.format( + "map<%s,%s>", + toGlueDataType(DataTypes.of(mapType.getKeyType())), + toGlueDataType(DataTypes.of(mapType.getValueType()))); + case ROW: + RowType rowType = (RowType) logicalType; + StringBuilder structBuilder = new StringBuilder("struct<"); + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (i > 0) { + structBuilder.append(","); + } + // Keep original field name for nested structs + structBuilder + .append(rowType.getFieldNames().get(i)) + .append(":") + .append(toGlueDataType(DataTypes.of(rowType.getChildren().get(i)))); + } + structBuilder.append(">"); + return structBuilder.toString(); + default: + throw new UnsupportedDataTypeMappingException( + "Flink type not supported by Glue Catalog: " + logicalType.getTypeRoot()); + } + } + + /** + * Converts a Glue type (as a string) to the corresponding Flink DataType. + * + * @param glueType The Glue type as a string. + * @return The corresponding Flink DataType. + * @throws IllegalArgumentException if the Glue type is invalid or unknown. + */ + public DataType toFlinkDataType(String glueType) { + if (glueType == null || glueType.trim().isEmpty()) { + throw new IllegalArgumentException("Glue type cannot be null or empty"); + } + + // Trim but don't lowercase - we'll handle case-insensitivity per type + String trimmedGlueType = glueType.trim(); + + // Handle DECIMAL type - using lowercase for pattern matching + Matcher decimalMatcher = DECIMAL_PATTERN.matcher(trimmedGlueType.toLowerCase()); + if (decimalMatcher.matches()) { + int precision = Integer.parseInt(decimalMatcher.group(1)); + int scale = Integer.parseInt(decimalMatcher.group(2)); + return DataTypes.DECIMAL(precision, scale); + } + + // Handle ARRAY type - using lowercase for pattern matching but preserving content + Matcher arrayMatcher = ARRAY_PATTERN.matcher(trimmedGlueType); + if (arrayMatcher.matches()) { + // Extract from original string to preserve case in content + int contentStart = trimmedGlueType.indexOf('<') + 1; + int contentEnd = trimmedGlueType.lastIndexOf('>'); + String elementType = trimmedGlueType.substring(contentStart, contentEnd); + return DataTypes.ARRAY(toFlinkDataType(elementType)); + } + + // Handle MAP type - using lowercase for pattern matching but preserving content + Matcher mapMatcher = MAP_PATTERN.matcher(trimmedGlueType); + if (mapMatcher.matches()) { + // Extract from original string to preserve case in content + int contentStart = trimmedGlueType.indexOf('<') + 1; + int contentEnd = trimmedGlueType.lastIndexOf('>'); + String mapContent = trimmedGlueType.substring(contentStart, contentEnd); + + // Split key and value types + int commaPos = findMapTypeSeparator(mapContent); + if (commaPos < 0) { + throw new IllegalArgumentException("Invalid map type format: " + glueType); + } + + String keyType = mapContent.substring(0, commaPos).trim(); + String valueType = mapContent.substring(commaPos + 1).trim(); + + return DataTypes.MAP(toFlinkDataType(keyType), toFlinkDataType(valueType)); + } + + // Handle STRUCT type - using lowercase for pattern matching but preserving content + Matcher structMatcher = STRUCT_PATTERN.matcher(trimmedGlueType); + if (structMatcher.matches()) { + // Extract from original string to preserve case in field names + int contentStart = trimmedGlueType.indexOf('<') + 1; + int contentEnd = trimmedGlueType.lastIndexOf('>'); + String structContent = trimmedGlueType.substring(contentStart, contentEnd); + + return parseStructType(structContent); + } + + // Handle primitive types (case insensitive) + switch (trimmedGlueType.toLowerCase()) { Review Comment: Two bugs, both fixed. `varchar(255)`/`char(n)` now parse to `VARCHAR(n)`/`CHAR(n)`. And the deeper one your run would have hit next: `getTable` threw `Unknown table type: EXTERNAL_TABLE` for every table not created by Flink (Athena, crawlers, Spark, Hive), and `VIRTUAL_VIEW` likewise — so `DESCRIBE`/`SHOW CREATE TABLE` failed on essentially every pre-existing Glue table. `EXTERNAL_TABLE`/`MANAGED_TABLE`/`GOVERNED` now map to TABLE and `VIRTUAL_VIEW`/`MATERIALIZED_VIEW` to VIEW (HiveCatalog behaviour); the planner then reports the missing `connector` option cleanly. `testForeignExternalTableWithHiveTypesIsReadable` and `testForeignVirtualViewIsReadAsView` cover it; the docs Limitations entry was rewritten accordingly. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
