fmorillo7694 commented on code in PR #206: URL: https://github.com/apache/flink-connector-aws/pull/206#discussion_r4125200266
########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFunctionsUtil.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.catalog.CatalogFunction; +import org.apache.flink.table.catalog.FunctionLanguage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.model.UserDefinedFunction; + +import java.util.Arrays; +import java.util.stream.Collectors; + +/** + * Utility class for handling Functions in AWS Glue Catalog integration. Provides methods for + * converting between Flink and Glue function representation. + */ +public class GlueFunctionsUtil { + + private static final Logger LOG = LoggerFactory.getLogger(GlueFunctionsUtil.class); + + /** + * Extracts the class name from a Glue UserDefinedFunction. + * + * @param udf The Glue UserDefinedFunction + * @return The extracted class name + */ + public static String getCatalogFunctionClassName(final UserDefinedFunction udf) { + String[] splitName = udf.className().split(GlueCatalogConstants.DEFAULT_SEPARATOR); + return splitName[splitName.length - 1]; + } + + /** + * Determines the function language from a Glue UserDefinedFunction. + * + * @param glueFunction The Glue UserDefinedFunction + * @return The corresponding Flink FunctionLanguage; class names without a Flink language prefix + * (e.g. functions created by Hive or Spark) are treated as JAVA + */ + public static FunctionLanguage getFunctionalLanguage(final UserDefinedFunction glueFunction) { + if (glueFunction.className().startsWith(GlueCatalogConstants.FLINK_JAVA_FUNCTION_PREFIX)) { + return FunctionLanguage.JAVA; + } else if (glueFunction + .className() + .startsWith(GlueCatalogConstants.FLINK_PYTHON_FUNCTION_PREFIX)) { + return FunctionLanguage.PYTHON; + } else if (glueFunction + .className() + .startsWith(GlueCatalogConstants.FLINK_SCALA_FUNCTION_PREFIX)) { + return FunctionLanguage.SCALA; + } else { + // Functions created by other engines (e.g. Hive or Spark) store their class name + // without a Flink language prefix. Treat them as JAVA functions, which is the + // representation both engines use, instead of failing the whole lookup. + LOG.warn( + "Function class name '{}' has no Flink language prefix; assuming JAVA. " + + "Functions created by other engines may not be loadable by Flink.", + glueFunction.className()); + return FunctionLanguage.JAVA; + } + } + + /** + * Creates a Glue function class name from a Flink CatalogFunction. + * + * @param function The Flink CatalogFunction + * @return The formatted function class name for Glue + * @throws UnsupportedOperationException if the function language is not supported + */ + public static String getGlueFunctionClassName(CatalogFunction function) { + switch (function.getFunctionLanguage()) { + case JAVA: + return GlueCatalogConstants.FLINK_JAVA_FUNCTION_PREFIX + function.getClassName(); + case SCALA: + return GlueCatalogConstants.FLINK_SCALA_FUNCTION_PREFIX + function.getClassName(); + case PYTHON: + return GlueCatalogConstants.FLINK_PYTHON_FUNCTION_PREFIX + function.getClassName(); + default: Review Comment: Correct, only three languages exist today. I kept the `default` branch because the `switch` is on `FunctionLanguage` and dropping it would make a future enum value compile silently into a wrong class name; it is unreachable by construction rather than dead logic. Happy to remove it if you prefer. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFunctionsUtil.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.catalog.CatalogFunction; +import org.apache.flink.table.catalog.FunctionLanguage; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.model.UserDefinedFunction; + +import java.util.Arrays; +import java.util.stream.Collectors; + +/** + * Utility class for handling Functions in AWS Glue Catalog integration. Provides methods for + * converting between Flink and Glue function representation. + */ +public class GlueFunctionsUtil { + + private static final Logger LOG = LoggerFactory.getLogger(GlueFunctionsUtil.class); + + /** + * Extracts the class name from a Glue UserDefinedFunction. + * + * @param udf The Glue UserDefinedFunction + * @return The extracted class name + */ + public static String getCatalogFunctionClassName(final UserDefinedFunction udf) { + String[] splitName = udf.className().split(GlueCatalogConstants.DEFAULT_SEPARATOR); Review Comment: Done — `getCatalogFunctionClassName` strips the matching `flink:<lang>:` prefix instead of splitting on `:`; a class name containing `:` or a foreign (non-prefixed) class name is returned unchanged. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTableUtils.java: ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.types.DataType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.model.Column; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.Table; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Utility class for working with Glue tables, including transforming Glue-specific metadata into + * Flink-compatible objects. + */ +public class GlueTableUtils { + + /** Logger for logging Glue table operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableUtils.class); + + /** Glue type converter for type conversions between Flink and Glue types. */ + private final GlueTypeConverter glueTypeConverter; + + /** + * Constructor to initialize GlueTableUtils with a GlueTypeConverter. + * + * @param glueTypeConverter The GlueTypeConverter instance for type mapping. + */ + public GlueTableUtils(GlueTypeConverter glueTypeConverter) { + this.glueTypeConverter = glueTypeConverter; + } + + /** + * Builds a Glue StorageDescriptor from the given table properties, columns, and location. + * + * @param tableProperties Table properties for the Glue table. + * @param glueColumns Columns to be included in the StorageDescriptor. + * @param tableLocation Location of the Glue table. + * @return A newly built StorageDescriptor object. + */ + public StorageDescriptor buildStorageDescriptor( Review Comment: Removed — `buildStorageDescriptor(columns, location)` takes only what it uses. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTableUtils.java: ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.types.DataType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.model.Column; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.Table; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Utility class for working with Glue tables, including transforming Glue-specific metadata into + * Flink-compatible objects. + */ +public class GlueTableUtils { + + /** Logger for logging Glue table operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableUtils.class); + + /** Glue type converter for type conversions between Flink and Glue types. */ + private final GlueTypeConverter glueTypeConverter; + + /** + * Constructor to initialize GlueTableUtils with a GlueTypeConverter. + * + * @param glueTypeConverter The GlueTypeConverter instance for type mapping. + */ + public GlueTableUtils(GlueTypeConverter glueTypeConverter) { + this.glueTypeConverter = glueTypeConverter; + } + + /** + * Builds a Glue StorageDescriptor from the given table properties, columns, and location. + * + * @param tableProperties Table properties for the Glue table. + * @param glueColumns Columns to be included in the StorageDescriptor. + * @param tableLocation Location of the Glue table. + * @return A newly built StorageDescriptor object. + */ + public StorageDescriptor buildStorageDescriptor( + Map<String, String> tableProperties, List<Column> glueColumns, String tableLocation) { + + return StorageDescriptor.builder().columns(glueColumns).location(tableLocation).build(); + } + + /** + * Extracts the table location based on the table properties and the table path. First, it + * checks for a location key from the connector registry. If no such key is found, it uses a + * default path based on the table path. + * + * @param tableProperties Table properties containing the connector and location. + * @param tablePath The Flink ObjectPath representing the table. + * @return The location of the Glue table. + */ + public String extractTableLocation(Map<String, String> tableProperties, ObjectPath tablePath) { Review Comment: It computed a default location (`db/tables/name`) for every table, which was wrong in two ways: it is not a URI (Athena chokes on it) and for Kinesis/Kafka tables it put a bootstrap-server list or stream ARN into `StorageDescriptor.location`. It is now `extractTableLocation` (only a connector's URI location, e.g. `s3://…`, `jdbc:…`) plus `resolveTableLocation`, which additionally writes a synthetic `flink://db/table` **only for partitioned tables** — the real-Glue run showed Glue rejects `CreatePartition` on a table without any location ("Cannot add partition to a table without location"), and I verified with the CLI that any non-empty location, URI or not, unblocks it. Unpartitioned tables of non-filesystem connectors keep no location. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTableUtils.java: ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.types.DataType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.model.Column; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.Table; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Utility class for working with Glue tables, including transforming Glue-specific metadata into + * Flink-compatible objects. + */ +public class GlueTableUtils { + + /** Logger for logging Glue table operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableUtils.class); + + /** Glue type converter for type conversions between Flink and Glue types. */ + private final GlueTypeConverter glueTypeConverter; + + /** + * Constructor to initialize GlueTableUtils with a GlueTypeConverter. + * + * @param glueTypeConverter The GlueTypeConverter instance for type mapping. + */ + public GlueTableUtils(GlueTypeConverter glueTypeConverter) { + this.glueTypeConverter = glueTypeConverter; + } + + /** + * Builds a Glue StorageDescriptor from the given table properties, columns, and location. + * + * @param tableProperties Table properties for the Glue table. + * @param glueColumns Columns to be included in the StorageDescriptor. + * @param tableLocation Location of the Glue table. + * @return A newly built StorageDescriptor object. + */ + public StorageDescriptor buildStorageDescriptor( + Map<String, String> tableProperties, List<Column> glueColumns, String tableLocation) { + + return StorageDescriptor.builder().columns(glueColumns).location(tableLocation).build(); + } + + /** + * Extracts the table location based on the table properties and the table path. First, it + * checks for a location key from the connector registry. If no such key is found, it uses a + * default path based on the table path. + * + * @param tableProperties Table properties containing the connector and location. + * @param tablePath The Flink ObjectPath representing the table. + * @return The location of the Glue table. + */ + public String extractTableLocation(Map<String, String> tableProperties, ObjectPath tablePath) { + String connectorType = tableProperties.get("connector"); + if (connectorType != null) { + String locationKey = ConnectorRegistry.getLocationKey(connectorType); + if (locationKey != null && tableProperties.containsKey(locationKey)) { + String location = tableProperties.get(locationKey); + return location; + } + } + + String defaultLocation = + tablePath.getDatabaseName() + "/tables/" + tablePath.getObjectName(); + return defaultLocation; + } + + /** + * Converts a Flink column to a Glue column. The column's data type is converted using the + * GlueTypeConverter. + * + * @param flinkColumn The Flink column to be converted. + * @return The corresponding Glue column. + */ + public Column mapFlinkColumnToGlueColumn(org.apache.flink.table.catalog.Column flinkColumn) { + String glueType = glueTypeConverter.toGlueDataType(flinkColumn.getDataType()); + + // AWS Glue lowercases column names on CreateTable/UpdateTable (verified empirically: + // a column created as "userId" is stored and returned as "userid"). To preserve the + // declared case, store the lowercased name explicitly and stash the original name in + // the "originalName" column parameter, which the read path restores. + String originalName = flinkColumn.getName(); + String glueName = originalName.toLowerCase(); + + Column.Builder builder = Column.builder().name(glueName).type(glueType); Review Comment: Done — physical column comments are stored natively on the Glue column, computed/metadata column comments as a `.comment` schema parameter; both restored on read (`testColumnCommentsSurviveRoundTrip`). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
