This is an automated email from the ASF dual-hosted git repository. blue pushed a commit to branch 0.14.x in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 950e6c01f94f9156c6321b462ebc7036b55ac06b Author: ChenLiang <[email protected]> AuthorDate: Fri Sep 2 03:48:32 2022 +0800 Core: Support deleting tables without metadata files (#5510) --- .../iceberg/aws/dynamodb/DynamoDbCatalog.java | 26 +++++++++++++++------ .../org/apache/iceberg/aws/glue/GlueCatalog.java | 27 ++++++++++++++++------ .../java/org/apache/iceberg/jdbc/JdbcCatalog.java | 16 +++++++++---- .../org/apache/iceberg/jdbc/TestJdbcCatalog.java | 15 ++++++++++++ .../java/org/apache/iceberg/hive/HiveCatalog.java | 16 +++++++++---- .../org/apache/iceberg/hive/TestHiveCatalog.java | 18 +++++++++++++++ 6 files changed, 94 insertions(+), 24 deletions(-) diff --git a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java index cf9a21ac57..1a69442707 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java +++ b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java @@ -43,6 +43,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; @@ -351,13 +352,24 @@ public class DynamoDbCatalog extends BaseMetastoreCatalog implements Closeable, } TableOperations ops = newTableOps(identifier); - TableMetadata lastMetadata = ops.current(); - dynamo.deleteItem(DeleteItemRequest.builder() - .tableName(awsProperties.dynamoDbTableName()) - .key(tablePrimaryKey(identifier)) - .conditionExpression(COL_VERSION + " = :v") - .expressionAttributeValues(ImmutableMap.of(":v", response.item().get(COL_VERSION))) - .build()); + TableMetadata lastMetadata = null; + if (purge) { + try { + lastMetadata = ops.current(); + } catch (NotFoundException e) { + LOG.warn( + "Failed to load table metadata for table: {}, continuing drop without purge", + identifier, + e); + } + } + dynamo.deleteItem( + DeleteItemRequest.builder() + .tableName(awsProperties.dynamoDbTableName()) + .key(tablePrimaryKey(identifier)) + .conditionExpression(COL_VERSION + " = :v") + .expressionAttributeValues(ImmutableMap.of(":v", response.item().get(COL_VERSION))) + .build()); LOG.info("Successfully dropped table {} from DynamoDb catalog", identifier); if (purge && lastMetadata != null) { diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java index 755005abe6..5e03cdd978 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java @@ -46,6 +46,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; @@ -273,13 +274,25 @@ public class GlueCatalog extends BaseMetastoreCatalog public boolean dropTable(TableIdentifier identifier, boolean purge) { try { TableOperations ops = newTableOps(identifier); - TableMetadata lastMetadata = ops.current(); - glue.deleteTable(DeleteTableRequest.builder() - .catalogId(awsProperties.glueCatalogId()) - .databaseName(IcebergToGlueConverter.getDatabaseName( - identifier, awsProperties.glueCatalogSkipNameValidation())) - .name(identifier.name()) - .build()); + TableMetadata lastMetadata = null; + if (purge) { + try { + lastMetadata = ops.current(); + } catch (NotFoundException e) { + LOG.warn( + "Failed to load table metadata for table: {}, continuing drop without purge", + identifier, + e); + } + } + glue.deleteTable( + DeleteTableRequest.builder() + .catalogId(awsProperties.glueCatalogId()) + .databaseName( + IcebergToGlueConverter.getDatabaseName( + identifier, awsProperties.glueCatalogSkipNameValidation())) + .name(identifier.name()) + .build()); LOG.info("Successfully dropped table {} from Glue", identifier); if (purge && lastMetadata != null) { CatalogUtil.dropTableData(ops.io(), lastMetadata); diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java index a5ffa2f7c0..f415f8c757 100644 --- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java +++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java @@ -49,6 +49,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.Joiner; @@ -154,11 +155,16 @@ public class JdbcCatalog extends BaseMetastoreCatalog @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { TableOperations ops = newTableOps(identifier); - TableMetadata lastMetadata; - if (purge && ops.current() != null) { - lastMetadata = ops.current(); - } else { - lastMetadata = null; + TableMetadata lastMetadata = null; + if (purge) { + try { + lastMetadata = ops.current(); + } catch (NotFoundException e) { + LOG.warn( + "Failed to load table metadata for table: {}, continuing drop without purge", + identifier, + e); + } } int deletedRecords = execute( diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java index ec8c3671aa..7e4e1fbf9f 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java @@ -41,6 +41,7 @@ import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; @@ -66,6 +67,7 @@ import org.junit.jupiter.api.io.TempDir; import static org.apache.iceberg.NullOrder.NULLS_FIRST; import static org.apache.iceberg.SortDirection.ASC; import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestJdbcCatalog extends CatalogTests<JdbcCatalog> { @@ -372,6 +374,19 @@ public class TestJdbcCatalog extends CatalogTests<JdbcCatalog> { Assert.assertFalse(catalog.dropTable(TableIdentifier.of("db", "tbl-not-exists"))); } + @Test + public void testDropTableWithoutMetadataFile() { + TableIdentifier testTable = TableIdentifier.of("db", "ns1", "ns2", "tbl"); + catalog.createTable(testTable, SCHEMA, PartitionSpec.unpartitioned()); + String metadataFileLocation = catalog.newTableOps(testTable).current().metadataFileLocation(); + TableOperations ops = catalog.newTableOps(testTable); + ops.io().deleteFile(metadataFileLocation); + Assert.assertTrue(catalog.dropTable(testTable)); + assertThatThrownBy(() -> catalog.loadTable(testTable)) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining("Table does not exist:"); + } + @Test public void testRenameTable() { TableIdentifier from = TableIdentifier.of("db", "tbl1"); diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java index 62c70438e6..20c7746f98 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java @@ -49,6 +49,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -157,11 +158,16 @@ public class HiveCatalog extends BaseMetastoreCatalog implements SupportsNamespa String database = identifier.namespace().level(0); TableOperations ops = newTableOps(identifier); - TableMetadata lastMetadata; - if (purge && ops.current() != null) { - lastMetadata = ops.current(); - } else { - lastMetadata = null; + TableMetadata lastMetadata = null; + if (purge) { + try { + lastMetadata = ops.current(); + } catch (NotFoundException e) { + LOG.warn( + "Failed to load table metadata for table: {}, continuing drop without purge", + identifier, + e); + } } try { diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java index 46ccb571a5..24487612bf 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java @@ -41,6 +41,7 @@ import org.apache.iceberg.SortOrder; import org.apache.iceberg.SortOrderParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableProperties; import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdateSchema; @@ -50,6 +51,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -74,6 +76,7 @@ import static org.apache.iceberg.TableProperties.DEFAULT_PARTITION_SPEC; import static org.apache.iceberg.TableProperties.DEFAULT_SORT_ORDER; import static org.apache.iceberg.expressions.Expressions.bucket; import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -440,6 +443,21 @@ public class TestHiveCatalog extends HiveMetastoreTest { }); } + @Test + public void testDropTableWithoutMetadataFile() { + TableIdentifier identifier = TableIdentifier.of(DB_NAME, "tbl"); + Schema tableSchema = + new Schema(Types.StructType.of(required(1, "id", Types.LongType.get())).fields()); + catalog.createTable(identifier, tableSchema); + String metadataFileLocation = catalog.newTableOps(identifier).current().metadataFileLocation(); + TableOperations ops = catalog.newTableOps(identifier); + ops.io().deleteFile(metadataFileLocation); + Assert.assertTrue(catalog.dropTable(identifier)); + assertThatThrownBy(() -> catalog.loadTable(identifier)) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining("Table does not exist:"); + } + @Test public void testTableName() { Schema schema = new Schema(
