This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 26cdc7a018 Core: Support deleting tables without metadata files (#5510)
26cdc7a018 is described below
commit 26cdc7a0181504718257a37bff4f34af77b0aad3
Author: ChenLiang <[email protected]>
AuthorDate: Fri Sep 2 03:48:32 2022 +0800
Core: Support deleting tables without metadata files (#5510)
---
.../apache/iceberg/aws/dynamodb/DynamoDbCatalog.java | 13 ++++++++++++-
.../java/org/apache/iceberg/aws/glue/GlueCatalog.java | 13 ++++++++++++-
.../main/java/org/apache/iceberg/jdbc/JdbcCatalog.java | 16 +++++++++++-----
.../java/org/apache/iceberg/jdbc/TestJdbcCatalog.java | 14 ++++++++++++++
.../main/java/org/apache/iceberg/hive/HiveCatalog.java | 16 +++++++++++-----
.../java/org/apache/iceberg/hive/TestHiveCatalog.java | 18 ++++++++++++++++++
6 files changed, 78 insertions(+), 12 deletions(-)
diff --git
a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
index accaad59dc..130fc921de 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
@@ -42,6 +42,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.io.FileIO;
@@ -371,7 +372,17 @@ public class DynamoDbCatalog extends BaseMetastoreCatalog
}
TableOperations ops = newTableOps(identifier);
- TableMetadata lastMetadata = ops.current();
+ TableMetadata lastMetadata = null;
+ if (purge) {
+ try {
+ lastMetadata = ops.current();
+ } catch (NotFoundException e) {
+ LOG.warn(
+ "Failed to load table metadata for table: {}, continuing drop
without purge",
+ identifier,
+ e);
+ }
+ }
dynamo.deleteItem(
DeleteItemRequest.builder()
.tableName(awsProperties.dynamoDbTableName())
diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
index 0f7e9ddff2..fe9315a524 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
@@ -45,6 +45,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.hadoop.Configurable;
import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.io.FileIO;
@@ -307,7 +308,17 @@ public class GlueCatalog extends BaseMetastoreCatalog
public boolean dropTable(TableIdentifier identifier, boolean purge) {
try {
TableOperations ops = newTableOps(identifier);
- TableMetadata lastMetadata = ops.current();
+ TableMetadata lastMetadata = null;
+ if (purge) {
+ try {
+ lastMetadata = ops.current();
+ } catch (NotFoundException e) {
+ LOG.warn(
+ "Failed to load table metadata for table: {}, continuing drop
without purge",
+ identifier,
+ e);
+ }
+ }
glue.deleteTable(
DeleteTableRequest.builder()
.catalogId(awsProperties.glueCatalogId())
diff --git a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
index 18159e40b4..24409bc3e5 100644
--- a/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
+++ b/core/src/main/java/org/apache/iceberg/jdbc/JdbcCatalog.java
@@ -48,6 +48,7 @@ import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.hadoop.Configurable;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
@@ -165,11 +166,16 @@ public class JdbcCatalog extends BaseMetastoreCatalog
@Override
public boolean dropTable(TableIdentifier identifier, boolean purge) {
TableOperations ops = newTableOps(identifier);
- TableMetadata lastMetadata;
- if (purge && ops.current() != null) {
- lastMetadata = ops.current();
- } else {
- lastMetadata = null;
+ TableMetadata lastMetadata = null;
+ if (purge) {
+ try {
+ lastMetadata = ops.current();
+ } catch (NotFoundException e) {
+ LOG.warn(
+ "Failed to load table metadata for table: {}, continuing drop
without purge",
+ identifier,
+ e);
+ }
}
int deletedRecords =
diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
index 4ca8cfd5af..68016af8ed 100644
--- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.jdbc;
import static org.apache.iceberg.NullOrder.NULLS_FIRST;
import static org.apache.iceberg.SortDirection.ASC;
import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.io.File;
import java.io.IOException;
@@ -387,6 +388,19 @@ public class TestJdbcCatalog extends
CatalogTests<JdbcCatalog> {
Assert.assertFalse(catalog.dropTable(TableIdentifier.of("db",
"tbl-not-exists")));
}
+ @Test
+ public void testDropTableWithoutMetadataFile() {
+ TableIdentifier testTable = TableIdentifier.of("db", "ns1", "ns2", "tbl");
+ catalog.createTable(testTable, SCHEMA, PartitionSpec.unpartitioned());
+ String metadataFileLocation =
catalog.newTableOps(testTable).current().metadataFileLocation();
+ TableOperations ops = catalog.newTableOps(testTable);
+ ops.io().deleteFile(metadataFileLocation);
+ Assert.assertTrue(catalog.dropTable(testTable));
+ assertThatThrownBy(() -> catalog.loadTable(testTable))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessageContaining("Table does not exist:");
+ }
+
@Test
public void testRenameTable() {
TableIdentifier from = TableIdentifier.of("db", "tbl1");
diff --git
a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
index aaaa6de99a..1a46580778 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
@@ -45,6 +45,7 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.hadoop.HadoopFileIO;
import org.apache.iceberg.io.FileIO;
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
@@ -168,11 +169,16 @@ public class HiveCatalog extends BaseMetastoreCatalog
implements SupportsNamespa
String database = identifier.namespace().level(0);
TableOperations ops = newTableOps(identifier);
- TableMetadata lastMetadata;
- if (purge && ops.current() != null) {
- lastMetadata = ops.current();
- } else {
- lastMetadata = null;
+ TableMetadata lastMetadata = null;
+ if (purge) {
+ try {
+ lastMetadata = ops.current();
+ } catch (NotFoundException e) {
+ LOG.warn(
+ "Failed to load table metadata for table: {}, continuing drop
without purge",
+ identifier,
+ e);
+ }
}
try {
diff --git
a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
index 704f4ea714..231f6144e5 100644
--- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
+++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveCatalog.java
@@ -28,6 +28,7 @@ import static
org.apache.iceberg.TableProperties.DEFAULT_PARTITION_SPEC;
import static org.apache.iceberg.TableProperties.DEFAULT_SORT_ORDER;
import static org.apache.iceberg.expressions.Expressions.bucket;
import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -53,6 +54,7 @@ import org.apache.iceberg.SortOrder;
import org.apache.iceberg.SortOrderParser;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.Transaction;
import org.apache.iceberg.UpdateSchema;
@@ -62,6 +64,7 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
@@ -458,6 +461,21 @@ public class TestHiveCatalog extends HiveMetastoreTest {
});
}
+ @Test
+ public void testDropTableWithoutMetadataFile() {
+ TableIdentifier identifier = TableIdentifier.of(DB_NAME, "tbl");
+ Schema tableSchema =
+ new Schema(Types.StructType.of(required(1, "id",
Types.LongType.get())).fields());
+ catalog.createTable(identifier, tableSchema);
+ String metadataFileLocation =
catalog.newTableOps(identifier).current().metadataFileLocation();
+ TableOperations ops = catalog.newTableOps(identifier);
+ ops.io().deleteFile(metadataFileLocation);
+ Assert.assertTrue(catalog.dropTable(identifier));
+ assertThatThrownBy(() -> catalog.loadTable(identifier))
+ .isInstanceOf(NoSuchTableException.class)
+ .hasMessageContaining("Table does not exist:");
+ }
+
@Test
public void testTableName() {
Schema schema =
