This is an automated email from the ASF dual-hosted git repository.

dengzhhu653 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new d00655d64de HIVE-30194: HiveMetaTool drops silently without clear 
error message (#6887)
d00655d64de is described below

commit d00655d64de27864f9e32adf17d8e28b2243ff59
Author: dengzh <[email protected]>
AuthorDate: Wed Nov 4 07:41:19 2026 +0800

    HIVE-30194: HiveMetaTool drops silently without clear error message (#6887)
---
 .../hive/metastore/tools/ColumnDeduplicator.java   | 124 +++++++++++++++------
 .../hive/metastore/tools/MetaToolObjectStore.java  |   4 +-
 .../metastore/tools/metatool/HiveMetaTool.java     |   1 +
 .../tools/metatool/HiveMetaToolCommandLine.java    |  26 ++++-
 .../tools/metatool/MetaToolTaskDedupColumns.java   |  37 ++++--
 .../metastore/TestHMSColumnDescriptorReuse.java    | 118 +++++++++++++++++++-
 .../metatool/TestHiveMetaToolCommandLine.java      |  23 ++++
 7 files changed, 287 insertions(+), 46 deletions(-)

diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
index 16cb5f29136..27e31a3f361 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/ColumnDeduplicator.java
@@ -20,6 +20,7 @@
 package org.apache.hadoop.hive.metastore.tools;
 
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.hadoop.hive.common.TableName;
 import org.apache.hadoop.hive.metastore.Batchable;
 import org.apache.hadoop.hive.metastore.Deadline;
 import org.apache.hadoop.hive.metastore.RawStore;
@@ -33,6 +34,7 @@
 import org.apache.hadoop.hive.metastore.model.MPartition;
 import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
 import org.apache.hadoop.hive.metastore.model.MTable;
+import javax.jdo.JDOObjectNotFoundException;
 import javax.jdo.PersistenceManager;
 import javax.jdo.Query;
 import java.util.ArrayList;
@@ -56,54 +58,75 @@
  * the same CD, reducing metadata bloat that can accumulate during replication.
  */
 public final class ColumnDeduplicator {
+  public static final String COLUMN_DEDUPLICATOR_MARKER = 
"__metatool_column_deduplication__";
   private final RawStore store;
   private final PersistenceManager pm;
   private final AtomicReference<String> progress;
   private final boolean isDryRun;
   private final boolean isVerbose;
+  private final boolean clearMarker;
 
   public ColumnDeduplicator(RawStoreBundle bundle, AtomicReference<String> 
progress,
-      boolean isDryRun, boolean isVerbose) {
+      boolean isDryRun, boolean isVerbose, boolean clearMarker) {
     this.store = bundle.getBaseStore();
     this.pm = bundle.getPersistentManager();
     this.progress = progress;
     this.isDryRun = isDryRun;
     this.isVerbose = isVerbose;
+    this.clearMarker = clearMarker;
   }
 
   MetaToolObjectStore.DedupColumnsResult run(String catalogFilter, String 
dbFilter, String tableFilter) {
-    List<TableInfo> tables = findPartitionedTables(catalogFilter, dbFilter, 
tableFilter);
+    List<Long> tables = findPartitionedTables(catalogFilter, dbFilter, 
tableFilter);
     MetaToolObjectStore.DedupColumnsResult result = new 
MetaToolObjectStore.DedupColumnsResult(tables.size());
 
     long start = System.currentTimeMillis();
     for (int i = 0; i < tables.size() && result.getException() == null; i++) {
       boolean committed = false;
-      TableInfo table = tables.get(i);
+      long tableId = tables.get(i);
       PostCommitValidation validation = null;
+      TableInfo table = null;
       store.openTransaction();
       try {
         Deadline.checkTimeout();
-        validation = deduplicateTable(table, result);
+        table = getTable(tableId);
+        // This target table might be dropped or deduplicated, ignore
+        // If this is the stage to clean up the marker, ignore
+        if (table != null) {
+          validation = deduplicateCols(table, result);
+          if (validation == null) {
+            // Nothing has been changed that requires a post-commit 
validation, mark the table right away.
+            // Otherwise the marker is added once the validation succeeds.
+            markProcessed(tableId);
+          }
+        }
         committed = store.commitTransaction();
       } catch (Exception ex) {
+        System.err.println("Failed to deduplicate the column in this table: " 
+ (table == null ? tableId : table));
         result.catchException(ex);
       } finally {
         if (!committed) {
           store.rollbackTransaction();
-          if (result.getException() == null) {
-            result.catchException(
-                new MetaException("Failed to apply column descriptor 
de-duplication updates for table " + table));
-          }
         }
       }
 
+      boolean success = false;
+      store.openTransaction();
       try {
         if (committed && validation != null) {
           validateTableAfterDedup(table, validation.schemaSnapshots());
           result.incrementTablesValidated();
+          // Only a table that passed the validation can be skipped by later 
runs
+          markProcessed(tableId);
         }
+        success = store.commitTransaction();
       } catch (Exception e) {
-        result.catchException(new IllegalStateException("Failed to validate 
the column deduplication for this table: " + table, e));
+        result.catchException(new IllegalStateException(
+            "Failed to validate the column deduplication or mark this table: " 
+ table, e));
+      } finally {
+        if (!success) {
+          store.rollbackTransaction();
+        }
       }
 
       if (progress != null) {
@@ -119,7 +142,56 @@ MetaToolObjectStore.DedupColumnsResult run(String 
catalogFilter, String dbFilter
     return result;
   }
 
-  private PostCommitValidation deduplicateTable(TableInfo table, 
MetaToolObjectStore.DedupColumnsResult result)
+  public TableInfo getTable(long tableId) {
+    final MTable table;
+    try {
+      table = pm.getObjectById(MTable.class, new MTable.PK(tableId));
+    } catch (JDOObjectNotFoundException e) {
+      // This table might have been dropped since the beginning
+      return null;
+    }
+    Map<String, String> params = table.getParameters() == null ?
+        new HashMap<>() : table.getParameters();
+    if (clearMarker) {
+      // Clean-up only mode: never add the marker or deduplicate anything, and 
don't touch anything in a dry run
+      if (!isDryRun && params.containsKey(COLUMN_DEDUPLICATOR_MARKER)) {
+        params.remove(COLUMN_DEDUPLICATOR_MARKER);
+        table.setParameters(params);
+      }
+      return null;
+    }
+    if (params.containsKey(COLUMN_DEDUPLICATOR_MARKER)) {
+      // This table has been processed before
+      return null;
+    }
+
+    pm.retrieve(table);
+    pm.retrieve(table.getSd().getCD());
+    pm.retrieve(table.getDatabase());
+    return new TableInfo(tableId, table.getSd().getCD().getId(),
+        new TableName(table.getDatabase().getCatalogName(), 
table.getDatabase().getName(), table.getTableName()));
+  }
+
+  /**
+   * Marks the table as processed, so later runs skip it until the marker is 
cleared. Must be called within a
+   * transaction. It is a no-op in a dry run.
+   */
+  private void markProcessed(long tableId) {
+    if (isDryRun || clearMarker) {
+      return;
+    }
+    try {
+      MTable table = pm.getObjectById(MTable.class, new MTable.PK(tableId));
+      Map<String, String> params = table.getParameters() == null ?
+          new HashMap<>() : table.getParameters();
+      params.put(COLUMN_DEDUPLICATOR_MARKER, 
String.valueOf(System.currentTimeMillis() / 1000));
+      table.setParameters(params);
+    } catch (JDOObjectNotFoundException e) {
+      // This table might have been dropped in the meantime, nothing to mark
+    }
+  }
+
+  private PostCommitValidation deduplicateCols(TableInfo table, 
MetaToolObjectStore.DedupColumnsResult result)
       throws MetaException {
     List<PartitionSdInfo> partitionSds = 
loadPartitionStorageDescriptors(table.tableId);
     if (partitionSds.isEmpty()) {
@@ -153,8 +225,8 @@ private PostCommitValidation deduplicateTable(TableInfo 
table, MetaToolObjectSto
       if (isVerbose) {
         PartitionSdInfo partSd = update.getKey();
         long newCdId = update.getValue();
-        result.addDetail(String.format("table %s.%s.%s: partition %d CD %d -> 
%d",
-            table.catalogName, table.dbName, table.tableName, partSd.partId, 
partSd.cdId, newCdId));
+        result.addDetail(String.format("table %s: partition %d CD %d -> %d",
+           table.tableName, partSd.partId, partSd.cdId, newCdId));
       }
     }
     if (isDryRun) {
@@ -213,7 +285,7 @@ private long pickCanonicalCdId(Set<Long> group, long 
tableCdId, List<PartitionSd
         .orElse(group.iterator().next());
   }
 
-  private List<TableInfo> findPartitionedTables(String catalogFilter, String 
dbFilter, String tableFilter) {
+  private List<Long> findPartitionedTables(String catalogFilter, String 
dbFilter, String tableFilter) {
     StringBuilder filter = new StringBuilder();
     List<String> parameterVals = new ArrayList<>();
     if (!isEmpty(catalogFilter)) {
@@ -229,24 +301,12 @@ private List<TableInfo> findPartitionedTables(String 
catalogFilter, String dbFil
     Query query = !filter.isEmpty() ?
         pm.newQuery(MPartition.class, filter.toString()) :
         pm.newQuery(MPartition.class);
-    query.setResult("DISTINCT this.table");
+    query.setResult("DISTINCT this.table.id");
     boolean success = false;
-    List<TableInfo> tables = new ArrayList<>();
+    List<Long> tables = new ArrayList<>();
     store.openTransaction();
     try {
-      List<MTable> mTables = (List<MTable>) 
query.executeWithArray(parameterVals.toArray(new String[0]));
-      pm.retrieveAll(mTables);
-      for (MTable mTable : mTables) {
-        pm.retrieve(mTable.getDatabase());
-        pm.retrieve(mTable.getSd());
-        pm.retrieve(mTable.getSd().getCD());
-        tables.add(new TableInfo(
-            mTable.getId(),
-            mTable.getSd().getCD().getId(),
-            mTable.getDatabase().getCatalogName(),
-            mTable.getDatabase().getName(),
-            mTable.getTableName()));
-      }
+      tables.addAll((List<Long>) 
query.executeWithArray(parameterVals.toArray(new String[0])));
       success = store.commitTransaction();
     } finally {
       query.closeAll();
@@ -273,7 +333,7 @@ private List<PartitionSdInfo> 
loadPartitionStorageDescriptors(long tableId) thro
         if (sd == null) {
           continue;
         }
-        pm.retrieve(sd);
+        pm.retrieve(sd.getCD());
         partitionSds.add(new PartitionSdInfo(partId, sd, sd.getCD().getId()));
       }
     } finally {
@@ -418,11 +478,11 @@ private void removeConstraintsForCd(MColumnDescriptor cd) 
{
     }
   }
 
-  public record TableInfo(long tableId, long tableCdId, String catalogName, 
String dbName, String tableName) {
+  public record TableInfo(long tableId, long tableCdId, TableName tableName) {
     @Override
     public String toString() {
-        return catalogName + "." + dbName + "." + tableName;
-      }
+      return tableName.toString();
+    }
   }
 
   private record PartitionSdInfo(long partId, MStorageDescriptor sd, long 
cdId) {
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
index 5c5a6124270..4322c5f80b5 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/MetaToolObjectStore.java
@@ -1035,8 +1035,8 @@ public Exception getException() {
   }
 
   public DedupColumnsResult dedupColumns(String catalogFilter, String 
dbFilter, String tableFilter,
-      AtomicReference<String> progress, boolean isDryRun, boolean isVerbose) {
-    return new ColumnDeduplicator(this.createRawStoreBundle(), progress, 
isDryRun, isVerbose)
+      AtomicReference<String> progress, boolean isDryRun, boolean isVerbose, 
boolean clearMarker) {
+    return new ColumnDeduplicator(this.createRawStoreBundle(), progress, 
isDryRun, isVerbose, clearMarker)
         .run(catalogFilter, dbFilter, tableFilter);
   }
 }
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
index 2ab17c67edc..6e615628354 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaTool.java
@@ -81,6 +81,7 @@ public static void main(String[] args) {
       execute(args);
     } catch (Exception e) {
       status = -1;
+      e.printStackTrace(System.err);
       LOGGER.error("Exception occured", e);
     } finally {
       ExitUtil.terminate(status);
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
index 5bec7e3bd97..5d80e97d45e 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/HiveMetaToolCommandLine.java
@@ -93,7 +93,10 @@ class HiveMetaToolCommandLine {
       .hasOptionalArgs(3)
       .withDescription("De-duplicate column descriptors for partitioned tables 
to reduce metadata bloat. "
           + "Optional catalog, database, and table filters support * 
wildcards. Use -dryRun to preview changes, "
-          + "-verbose to print per-partition details, and -timeout to set a 
timeout in seconds. "
+          + "-verbose to print per-partition details, -clearMarker to drop the 
marker that has been added in previous "
+          + "runs (cannot be combined with -dryRun), and -timeout to set a 
timeout in seconds. "
+          + "Live runs mark each successfully processed table with the table 
property "
+          + "__metatool_column_deduplication__, and marked tables are skipped 
on re-run until -clearMarker is used. "
           + "Live runs validate committed state (partition column schemas 
unchanged, canonical CD IDs applied) "
           + "and print a post-commit validation summary.")
       .create("dedupColumns");
@@ -102,6 +105,11 @@ class HiveMetaToolCommandLine {
       .withDescription("Print per-partition de-duplication details. Valid only 
with -dedupColumns.")
       .create("verbose");
 
+  private static final Option CLEAR_MARKER = OptionBuilder
+      .withDescription("Clear the column de-duplication table marker made by 
previous runs, no de-duplication "
+          + "is performed. Valid only with -dedupColumns, and not with 
-dryRun.")
+      .create("clearMarker");
+
   @SuppressWarnings("static-access")
   private static final Option TIMEOUT = OptionBuilder
       .withArgName("timeout-seconds")
@@ -153,6 +161,7 @@ class HiveMetaToolCommandLine {
     OPTIONS.addOption(METADATA_SUMMARY);
     OPTIONS.addOption(DEDUP_COLUMNS);
     OPTIONS.addOption(VERBOSE);
+    OPTIONS.addOption(CLEAR_MARKER);
     OPTIONS.addOption(TIMEOUT);
   }
 
@@ -163,6 +172,7 @@ class HiveMetaToolCommandLine {
   private String[] diffExtTblLocsParams;
   private boolean dryRun;
   private boolean verbose;
+  private boolean clearMarker;
   private Long dedupColumnsTimeoutSeconds;
   private String serdePropKey;
   private String tablePropKey;
@@ -204,6 +214,7 @@ private void parseCommandLine(String[] args) throws 
ParseException {
     diffExtTblLocsParams = cl.getOptionValues(DIFF_EXT_TBL_LOCS.getOpt());
     dryRun = cl.hasOption(DRY_RUN.getOpt());
     verbose = cl.hasOption(VERBOSE.getOpt());
+    clearMarker = cl.hasOption(CLEAR_MARKER.getOpt());
     if (cl.hasOption(TIMEOUT.getOpt())) {
       dedupColumnsTimeoutSeconds = 
Long.parseLong(cl.getOptionValue(TIMEOUT.getOpt()));
       if (dedupColumnsTimeoutSeconds <= 0) {
@@ -260,6 +271,14 @@ private void parseCommandLine(String[] args) throws 
ParseException {
       throw new IllegalArgumentException("-timeout may be used only for the 
-dedupColumns command");
     }
 
+    if (clearMarker && !isDedupColumns()) {
+      throw new IllegalArgumentException("-clearMarker may be used only for 
the -dedupColumns command");
+    }
+
+    if (clearMarker && dryRun) {
+      throw new IllegalArgumentException("-clearMarker cannot be used together 
with -dryRun");
+    }
+
     if (metadataSummaryParams != null && metadataSummaryParams.length < 1) {
       throw new IllegalArgumentException("HiveMetaTool:metadataSummary takes 
in 1 required arguments but was passed " +
               metadataSummaryParams.length + " arguments");
@@ -288,6 +307,7 @@ private void printArguments() {
         "\tdiffExtTblLocs: " + Arrays.toString(diffExtTblLocsParams) + "\n" +
         "\tdryRun        : " + dryRun + "\n" +
         "\tverbose       : " + verbose + "\n" +
+        "\tclearMarker   : " + clearMarker + "\n" +
         "\ttimeout       : " + dedupColumnsTimeoutSeconds + "\n" +
         "\tserdePropKey  : " + serdePropKey + "\n" +
         "\ttablePropKey  : " + tablePropKey + "\n" +
@@ -339,6 +359,10 @@ boolean isVerbose() {
     return verbose;
   }
 
+  boolean isClearMarker() {
+    return clearMarker;
+  }
+
   Long getDedupColumnsTimeoutSeconds() {
     return dedupColumnsTimeoutSeconds;
   }
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
index f86c7b9480c..caf8824b756 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/metatool/MetaToolTaskDedupColumns.java
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hive.metastore.DeadlineException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.tools.ColumnDeduplicator;
 import org.apache.hadoop.hive.metastore.tools.MetaToolObjectStore;
 
 class MetaToolTaskDedupColumns extends MetaToolTask {
@@ -41,6 +42,7 @@ void execute() {
     
     Thread daemon = null;
     MetaToolObjectStore.DedupColumnsResult result = null;
+    Exception failure = null;
     AtomicReference<String> progress = new AtomicReference<>();
     AtomicBoolean stopped = new AtomicBoolean(false);
     long timeoutMs = getCl().getDedupColumnsTimeoutSeconds() != null ?
@@ -69,10 +71,11 @@ void execute() {
         daemon.setDaemon(true);
         daemon.start();
       }
-      result = getObjectStore().dedupColumns(catalogFilter, dbFilter, 
tableFilter, progress, isDryRun, isVerbose);
-    } catch (MetaException ex) {
-      // Deadline.startTimer throws MetaException, this shouldn't happen as 
the Deadline has been registered.
-      System.err.println(ex.getMessage());
+      result = getObjectStore().dedupColumns(catalogFilter, dbFilter, 
tableFilter, progress, isDryRun, isVerbose,
+          getCl().isClearMarker());
+    } catch (Exception ex) {
+      // Remember the failure and rethrow it once the clean-up is done, so 
that the tool exits with a failure status
+      failure = ex;
     } finally {
       if (daemon != null) {
         stopped.set(true);
@@ -86,12 +89,23 @@ void execute() {
       }
     }
 
+    if (failure != null) {
+      throw new IllegalStateException("HiveMetaTool: failed to run 
-dedupColumns", failure);
+    }
+
     if (result != null) {
       printSummary(result, isDryRun, isVerbose);
       if (result.getException() != null) {
-        if (result.getException().getCause() instanceof DeadlineException 
deadline) {
-          // Just print the timeout message
-          System.err.println(deadline.getMessage());
+        DeadlineException deadline = findDeadlineCause(result.getException());
+        if (deadline != null) {
+          long timeoutSeconds = Math.max(1, timeoutMs / 1000);
+          System.err.println("HiveMetaTool -dedupColumns timed out after " + 
timeoutSeconds
+              + " second(s). The configured limit is -timeout <seconds> or "
+              + MetastoreConf.ConfVars.DEDUP_COLUMNS_TIMEOUT.getVarname() + " 
(default in minutes).");
+          System.err.println("Increase the limit with -timeout <seconds> or 
set "
+              + MetastoreConf.ConfVars.DEDUP_COLUMNS_TIMEOUT.getVarname()
+              + " to a larger value (minutes), then re-run. Tables already 
processed can be skipped if marked "
+              + "with " + ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER + ".");
           return;
         }
         throw new IllegalStateException("HiveMetaTool: failed to de-duplicate 
column descriptors for all tables",
@@ -100,6 +114,15 @@ void execute() {
     }
   }
 
+  private static DeadlineException findDeadlineCause(Throwable ex) {
+    for (Throwable t = ex; t != null; t = t.getCause()) {
+      if (t instanceof DeadlineException) {
+        return (DeadlineException) t;
+      }
+    }
+    return null;
+  }
+
   private void printSummary(MetaToolObjectStore.DedupColumnsResult result, 
boolean isDryRun, boolean isVerbose) {
     System.out.println(isDryRun ?
         "Dry run of -dedupColumns.." :
diff --git 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
index 0f63bda5f80..886574e2106 100644
--- 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
+++ 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHMSColumnDescriptorReuse.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.metastore;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.TableName;
 import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -37,6 +38,7 @@
 import org.apache.hadoop.hive.metastore.tools.ColumnDeduplicator;
 import org.apache.hadoop.hive.metastore.tools.MetaToolObjectStore;
 import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -54,7 +56,9 @@
 
 import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
@@ -98,6 +102,14 @@ public void setUp() throws Exception {
 
   @After
   public void tearDown() {
+    if (objectStore != null) {
+      objectStore.shutdown();
+    }
+    // The PersistenceManagerFactory and its connection pools are static and 
survive across tests,
+    // as the JDBC url is the same. Close them before dropping the in-memory 
Derby database, otherwise
+    // the pool silently re-creates an empty database (create=true) that races 
with the next schema init.
+    PersistenceManagerProvider.closePmfIfNeeded(false, true);
+    PersistenceManagerProvider.closePmfIfNeeded(true, true);
     db.after();
   }
 
@@ -198,7 +210,7 @@ public void testDeduplicateColumnDescriptorsTool() throws 
Exception {
     Deadline.startTimer("testDeduplicateColumnDescriptorsTool");
     try {
       MetaToolObjectStore.DedupColumnsResult result = 
metaToolStore.dedupColumns(null, "default", "person", progress,
-          false, false);
+          false, false, false);
       assertEquals(0, result.getTablesWithDuplicates());
       assertEquals(0, result.getTablesValidated());
 
@@ -211,12 +223,31 @@ public void testDeduplicateColumnDescriptorsTool() throws 
Exception {
       objectStore.addPartition(newPart(tbl2, "England"));
       cdsBeforeDedup = countColumnDescriptors();
       assertTrue(cdsBeforeDedup > 2);
+      Table table = objectStore.getTable("hive", "default", "person");
+      
assertFalse(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
 
-      result = metaToolStore.dedupColumns(null, "default", "person", progress, 
false, false);
+      result = metaToolStore.dedupColumns(null, "default", "person", progress, 
false, false, false);
       assertTrue(result.getStorageDescriptorsUpdated() > 0);
       assertEquals(1, result.getTablesValidated());
       assertEquals(2, countColumnDescriptors());
       assertNotNull(progress.get());
+      table = objectStore.getTable("hive", "default", "person");
+      
assertTrue(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+      result = metaToolStore.dedupColumns(null, "default", "person", progress, 
false, false, false);
+      assertEquals(0, result.getStorageDescriptorsUpdated());
+      assertEquals(0, result.getTablesValidated());
+
+      result = metaToolStore.dedupColumns(null, "default", "person", progress, 
false, false, true);
+      assertEquals(0, result.getStorageDescriptorsUpdated());
+      table = objectStore.getTable("hive", "default", "person");
+      // the marker has been cleaned
+      
assertFalse(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+      result = metaToolStore.dedupColumns(null, "default", "person", progress, 
false, false, true);
+      assertEquals(0, result.getStorageDescriptorsUpdated());
+      table = objectStore.getTable("hive", "default", "person");
+      
assertFalse(table.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
 
       Deadline.registerIfNot(30 * 1000);
       Deadline.startTimer("testDeduplicateColumnDescriptorsTool");
@@ -236,6 +267,85 @@ public void testDeduplicateColumnDescriptorsTool() throws 
Exception {
     }
   }
 
+  @Test
+  public void testColDupGetTable() throws Exception {
+    FieldSchema id = new FieldSchema("id", ColumnType.STRING_TYPE_NAME, "");
+    FieldSchema fname = new FieldSchema("fname", ColumnType.STRING_TYPE_NAME, 
"");
+    FieldSchema country = new FieldSchema("country", 
ColumnType.STRING_TYPE_NAME, "");
+    Table tbl = newTable(Arrays.asList(id, fname), 
Collections.singletonList(country));
+    objectStore.createTable(tbl);
+    long tableId = objectStore.getTable("hive", "default", "person").getId();
+
+    MetaToolObjectStore metaToolStore = new MetaToolObjectStore();
+    metaToolStore.setConf(conf);
+    ColumnDeduplicator deduplicator =
+        new ColumnDeduplicator(metaToolStore.createRawStoreBundle(), null, 
false, false, false);
+    assertNull(getTableInTrx(metaToolStore, deduplicator, 9_999_999_999L));
+    ColumnDeduplicator.TableInfo tableInfo = getTableInTrx(metaToolStore, 
deduplicator, tableId);
+    assertNotNull(tableInfo);
+    assertEquals("person", tableInfo.tableName().getTable());
+    // Looking up the table doesn't mark it, the marker is only added once the 
table has been processed
+    tbl = objectStore.getTable("hive", "default", "person");
+    
assertFalse(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+    Table newTable = newTable(Arrays.asList(id, fname), 
Collections.singletonList(country));
+    newTable.setTableName("person1");
+    objectStore.createTable(newTable);
+    long person1TableId = objectStore.getTable("hive", "default", 
"person1").getId();
+
+    // Process the table "person", it has no duplicates but should be marked, 
and be skipped afterwards
+    objectStore.addPartition(newPart(tbl, "US"));
+    Deadline.registerIfNot(30 * 1000);
+    Deadline.startTimer("testColDupGetTable");
+    try {
+      MetaToolObjectStore.DedupColumnsResult result =
+          metaToolStore.dedupColumns(null, "default", "person", new 
AtomicReference<>(), false, false, false);
+      assertNull(result.getException());
+      assertEquals(1, result.getTablesScanned());
+    } finally {
+      Deadline.stopTimer();
+    }
+    tbl = objectStore.getTable("hive", "default", "person");
+    
assertTrue(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+    assertNull(getTableInTrx(metaToolStore, deduplicator, tableId));
+
+    // A dry run of the clean-up shouldn't remove the marker
+    ColumnDeduplicator dryRunCleaner =
+        new ColumnDeduplicator(metaToolStore.createRawStoreBundle(), null, 
true, false, true);
+    assertNull(getTableInTrx(metaToolStore, dryRunCleaner, tableId));
+    tbl = objectStore.getTable("hive", "default", "person");
+    
assertTrue(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+    // Marker should be cleared
+    deduplicator =
+        new ColumnDeduplicator(metaToolStore.createRawStoreBundle(), null, 
false, false, true);
+    assertNull(getTableInTrx(metaToolStore, deduplicator, tableId));
+    tbl = objectStore.getTable("hive", "default", "person");
+    
assertFalse(tbl.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+
+    // A table without the deduplication marker shouldn't be affected
+    newTable = objectStore.getTable("hive", "default", "person1");
+    
assertFalse(newTable.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+    assertNull(getTableInTrx(metaToolStore, deduplicator, person1TableId));
+    newTable = objectStore.getTable("hive", "default", "person1");
+    
assertFalse(newTable.getParameters().containsKey(ColumnDeduplicator.COLUMN_DEDUPLICATOR_MARKER));
+  }
+
+  private static ColumnDeduplicator.TableInfo 
getTableInTrx(MetaToolObjectStore store,
+      ColumnDeduplicator deduplicator, long tableId) {
+    boolean committed = false;
+    store.openTransaction();
+    try {
+      ColumnDeduplicator.TableInfo tableInfo = deduplicator.getTable(tableId);
+      committed = store.commitTransaction();
+      return tableInfo;
+    } finally {
+      if (!committed) {
+        store.rollbackTransaction();
+      }
+    }
+  }
+
   @Test
   public void testValidateTableAfterDedupReloadsPartitionById() throws 
Exception {
     MetastoreConf.setBoolVar(conf, 
MetastoreConf.ConfVars.PARTITION_REUSE_COLUMN_DESCRIPTORS, false);
@@ -332,8 +442,8 @@ private long lookupPartitionCdId(long partId) throws 
Exception {
   private void validate(List<ColumnDeduplicator.PartitionSchemaSnapshot> 
snapshots)
       throws Exception {
     ColumnDeduplicator deduplicator =
-        new ColumnDeduplicator(objectStore.createRawStoreBundle(), null, 
false, false);
-    ColumnDeduplicator.TableInfo tableInfo = new 
ColumnDeduplicator.TableInfo(0L, 0L, "hive", "default", "person");
+        new ColumnDeduplicator(objectStore.createRawStoreBundle(), null, 
false, false, false);
+    ColumnDeduplicator.TableInfo tableInfo = new 
ColumnDeduplicator.TableInfo(0L, 0L, new TableName("hive", "default", 
"person"));
     deduplicator.validateTableAfterDedup(tableInfo, snapshots);
   }
 
diff --git 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
index f28b177eb02..36e12b7a334 100644
--- 
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
+++ 
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaToolCommandLine.java
@@ -154,6 +154,21 @@ public void testParseDedupColumns() throws ParseException {
 
     cl = new HiveMetaToolCommandLine(new String[] {"-dedupColumns", 
"-timeout", "3600"});
     assertEquals(Long.valueOf(3600L), cl.getDedupColumnsTimeoutSeconds());
+
+    cl = new HiveMetaToolCommandLine(new String[] {"-dedupColumns", "hive", 
"default", "person", "-clearMarker"});
+    assertTrue(cl.isClearMarker());
+
+    cl = new HiveMetaToolCommandLine(new String[] {"-dedupColumns", 
"-clearMarker"});
+    assertTrue(cl.isClearMarker());
+    assertEquals(0, cl.getDedupColumnsParams().length);
+  }
+
+  @Test
+  public void testClearMarkerWithDryRunNotAllowed() throws ParseException {
+    exception.expect(IllegalArgumentException.class);
+    exception.expectMessage("-clearMarker cannot be used together with 
-dryRun");
+
+    new HiveMetaToolCommandLine(new String[] {"-dedupColumns", "-clearMarker", 
"-dryRun"});
   }
 
   @Test
@@ -172,6 +187,14 @@ public void testInvalidTimeout() throws ParseException {
     new HiveMetaToolCommandLine(new String[] {"-dedupColumns", "-timeout", 
"0"});
   }
 
+  @Test
+  public void testClearMarkerNotAllowed() throws ParseException {
+    exception.expect(IllegalArgumentException.class);
+    exception.expectMessage("-clearMarker may be used only for the 
-dedupColumns command");
+
+    new HiveMetaToolCommandLine(new String[] {"-listFSRoot", "-clearMarker"});
+  }
+
   @Test
   public void testVerboseNotAllowed() throws ParseException {
     exception.expect(IllegalArgumentException.class);

Reply via email to