This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 6e1f1ccf2ac HIVE-28366: Iceberg: Concurrent Insert and IOW produce 
incorrect result (Denys Kuzmenko, reviewed by Ayush Saxena)
6e1f1ccf2ac is described below

commit 6e1f1ccf2aca58af4be06d6ceea1aa2b877a5d08
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Tue Sep 17 09:24:06 2024 +0200

    HIVE-28366: Iceberg: Concurrent Insert and IOW produce incorrect result 
(Denys Kuzmenko, reviewed by Ayush Saxena)
    
    Closes #5386
---
 .../mr/hive/HiveIcebergOutputCommitter.java        | 50 +++++++++------
 .../iceberg/mr/hive/HiveIcebergStorageHandler.java | 32 ++++++++-
 .../iceberg/mr/hive/TestConflictingDataFiles.java  | 75 +++++++++++++++++++++-
 .../org/apache/iceberg/mr/hive/TestHiveShell.java  |  2 +
 .../org/apache/hadoop/hive/ql/TestAcidOnTez.java   | 23 ++++---
 .../ql/txn/compactor/TestCrudCompactorOnTez.java   | 12 ++--
 .../java/org/apache/hadoop/hive/ql/Compiler.java   |  2 +-
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java  | 49 ++++++--------
 .../apache/hadoop/hive/ql/DriverTxnHandler.java    | 33 +++++++---
 .../java/org/apache/hadoop/hive/ql/Executor.java   |  2 +-
 .../java/org/apache/hadoop/hive/ql/QueryPlan.java  | 20 +++---
 .../ql/ddl/database/drop/DropDatabaseAnalyzer.java |  2 +-
 .../storage/compact/AlterTableCompactAnalyzer.java |  5 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java    | 11 +---
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java       | 46 ++++++-------
 .../hive/ql/metadata/HiveStorageHandler.java       | 31 ++++-----
 .../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java |  6 +-
 .../hive/ql/parse/ExecuteStatementAnalyzer.java    |  2 +-
 .../hive/ql/parse/ExportSemanticAnalyzer.java      |  2 +-
 .../org/apache/hadoop/hive/ql/TestTxnCommands.java |  8 +--
 .../apache/hadoop/hive/ql/TestTxnCommands2.java    | 30 ++++-----
 .../apache/hadoop/hive/ql/TestTxnCommands3.java    |  8 +--
 .../org/apache/hadoop/hive/ql/TestTxnExIm.java     |  6 +-
 .../org/apache/hadoop/hive/ql/TestTxnLoadData.java | 50 +++++++--------
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java    | 52 +++++++--------
 .../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java  |  1 -
 .../hadoop/hive/metastore/DatabaseProduct.java     | 11 ++++
 .../hadoop/hive/metastore/MetaStoreDirectSql.java  |  2 +-
 .../apache/hive/streaming/TransactionBatch.java    | 16 +++--
 29 files changed, 357 insertions(+), 232 deletions(-)

diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
index 189661b4736..f22e66c0623 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
@@ -508,7 +508,7 @@ public class HiveIcebergOutputCommitter extends 
OutputCommitter {
 
         commitCompaction(table, snapshotId, startTime, filesForCommit, 
partitionPath);
       } else {
-        commitOverwrite(table, branchName, startTime, filesForCommit);
+        commitOverwrite(table, branchName, snapshotId, startTime, 
filesForCommit);
       }
     }
   }
@@ -526,6 +526,7 @@ public class HiveIcebergOutputCommitter extends 
OutputCommitter {
    * Creates and commits an Iceberg change with the provided data and delete 
files.
    * If there are no delete files then an Iceberg 'append' is created, 
otherwise Iceberg 'overwrite' is created.
    * @param table      The table we are changing
+   * @param snapshotId The snapshot id of the table to use for validation
    * @param startTime  The start time of the commit - used only for logging
    * @param results    The object containing the new files we would like to 
add to the table
    * @param filterExpr Filter expression for conflict detection filter
@@ -563,6 +564,7 @@ public class HiveIcebergOutputCommitter extends 
OutputCommitter {
       RowDelta write = table.newRowDelta();
       results.dataFiles().forEach(write::addRows);
       results.deleteFiles().forEach(write::addDeletes);
+
       if (StringUtils.isNotEmpty(branchName)) {
         write.toBranch(HiveUtils.getTableSnapshotRef(branchName));
       }
@@ -589,11 +591,11 @@ public class HiveIcebergOutputCommitter extends 
OutputCommitter {
    * Creates and commits an Iceberg compaction change with the provided data 
files.
    * Either full table or a selected partition contents is replaced with 
compacted files.
    *
-   * @param table             The table we are changing
-   * @param snapshotId        The snapshot id of the table to use for 
validation
-   * @param startTime         The start time of the commit - used only for 
logging
-   * @param results           The object containing the new files
-   * @param partitionPath     The path of the compacted partition
+   * @param table         The table we are changing
+   * @param snapshotId    The snapshot id of the table to use for validation
+   * @param startTime     The start time of the commit - used only for logging
+   * @param results       The object containing the new files
+   * @param partitionPath The path of the compacted partition
    */
   private void commitCompaction(Table table, Long snapshotId, long startTime, 
FilesForCommit results,
       String partitionPath) {
@@ -601,46 +603,53 @@ public class HiveIcebergOutputCommitter extends 
OutputCommitter {
     List<DeleteFile> existingDeleteFiles = 
IcebergTableUtil.getDeleteFiles(table, partitionPath);
 
     RewriteFiles rewriteFiles = table.newRewrite();
-    rewriteFiles.validateFromSnapshot(getSnapshotId(table, null));
-    if (snapshotId != null) {
-      rewriteFiles.validateFromSnapshot(snapshotId);
-    }
-
     existingDataFiles.forEach(rewriteFiles::deleteFile);
     existingDeleteFiles.forEach(rewriteFiles::deleteFile);
     results.dataFiles().forEach(rewriteFiles::addFile);
 
+    if (snapshotId != null) {
+      rewriteFiles.validateFromSnapshot(snapshotId);
+    }
     rewriteFiles.commit();
     LOG.info("Compaction commit took {} ms for table: {} partition: {} with {} 
file(s)",
-        System.currentTimeMillis() - startTime, table, partitionPath == null ? 
"N/A" : partitionPath,
+        System.currentTimeMillis() - startTime, table, 
StringUtils.defaultString(partitionPath, "N/A"),
         results.dataFiles().size());
   }
 
   /**
    * Creates and commits an Iceberg insert overwrite change with the provided 
data files.
-   * For unpartitioned tables the table content is replaced with the new data 
files. If not data files are provided
-   * then the unpartitioned table is truncated.
-   * For partitioned tables the relevant partitions are replaced with the new 
data files. If no data files are provided
-   * then the unpartitioned table remains unchanged.
+   * For unpartitioned tables the table content is replaced with the new data 
files. Table is truncated
+   * if no data files are provided.
+   * For partitioned tables the relevant partitions are replaced with the new 
data files. Table remains unchanged
+   * unless data files are provided.
    *
-   * @param table                   The table we are changing
-   * @param startTime               The start time of the commit - used only 
for logging
-   * @param results                 The object containing the new files
+   * @param table      The table we are changing
+   * @param snapshotId The snapshot id of the table to use for validation
+   * @param startTime  The start time of the commit - used only for logging
+   * @param results    The object containing the new files
    */
-  private void commitOverwrite(Table table, String branchName, long startTime, 
FilesForCommit results) {
+  private void commitOverwrite(Table table, String branchName, Long 
snapshotId, long startTime,
+      FilesForCommit results) {
     Preconditions.checkArgument(results.deleteFiles().isEmpty(), "Can not 
handle deletes with overwrite");
     if (!results.dataFiles().isEmpty()) {
       ReplacePartitions overwrite = table.newReplacePartitions();
       results.dataFiles().forEach(overwrite::addFile);
+
       if (StringUtils.isNotEmpty(branchName)) {
         overwrite.toBranch(HiveUtils.getTableSnapshotRef(branchName));
       }
+      if (snapshotId != null) {
+        overwrite.validateFromSnapshot(snapshotId);
+      }
+      overwrite.validateNoConflictingDeletes();
+      overwrite.validateNoConflictingData();
       overwrite.commit();
       LOG.info("Overwrite commit took {} ms for table: {} with {} file(s)", 
System.currentTimeMillis() - startTime,
           table, results.dataFiles().size());
     } else if (table.spec().isUnpartitioned()) {
       DeleteFiles deleteFiles = table.newDelete();
       deleteFiles.deleteFromRowFilter(Expressions.alwaysTrue());
+
       if (StringUtils.isNotEmpty(branchName)) {
         deleteFiles.toBranch(HiveUtils.getTableSnapshotRef(branchName));
       }
@@ -881,7 +890,6 @@ public class HiveIcebergOutputCommitter extends 
OutputCommitter {
                 LOG.info("Cleaning job for jobID: {}, table: {}", 
jobContext.getJobID(), output);
 
                 Table table = output.getKey();
-                FileSystem fileSystem = new 
Path(table.location()).getFileSystem(jobConf);
                 String jobLocation = generateJobLocation(table.location(), 
jobConf, jobContext.getJobID());
                 // list jobLocation to get number of forCommit files
                 // we do this because map/reduce num in jobConf is unreliable
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index a78ea2069f4..4b7f87f0206 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -164,10 +164,12 @@ import org.apache.iceberg.SortField;
 import org.apache.iceberg.SortOrder;
 import org.apache.iceberg.StatisticsFile;
 import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.TableScan;
 import org.apache.iceberg.actions.DeleteOrphanFiles;
 import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
 import org.apache.iceberg.expressions.Evaluator;
 import org.apache.iceberg.expressions.Expression;
 import org.apache.iceberg.expressions.ExpressionUtil;
@@ -175,6 +177,7 @@ import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.expressions.Projections;
 import org.apache.iceberg.expressions.ResidualEvaluator;
 import org.apache.iceberg.expressions.StrictMetricsEvaluator;
+import org.apache.iceberg.hadoop.ConfigProperties;
 import org.apache.iceberg.hadoop.HadoopConfigurable;
 import org.apache.iceberg.hive.HiveSchemaUtil;
 import org.apache.iceberg.io.CloseableIterable;
@@ -640,12 +643,15 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
   }
 
   /**
-   * No need for exclusive locks when writing, since Iceberg tables use 
optimistic concurrency when writing
-   * and only lock the table during the commit operation.
+   * Iceberg's optimistic concurrency control fails to provide means for IOW 
and Insert operations isolation.
+   * Use `hive.txn.ext.locking.enabled` config to create Hive locks in order 
to guarantee data consistency.
    */
   @Override
   public LockType getLockType(WriteEntity writeEntity) {
-    return LockType.SHARED_READ;
+    if (WriteEntity.WriteType.INSERT_OVERWRITE == writeEntity.getWriteType()) {
+      return LockType.EXCL_WRITE;
+    }
+    return LockType.SHARED_WRITE;
   }
 
   @Override
@@ -1443,6 +1449,26 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
     map.remove("columns.comments");
   }
 
+  @Override
+  public void validateCurrentSnapshot(TableDesc tableDesc) {
+    if (conf.getBoolean(ConfigProperties.LOCK_HIVE_ENABLED, 
TableProperties.HIVE_LOCK_ENABLED_DEFAULT) ||
+        !HiveConf.getBoolVar(conf, ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED)) {
+      return;
+    }
+    Table table = IcebergTableUtil.getTable(conf, tableDesc.getProperties());
+    if (table.currentSnapshot() != null || table instanceof BaseTable) {
+      TableMetadata currentMetadata = ((BaseTable) 
table).operations().current();
+      if (currentMetadata.propertyAsBoolean(TableProperties.HIVE_LOCK_ENABLED, 
false)) {
+        return;
+      }
+      TableMetadata newMetadata = ((BaseTable) table).operations().refresh();
+      ValidationException.check(
+          Objects.equals(newMetadata.metadataFileLocation(), 
currentMetadata.metadataFileLocation()),
+          "Table snapshot is outdated: %s", tableDesc.getTableName()
+      );
+    }
+  }
+
   /**
    * Recursively replaces the ExprNodeDynamicListDesc nodes by a dummy 
ExprNodeConstantDesc so we can test if we can
    * convert the predicate to an Iceberg predicate when pruning the partitions 
later. Also collects the column names
diff --git 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
index b107041105d..a07d2d5203f 100644
--- 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
+++ 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
@@ -28,12 +28,15 @@ import java.util.concurrent.Executors;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.iceberg.FileFormat;
 import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.data.Record;
 import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
 import org.apache.iceberg.hive.HiveTableOperations;
 import org.apache.iceberg.mr.TestHelper;
 import org.apache.iceberg.relocated.com.google.common.base.Throwables;
+import org.apache.iceberg.types.Types;
 import org.apache.iceberg.util.Tasks;
 import org.junit.After;
 import org.junit.Assert;
@@ -43,6 +46,7 @@ import org.junit.Test;
 import org.mockito.MockedStatic;
 
 import static 
org.apache.iceberg.mr.hive.HiveIcebergStorageHandlerTestUtils.init;
+import static org.apache.iceberg.types.Types.NestedField.required;
 import static org.mockito.ArgumentMatchers.anyMap;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.CALLS_REAL_METHODS;
@@ -65,8 +69,9 @@ public class TestConflictingDataFiles extends 
HiveIcebergStorageHandlerWithEngin
       tableOps.when(() -> method.invoke(null, anyMap(), eq(true)))
           .thenAnswer(invocation -> null);
       // create and insert an initial batch of records
-      testTables.createTable(shell, "customers", 
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, fileFormat,
-          HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2, 2, 
Collections.emptyMap(), storageHandlerStub);
+      testTables.createTable(shell, "customers", 
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec,
+          fileFormat, 
HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2, 2, 
Collections.emptyMap(),
+          storageHandlerStub);
     }
     // insert one more batch so that we have multiple data files within the 
same partition
     
shell.executeStatement(testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_1,
@@ -259,6 +264,72 @@ public class TestConflictingDataFiles extends 
HiveIcebergStorageHandlerWithEngin
         .build();
     HiveIcebergTestUtils.validateData(expected,
         
HiveIcebergTestUtils.valueForRow(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
 objects), 0);
+  }
+
+  @Test
+  public void testConcurrentInsertAndInsertOverwrite() {
+    Assume.assumeTrue(formatVersion == 2);
+
+    Schema schema = new Schema(
+        required(1, "i", Types.IntegerType.get()),
+        required(2, "p", Types.IntegerType.get())
+    );
+    PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("i", 
10).build();
+
+    // create and insert an initial batch of records
+    testTables.createTable(shell, "ice_t", schema, spec, fileFormat,
+        TestHelper.RecordsBuilder.newInstance(schema)
+          .add(1, 1)
+          .add(2, 2)
+          .add(10, 10)
+          .add(20, 20)
+          .add(40, 40)
+          .add(30, 30)
+          .build(),
+        formatVersion);
+
+    String[] singleFilterQuery = new String[] { "INSERT INTO ice_t SELECT 
i*100, p*100 FROM ice_t",
+        "INSERT OVERWRITE TABLE ice_t SELECT i+1, p+1 FROM ice_t" };
+
+    Tasks.range(2).executeWith(Executors.newFixedThreadPool(2)).run(i -> {
+      if (i == 1) {
+        try {
+          Thread.sleep(100);
+        } catch (InterruptedException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      init(shell, testTables, temp, executionEngine);
+      HiveConf.setBoolVar(shell.getHiveConf(), 
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, isVectorized);
+      HiveConf.setVar(shell.getHiveConf(), 
HiveConf.ConfVars.HIVE_FETCH_TASK_CONVERSION, "none");
+
+      HiveConf.setBoolVar(shell.getHiveConf(), 
HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
+      shell.getHiveConf().setBoolean(ConfigProperties.LOCK_HIVE_ENABLED, 
false);
 
+      HiveConf.setVar(shell.getHiveConf(), 
HiveConf.ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+          RETRY_STRATEGIES_WITHOUT_WRITE_CONFLICT);
+      shell.executeStatement(singleFilterQuery[i]);
+      shell.closeSession();
+    });
+
+    List<Object[]> objects =
+        shell.executeStatement("SELECT * FROM ice_t");
+    Assert.assertEquals(12, objects.size());
+    List<Record> expected = TestHelper.RecordsBuilder.newInstance(schema)
+        .add(2, 2)
+        .add(3, 3)
+        .add(11, 11)
+        .add(21, 21)
+        .add(31, 31)
+        .add(41, 41)
+        .add(101, 101)
+        .add(201, 201)
+        .add(1001, 1001)
+        .add(2001, 2001)
+        .add(3001, 3001)
+        .add(4001, 4001)
+        .build();
+    HiveIcebergTestUtils.validateData(expected,
+        HiveIcebergTestUtils.valueForRow(schema, objects), 0);
   }
 }
diff --git 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
index 4898ce81338..43b22039baf 100644
--- 
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
+++ 
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.stream.Collectors;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 import 
org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -227,6 +228,7 @@ public class TestHiveShell {
     // set lifecycle hooks
     hiveConf.setVar(HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS, 
HiveIcebergQueryLifeTimeHook.class.getName());
 
+    MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.TRY_DIRECT_SQL, 
true);
     return hiveConf;
   }
 }
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index 0fbf6427338..a9ede32a591 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -92,7 +92,7 @@ public class TestAcidOnTez {
   public TestName testName = new TestName();
   private HiveConf hiveConf;
   private IDriver d;
-  private static enum Table {
+  private enum Table {
     ACIDTBL("acidTbl"),
     ACIDTBLPART("acidTblPart"),
     ACIDNOBUCKET("acidNoBucket"),
@@ -166,7 +166,8 @@ public class TestAcidOnTez {
   }
 
   private void dropTables() throws Exception {
-    for(Table t : Table.values()) {
+    runStatementOnDriver("drop table if exists T");
+    for (Table t : Table.values()) {
       runStatementOnDriver("drop table if exists " + t);
     }
   }
@@ -544,7 +545,7 @@ public class TestAcidOnTez {
     //check we have right delete delta files after minor compaction
     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
       (Table.ACIDNOBUCKET).toString().toLowerCase()), 
FileUtils.STAGING_DIR_PATH_FILTER);
-    String[] expectedDelDelta2 = { "delete_delta_0000002_0000002_0000", 
"delete_delta_0000003_0000003_0000", "delete_delta_0000001_0000003_v0000015"};
+    String[] expectedDelDelta2 = { "delete_delta_0000002_0000002_0000", 
"delete_delta_0000003_0000003_0000", "delete_delta_0000001_0000003_v0000014"};
     for(FileStatus stat : status) {
       for(int i = 0; i < expectedDelDelta2.length; i++) {
         if(expectedDelDelta2[i] != null && 
stat.getPath().toString().endsWith(expectedDelDelta2[i])) {
@@ -569,7 +570,7 @@ public class TestAcidOnTez {
     for(int i = 0; i < expected2.length; i++) {
       Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), 
rs.get(i).startsWith(expected2[i][0]));
       //everything is now in base/
-      Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), 
rs.get(i).endsWith("base_0000003_v0000019/bucket_00000"));
+      Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), 
rs.get(i).endsWith("base_0000003_v0000017/bucket_00000"));
     }
   }
   /**
@@ -582,7 +583,6 @@ public class TestAcidOnTez {
   @Test
   public void testInsertWithRemoveUnion() throws Exception {
     int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}};
-    runStatementOnDriver("drop table if exists T", hiveConf);
     runStatementOnDriver("create table T (a int, b int) stored as ORC  
TBLPROPERTIES ('transactional'='false')", hiveConf);
     /*
 ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505502329802/warehouse/t/.hive-staging_hive_2017-09-15_12-07-33_224_7717909516029836949-1/
@@ -630,15 +630,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/h
     }
     String[][] expected2 = {
        {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
-           "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+          "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
       {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
-          "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+          "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
       {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6",
-          "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+          "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
       {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t7\t8",
-          "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+          "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
       {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10",
-          "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"}
+          "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"}
     };
     Assert.assertEquals("Unexpected row count after major compact", 
expected2.length, rs.size());
     for(int i = 0; i < expected2.length; i++) {
@@ -655,7 +655,6 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/h
    */
   @Test
   public void testAcidInsertWithRemoveUnion() throws Exception {
-    runStatementOnDriver("drop table if exists T", hiveConf);
     runStatementOnDriver("create table T (a int, b int) stored as ORC  
TBLPROPERTIES ('transactional'='true')", hiveConf);
     /*On Tez, below (T is transactional), we get the following layout
 ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505500035574/warehouse/t/.hive-staging_hive_2017-09-15_11-28-33_960_9111484239090506828-1/
@@ -698,13 +697,13 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  
~/dev/hiverwgit/itests/h
       Assert.assertTrue("Actual line(file) " + i + " ac: " + rs.get(i), 
rs.get(i).endsWith(expected2[i][1]));
     }
   }
+  
   @Test
   public void testBucketedAcidInsertWithRemoveUnion() throws Exception {
     int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}};
     runStatementOnDriver("delete from " + Table.ACIDTBL, hiveConf);
     //make sure both buckets are not empty
     runStatementOnDriver("insert into " + Table.ACIDTBL + 
makeValuesClause(values), hiveConf);
-    runStatementOnDriver("drop table if exists T", hiveConf);
     /*
     With bucketed target table Union All is not removed
 
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index 148a7cb3225..b8d1827de0a 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -147,7 +147,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
       }
       aborted = true;
       Assert.assertEquals(LockException.class, e.getCause().getClass());
-      Assert.assertEquals( "Transaction manager has aborted the transaction 
txnid:20.  Reason: Aborting [txnid:20,20] due to a write conflict on 
default/rebalance_test committed by [txnid:19,20] d/u", e.getCauseMessage());
+      Assert.assertEquals( "Transaction manager has aborted the transaction 
txnid:19.  Reason: Aborting [txnid:19,19] due to a write conflict on 
default/rebalance_test committed by [txnid:18,19] d/u", e.getCauseMessage());
       // Delete the record, so the rest of the test can be the same in both 
cases
       executeStatementOnDriver("DELETE FROM " + tableName + " WHERE b = 12", 
driver);
     } finally {
@@ -197,7 +197,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
         },
     };
     verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
-        new String[] {"bucket_00000", "bucket_00001", "bucket_00002", 
"bucket_00003"}, "base_0000007_v0000019");
+        new String[] {"bucket_00000", "bucket_00001", "bucket_00002", 
"bucket_00003"}, "base_0000007_v0000018");
   }
 
   @Test
@@ -253,7 +253,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
         },
     };
     verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
-        new String[] {"bucket_00000", "bucket_00001", 
"bucket_00002","bucket_00003"}, "base_0000007_v0000019");
+        new String[] {"bucket_00000", "bucket_00001", 
"bucket_00002","bucket_00003"}, "base_0000007_v0000018");
   }
 
   @Test
@@ -307,7 +307,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
         },
     };
     verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
-        new String[] {"bucket_00000", "bucket_00001", 
"bucket_00002","bucket_00003"}, "base_0000007_v0000019");
+        new String[] {"bucket_00000", "bucket_00001", 
"bucket_00002","bucket_00003"}, "base_0000007_v0000018");
   }
 
   @Test
@@ -416,7 +416,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
         },
     };
     verifyRebalance(testDataProvider, tableName, "ds=tomorrow", 
expectedBuckets,
-        new String[] {"bucket_00000", "bucket_00001", "bucket_00002"}, 
"base_0000007_v0000015");
+        new String[] {"bucket_00000", "bucket_00001", "bucket_00002"}, 
"base_0000007_v0000014");
   }
 
   @Test
@@ -490,7 +490,7 @@ public class TestCrudCompactorOnTez extends 
CompactorOnTezTest {
         },
     };
     verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
-        new String[] {"bucket_00000", "bucket_00001", "bucket_00002", 
"bucket_00003"}, "base_0000007_v0000019");
+        new String[] {"bucket_00000", "bucket_00001", "bucket_00002", 
"bucket_00003"}, "base_0000007_v0000018");
   }
 
   private TestDataProvider prepareRebalanceTestData(String tableName) throws 
Exception {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
index fabe05dbee8..5bacbf3e406 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
@@ -113,7 +113,7 @@ public class Compiler {
       plan = createPlan(sem);
       
       if (HiveOperation.START_TRANSACTION == queryState.getHiveOperation()
-          || plan.hasAcidResources()) {
+          || plan.isRequiresOpenTransaction()) {
         openTxnAndGetValidTxnList();
       }
       verifyTxnState();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 2241baf3f5b..392a1ffbc00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
 import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.reexec.ReCompileException;
 import org.apache.hadoop.hive.ql.session.LineageState;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -173,6 +174,11 @@ public class Driver implements IDriver {
         
driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
       }
 
+      DriverUtils.checkInterrupted(driverState, driverContext, "at acquiring 
the lock.", null, null);
+
+      lockAndRespond();
+      validateCurrentSnapshot();
+
       // Reset the PerfLogger so that it doesn't retain any previous values.
       // Any value from compilation phase can be obtained through the map set 
in queryDisplay during compilation.
       PerfLogger perfLogger = SessionState.getPerfLogger(true);
@@ -180,17 +186,7 @@ public class Driver implements IDriver {
       // the reason that we set the txn manager for the cxt here is because 
each query has its own ctx object.
       // The txn mgr is shared across the same instance of Driver, which can 
run multiple queries.
       context.setHiveTxnManager(driverContext.getTxnManager());
-
-      DriverUtils.checkInterrupted(driverState, driverContext, "at acquiring 
the lock.", null, null);
-
-      lockAndRespond();
-
-      if (validateTxnList()) {
-        // the reason that we set the txn manager for the cxt here is because 
each query has its own ctx object.
-        // The txn mgr is shared across the same instance of Driver, which can 
run multiple queries.
-        context.setHiveTxnManager(driverContext.getTxnManager());
-        perfLogger = SessionState.getPerfLogger(true);
-      }
+      
       execute();
 
       FetchTask fetchTask = driverContext.getPlan().getFetchTask();
@@ -224,20 +220,21 @@ public class Driver implements IDriver {
   }
 
   /**
-   * @return If the txn manager should be set.
+   * Checks if the recorded snapshot is up-to-date
    */
-  private boolean validateTxnList() throws CommandProcessorException {
+  private void validateCurrentSnapshot() throws CommandProcessorException {
     int retryShapshotCount = 0;
+    
     int maxRetrySnapshotCount = HiveConf.getIntVar(driverContext.getConf(),
         HiveConf.ConfVars.HIVE_TXN_MAX_RETRYSNAPSHOT_COUNT);
-    boolean shouldSet = false;
-
     try {
       do {
         driverContext.setOutdatedTxn(false);
         // Inserts will not invalidate the snapshot, that could cause 
duplicates.
+        
         if (!driverTxnHandler.isValidTxnListState()) {
           LOG.info("Re-compiling after acquiring locks, attempt #" + 
retryShapshotCount);
+          HiveTxnManager txnMgr = driverContext.getTxnManager();
           // Snapshot was outdated when locks were acquired, hence regenerate 
context, txn list and retry.
           // TODO: Lock acquisition should be moved before analyze, this is a 
bit hackish.
           // Currently, we acquire a snapshot, compile the query with that 
snapshot, and then - acquire locks.
@@ -246,10 +243,10 @@ public class Driver implements IDriver {
           if (driverContext.isOutdatedTxn()) {
             // Later transaction invalidated the snapshot, a new transaction 
is required
             LOG.info("Snapshot is outdated, re-initiating transaction ...");
-            driverContext.getTxnManager().rollbackTxn();
+            txnMgr.rollbackTxn();
 
             String userFromUGI = DriverUtils.getUserFromUGI(driverContext);
-            driverContext.getTxnManager().openTxn(context, userFromUGI, 
driverContext.getTxnType());
+            txnMgr.openTxn(context, userFromUGI, driverContext.getTxnType());
             lockAndRespond();
           } else {
             // We need to clear the possibly cached writeIds for the prior 
transaction, so new writeIds
@@ -265,42 +262,36 @@ public class Driver implements IDriver {
             // The scan basically does last writer wins for a given row which 
is determined by
             // max(committingWriteId) for a given ROW__ID(originalWriteId, 
bucketId, rowId). So the
             // data add ends up being > than the data delete.
-            driverContext.getTxnManager().clearCaches();
+            txnMgr.clearCaches();
           }
           driverContext.getConf().unset(ValidTxnList.VALID_TXNS_KEY);
           driverContext.setRetrial(true);
 
-          if (driverContext.getPlan().hasAcidReadWrite()) {
-            compileInternal(context.getCmd(), true);
+          compileInternal(context.getCmd(), true);
+
+          if (driverContext.getPlan().hasAcidResourcesInQuery()) {
             driverTxnHandler.recordValidWriteIds();
             driverTxnHandler.setWriteIdForAcidFileSinks();
           }
           // Since we're reusing the compiled plan, we need to update its 
start time for current run
-          
driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
+          driverContext.getPlan().setQueryStartTime(
+              driverContext.getQueryDisplay().getQueryStartTime());
           driverContext.setRetrial(false);
         }
         // Re-check snapshot only in case we had to release locks and open a 
new transaction,
         // otherwise exclusive locks should protect output tables/partitions 
in snapshot from concurrent writes.
       } while (driverContext.isOutdatedTxn() && ++retryShapshotCount <= 
maxRetrySnapshotCount);
 
-      shouldSet = shouldSetTxnManager(retryShapshotCount, 
maxRetrySnapshotCount);
     } catch (LockException | SemanticException e) {
       DriverUtils.handleHiveException(driverContext, e, 13, null);
     }
 
-    return shouldSet;
-  }
-
-  private boolean shouldSetTxnManager(int retryShapshotCount, int 
maxRetrySnapshotCount)
-      throws CommandProcessorException {
     if (retryShapshotCount > maxRetrySnapshotCount) {
       // Throw exception
       HiveException e = new HiveException(
           "Operation could not be executed, " + 
SNAPSHOT_WAS_OUTDATED_WHEN_LOCKS_WERE_ACQUIRED + ".");
       DriverUtils.handleHiveException(driverContext, e, 14, null);
     }
-
-    return retryShapshotCount != 0;
   }
 
   private void setInitialStateForRun(boolean alreadyCompiled) throws 
CommandProcessorException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java 
b/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
index 05c96d9452c..5f10cd129ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
 import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
 import org.apache.hadoop.hive.ql.lockmgr.LockException;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.HiveTableName;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -64,6 +65,7 @@ import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
 import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
+import org.apache.hadoop.hive.ql.reexec.ReCompileException;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 import org.apache.hadoop.util.StringUtils;
@@ -218,6 +220,8 @@ class DriverTxnHandler {
     perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
 
     if (!driverContext.getTxnManager().isTxnOpen() && 
driverContext.getTxnManager().supportsAcid() 
+        && (driverContext.getPlan().isRequiresOpenTransaction() 
+          || !HiveConf.getBoolVar(driverContext.getConf(), 
HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED)) 
         && !SessionState.get().isCompaction()) {
       /* non acid txn managers don't support txns but fwd lock requests to 
lock managers
          acid txn manager requires all locks to be associated with a txn so if 
we end up here w/o an open txn
@@ -235,7 +239,7 @@ class DriverTxnHandler {
       if (!SessionState.get().isCompaction()) {
         acquireLocksInternal();
       }
-      if (driverContext.getPlan().hasAcidReadWrite() || hasAcidDdl) {
+      if (driverContext.getPlan().hasAcidResourcesInQuery() || hasAcidDdl) {
         recordValidWriteIds();
       }
     } catch (Exception e) {
@@ -336,8 +340,8 @@ class DriverTxnHandler {
     LOG.info("Operation {} obtained {} locks", 
driverContext.getPlan().getOperation(),
         ((locks == null) ? 0 : locks.size()));
     // This check is for controlling the correctness of the current state
-    if (driverContext.getTxnManager().recordSnapshot(driverContext.getPlan()) 
&&
-        !driverContext.isValidTxnListsGenerated()) {
+    if (driverContext.getTxnManager().recordSnapshot(driverContext.getPlan()) 
+        && !driverContext.isValidTxnListsGenerated()) {
       throw new IllegalStateException("Need to record valid WriteID list but 
there is no valid TxnID list (" +
           
JavaUtils.txnIdToString(driverContext.getTxnManager().getCurrentTxnId()) +
           ", queryId: " + driverContext.getPlan().getQueryId() + ")");
@@ -418,6 +422,18 @@ class DriverTxnHandler {
    * on the table over which the lock is required.
    */
   boolean isValidTxnListState() throws LockException {
+    try {
+      context.getLoadTableOutputMap().forEach(
+        (ltd, we) -> {
+          HiveStorageHandler handler = we.getTable().getStorageHandler();
+          if (handler != null) {
+            handler.validateCurrentSnapshot(ltd.getTable());
+          }
+        });
+    } catch (Exception e) {
+      LOG.warn(e.getMessage());
+      return false;
+    }
     // 1) Get valid txn list.
     String txnString = 
driverContext.getConf().get(ValidTxnList.VALID_TXNS_KEY);
     if (txnString == null) {
@@ -516,17 +532,18 @@ class DriverTxnHandler {
   void handleTransactionAfterExecution() throws CommandProcessorException {
     try {
       //since set autocommit starts an implicit txn, close it
-      if (driverContext.getTxnManager().isImplicitTransactionOpen(context) ||
-          driverContext.getPlan().getOperation() == HiveOperation.COMMIT) {
+      if (driverContext.getTxnManager().isImplicitTransactionOpen(context)
+          || driverContext.getPlan().getOperation() == HiveOperation.COMMIT) {
         endTransactionAndCleanup(true);
       } else if (driverContext.getPlan().getOperation() == 
HiveOperation.ROLLBACK) {
         endTransactionAndCleanup(false);
-      } else if (!driverContext.getTxnManager().isTxnOpen() &&
-          driverContext.getQueryState().getHiveOperation() == 
HiveOperation.REPLLOAD) {
+      } else if (!driverContext.getTxnManager().isTxnOpen()
+          && (driverContext.getQueryState().getHiveOperation() == 
HiveOperation.REPLLOAD 
+            || !SessionState.get().isCompaction())) {
         // repl load during migration, commits the explicit txn and start some 
internal txns. Call
         // releaseLocksAndCommitOrRollback to do the clean up.
         endTransactionAndCleanup(false);
-      }
+      } 
       // if none of the above is true, then txn (if there is one started) is 
not finished
     } catch (LockException e) {
       DriverUtils.handleHiveException(driverContext, e, 12, null);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Executor.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
index 9d0ce22da32..708e3870efa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
@@ -219,7 +219,7 @@ public class Executor {
     if (driverContext.getCacheUsage().getStatus() == 
CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
         driverContext.getPlan().getFetchTask() != null) {
       ValidTxnWriteIdList txnWriteIdList = null;
-      if (driverContext.getPlan().hasAcidReadWrite()) {
+      if (driverContext.getPlan().hasAcidResourcesInQuery()) {
         txnWriteIdList = 
AcidUtils.getValidTxnWriteIdList(driverContext.getConf());
       }
       // The results of this query execution might be cacheable.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java 
b/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
index 92d395bfefe..6c47d3bc169 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
@@ -116,8 +116,8 @@ public class QueryPlan implements Serializable {
 
   private transient Long queryStartTime;
   private final HiveOperation operation;
-  private final boolean hasAcidReadWrite;
-  private final boolean hasAcidResources;
+  private final boolean acidResourcesInQuery;
+  private final boolean requiresOpenTransaction;
   private final Set<FileSinkDesc> acidSinks; // Note: both full-ACID and 
insert-only sinks.
   private final WriteEntity acidAnalyzeTable;
   private final DDLDescWithWriteId acidDdlDesc;
@@ -132,8 +132,8 @@ public class QueryPlan implements Serializable {
   protected QueryPlan(HiveOperation command) {
     this.reducerTimeStatsPerJobList = new ArrayList<>();
     this.operation = command;
-    this.hasAcidReadWrite = false;
-    this.hasAcidResources = false;
+    this.acidResourcesInQuery = false;
+    this.requiresOpenTransaction = false;
     this.acidSinks = Collections.emptySet();
     this.acidDdlDesc = null;
     this.acidAnalyzeTable = null;
@@ -165,8 +165,8 @@ public class QueryPlan implements Serializable {
     this.autoCommitValue = sem.getAutoCommitValue();
     this.resultSchema = resultSchema;
     // TODO: all this ACID stuff should be in some sub-object
-    this.hasAcidReadWrite = sem.hasAcidReadWrite();
-    this.hasAcidResources = sem.hasAcidResources();
+    this.acidResourcesInQuery = sem.hasTransactionalInQuery();
+    this.requiresOpenTransaction = sem.isRequiresOpenTransaction();
     this.acidSinks = sem.getAcidFileSinks();
     this.acidDdlDesc = sem.getAcidDdlDesc();
     this.acidAnalyzeTable = sem.getAcidAnalyzeTable();
@@ -177,12 +177,12 @@ public class QueryPlan implements Serializable {
   /**
    * @return true if any acid resources are read/written
    */
-  public boolean hasAcidReadWrite() {
-    return hasAcidReadWrite;
+  public boolean hasAcidResourcesInQuery() {
+    return acidResourcesInQuery;
   }
 
-  public boolean hasAcidResources() {
-    return hasAcidResources;
+  public boolean isRequiresOpenTransaction() {
+    return requiresOpenTransaction;
   }
 
   public WriteEntity getAcidAnalyzeTable() {
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
index 09d94383dd9..6d01c2e88b3 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
@@ -116,7 +116,7 @@ public class DropDatabaseAnalyzer extends 
BaseSemanticAnalyzer {
   }
 
   @Override
-  public boolean hasAcidResources() {
+  public boolean isRequiresOpenTransaction() {
     // check DB tags once supported (i.e. ICEBERG_ONLY, ACID_ONLY, 
EXTERNAL_ONLY)
     return true;
   }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
index 309d356209d..69e0a77a2d5 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.metastore.api.CompactionType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId;
 import org.apache.hadoop.hive.ql.ddl.DDLWork;
 import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
 import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
@@ -120,7 +119,7 @@ public class AlterTableCompactAnalyzer extends 
AbstractAlterTableAnalyzer {
   }
   
   @Override
-  protected void setAcidDdlDesc(DDLDescWithWriteId desc) {
-    // doesn't need an open txn
+  public boolean isRequiresOpenTransaction() {
+    return false; // doesn't need an open txn
   }
 }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index ff125a30740..cbd99aa5994 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -3057,15 +3057,10 @@ public class AcidUtils {
           boolean isExclMergeInsert = 
conf.getBoolVar(ConfVars.TXN_MERGE_INSERT_X_LOCK) && isMerge;
           compBuilder.setSharedRead();
 
-          if (sharedWrite) {
+          if (sharedWrite || !isExclMergeInsert && isLocklessReadsEnabled) {
             compBuilder.setSharedWrite();
-          } else {
-            if (isExclMergeInsert) {
-              compBuilder.setExclWrite();
-
-            } else if (isLocklessReadsEnabled) {
-              compBuilder.setSharedWrite();
-            }
+          } else if (isExclMergeInsert) {
+            compBuilder.setExclWrite();
           }
           if (isExclMergeInsert) {
             compBuilder.setOperationType(DataOperationType.UPDATE);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java 
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 3a7eb675256..d9593e514e4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -251,7 +251,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
     contention.*/
     init();
     getLockManager();
-    if(isTxnOpen()) {
+    if (isTxnOpen()) {
       throw new LockException("Transaction already opened. " + 
JavaUtils.txnIdToString(txnId));
     }
     try {
@@ -295,7 +295,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
       acquireLocksWithHeartbeatDelay(plan, ctx, username, 0);
     }
     catch(LockException e) {
-      if(e.getCause() instanceof TxnAbortedException) {
+      if (e.getCause() instanceof TxnAbortedException) {
         resetTxnInfo();
       }
       throw e;
@@ -313,7 +313,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
 
   private void markExplicitTransaction(QueryPlan queryPlan) throws 
LockException {
     isExplicitTransaction = true;
-    if(++startTransactionCount > 1) {
+    if (++startTransactionCount > 1) {
       throw new LockException(null, ErrorMsg.OP_NOT_ALLOWED_IN_TXN, 
queryPlan.getOperationName(),
         JavaUtils.txnIdToString(getCurrentTxnId()), queryPlan.getQueryId());
     }
@@ -328,11 +328,11 @@ public final class DbTxnManager extends 
HiveTxnManagerImpl {
    * @throws LockException
    */
   private void verifyState(QueryPlan queryPlan) throws LockException {
-    if(!isTxnOpen() && queryPlan.hasAcidResources()) {
+    if (!isTxnOpen() && queryPlan.isRequiresOpenTransaction()) {
       throw new LockException("No transaction context for operation: " + 
queryPlan.getOperationName() +
         " for " + getQueryIdWaterMark(queryPlan));
     }
-    if(queryPlan.getOperation() == null) {
+    if (queryPlan.getOperation() == null) {
       throw new IllegalStateException("Unknown HiveOperation(null) for " + 
getQueryIdWaterMark(queryPlan));
     }
     numStatements++;
@@ -342,13 +342,13 @@ public final class DbTxnManager extends 
HiveTxnManagerImpl {
         break;
       case COMMIT:
       case ROLLBACK:
-        if(!isExplicitTransaction) {
+        if (!isExplicitTransaction) {
           throw new LockException(null, 
ErrorMsg.OP_NOT_ALLOWED_IN_IMPLICIT_TXN, queryPlan.getOperationName());
         }
         break;
       default:
-        if(!queryPlan.getOperation().isAllowedInTransaction() && 
isExplicitTransaction) {
-          if(allowOperationInATransaction(queryPlan)) {
+        if (!queryPlan.getOperation().isAllowedInTransaction() && 
isExplicitTransaction) {
+          if (allowOperationInATransaction(queryPlan)) {
             break;
           }
           //look at queryPlan.outputs(WriteEntity.t - that's the table)
@@ -374,9 +374,9 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
   private boolean allowOperationInATransaction(QueryPlan queryPlan) {
     //Acid and MM tables support Load Data with transactional semantics.  This 
will allow Load Data
     //in a txn assuming we can determine the target is a suitable table type.
-    if(queryPlan.getOperation() == HiveOperation.LOAD && 
queryPlan.getOutputs() != null && queryPlan.getOutputs().size() == 1) {
+    if (queryPlan.getOperation() == HiveOperation.LOAD && 
queryPlan.getOutputs() != null && queryPlan.getOutputs().size() == 1) {
       WriteEntity writeEntity = queryPlan.getOutputs().iterator().next();
-      if(AcidUtils.isTransactionalTable(writeEntity.getTable())) {
+      if (AcidUtils.isTransactionalTable(writeEntity.getTable())) {
         switch (writeEntity.getWriteType()) {
           case INSERT:
             //allow operation in a txn
@@ -645,7 +645,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
   @Override
   public void heartbeat() throws LockException {
     List<HiveLock> locks;
-    if(isTxnOpen()) {
+    if (isTxnOpen()) {
       // Create one dummy lock so we can go through the loop below, though we 
only
       //really need txnId
       locks = Collections.singletonList(new DbLockManager.DbHiveLock(0L));
@@ -653,7 +653,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
     else {
       locks = lockMgr.getLocks(false, false);
     }
-    if(LOG.isInfoEnabled()) {
+    if (LOG.isInfoEnabled()) {
       StringBuilder sb = new StringBuilder("Sending heartbeat for ")
         .append(JavaUtils.txnIdToString(txnId)).append(" and");
       for(HiveLock lock : locks) {
@@ -661,7 +661,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
       }
       LOG.info(sb.toString());
     }
-    if(!isTxnOpen() && locks.isEmpty()) {
+    if (!isTxnOpen() && locks.isEmpty()) {
       // No locks, no txn, we outta here.
       LOG.debug("No need to send heartbeat as there is no transaction and no 
locks.");
       return;
@@ -716,7 +716,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
 
   private ScheduledFuture<?> startHeartbeat(long initialDelay, long 
heartbeatInterval, Runnable heartbeater) {
     // For negative testing purpose..
-    if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && 
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
+    if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && 
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
       initialDelay = 0;
     } else if (initialDelay == 0) {
       /*make initialDelay a random number in [0, 0.75*heartbeatInterval] so 
that if a lot
@@ -852,17 +852,19 @@ public final class DbTxnManager extends 
HiveTxnManagerImpl {
    */
   @Override
   public boolean recordSnapshot(QueryPlan queryPlan) {
-    assert isTxnOpen();
+    if (!isTxnOpen()) {
+      return false;
+    }
     assert numStatements > 0 : "was acquireLocks() called already?";
-    if(queryPlan.getOperation() == HiveOperation.START_TRANSACTION) {
+    if (queryPlan.getOperation() == HiveOperation.START_TRANSACTION) {
       //here if start of explicit txn
       assert isExplicitTransaction;
       assert numStatements == 1;
       return true;
     }
-    else if(!isExplicitTransaction) {
+    else if (!isExplicitTransaction) {
       assert numStatements == 1 : "numStatements=" + numStatements + " in 
implicit txn";
-      if (queryPlan.hasAcidReadWrite()) {
+      if (queryPlan.hasAcidResourcesInQuery()) {
         //1st and only stmt in implicit txn and uses acid resource
         return true;
       }
@@ -877,11 +879,11 @@ public final class DbTxnManager extends 
HiveTxnManagerImpl {
 
   @Override
   public boolean isImplicitTransactionOpen(Context ctx) {
-    if(!isTxnOpen()) {
+    if (!isTxnOpen()) {
       //some commands like "show databases" don't start implicit transactions
       return false;
     }
-    if(!isExplicitTransaction) {
+    if (!isExplicitTransaction) {
       if (ctx == null || !ctx.isExplainSkipExecution()) {
         assert numStatements == 1 : "numStatements=" + numStatements;
       }
@@ -915,7 +917,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
   }
 
   private synchronized static void initHeartbeatExecutorService(int 
corePoolSize) {
-      if(heartbeatExecutorService != null) {
+      if (heartbeatExecutorService != null) {
         return;
       }
       // The following code will be executed only once when the service is not 
initialized
@@ -1109,7 +1111,7 @@ public final class DbTxnManager extends 
HiveTxnManagerImpl {
     public void run() {
       try {
         // For negative testing purpose..
-        if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && 
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
+        if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) && 
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
           throw new 
LockException(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER.name() + 
"=true");
         }
         LOG.debug("Heartbeating...for currentUser: " + currentUser);
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 6ff715cdf44..2b05837a884 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -105,23 +105,23 @@ public interface HiveStorageHandler extends Configurable {
   /**
    * @return Class providing an implementation of {@link InputFormat}
    */
-  public Class<? extends InputFormat> getInputFormatClass();
+  Class<? extends InputFormat> getInputFormatClass();
 
   /**
    * @return Class providing an implementation of {@link OutputFormat}
    */
-  public Class<? extends OutputFormat> getOutputFormatClass();
+  Class<? extends OutputFormat> getOutputFormatClass();
 
   /**
    * @return Class providing an implementation of {@link AbstractSerDe}
    */
-  public Class<? extends AbstractSerDe> getSerDeClass();
+  Class<? extends AbstractSerDe> getSerDeClass();
 
   /**
    * @return metadata hook implementation, or null if this
    * storage handler does not need any metadata notifications
    */
-  public HiveMetaHook getMetaHook();
+  HiveMetaHook getMetaHook();
 
   /**
    * Returns the implementation specific authorization provider
@@ -129,8 +129,7 @@ public interface HiveStorageHandler extends Configurable {
    * @return authorization provider
    * @throws HiveException
    */
-  public HiveAuthorizationProvider getAuthorizationProvider()
-    throws HiveException;
+  HiveAuthorizationProvider getAuthorizationProvider() throws HiveException;
 
   /**
    * This method is called to allow the StorageHandlers the chance
@@ -149,14 +148,13 @@ public interface HiveStorageHandler extends Configurable {
    * @param jobProperties receives properties copied or transformed
    * from the table properties
    */
-  public abstract void configureInputJobProperties(TableDesc tableDesc,
-    Map<String, String> jobProperties);
+  void configureInputJobProperties(TableDesc tableDesc, Map<String, String> 
jobProperties);
 
   /**
    * This method is called to allow the StorageHandlers the chance to
    * populate secret keys into the job's credentials.
    */
-  public abstract void configureInputJobCredentials(TableDesc tableDesc, 
Map<String, String> secrets);
+  void configureInputJobCredentials(TableDesc tableDesc, Map<String, String> 
secrets);
 
   /**
    * This method is called to allow the StorageHandlers the chance
@@ -175,8 +173,7 @@ public interface HiveStorageHandler extends Configurable {
    * @param jobProperties receives properties copied or transformed
    * from the table properties
    */
-  public abstract void configureOutputJobProperties(TableDesc tableDesc,
-    Map<String, String> jobProperties);
+  void configureOutputJobProperties(TableDesc tableDesc, Map<String, String> 
jobProperties);
 
   /**
    * Deprecated use configureInputJobProperties/configureOutputJobProperties
@@ -191,9 +188,7 @@ public interface HiveStorageHandler extends Configurable {
    * from the table properties
    */
   @Deprecated
-  public void configureTableJobProperties(
-    TableDesc tableDesc,
-    Map<String, String> jobProperties);
+  void configureTableJobProperties(TableDesc tableDesc, Map<String, String> 
jobProperties);
 
   /**
    * Called just before submitting MapReduce job.
@@ -201,7 +196,7 @@ public interface HiveStorageHandler extends Configurable {
    * @param tableDesc descriptor for the table being accessed
    * @param jobConf jobConf for MapReduce job
    */
-  public void configureJobConf(TableDesc tableDesc, JobConf jobConf);
+  void configureJobConf(TableDesc tableDesc, JobConf jobConf);
 
   /**
    * Used to fetch runtime information about storage handler during DESCRIBE 
EXTENDED statement
@@ -210,8 +205,7 @@ public interface HiveStorageHandler extends Configurable {
    * @return StorageHandlerInfo containing runtime information about storage 
handler
    * OR `null` if the storage handler choose to not provide any runtime 
information.
    */
-  public default StorageHandlerInfo getStorageHandlerInfo(Table table) throws 
MetaException
-  {
+  default StorageHandlerInfo getStorageHandlerInfo(Table table) throws 
MetaException {
     return null;
   }
   
@@ -689,6 +683,9 @@ public interface HiveStorageHandler extends Configurable {
   default SnapshotContext 
getCurrentSnapshotContext(org.apache.hadoop.hive.ql.metadata.Table table) {
     return null;
   }
+  
+  default void validateCurrentSnapshot(TableDesc tableDesc) {
+  }
 
   /**
    * Return snapshot metadata of table snapshots which are newer than the 
specified.
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index fc87e9d736c..f528d67716a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -1571,12 +1571,12 @@ public abstract class BaseSemanticAnalyzer {
     return acidFileSinks;
   }
 
-  public boolean hasAcidReadWrite() {
+  public boolean hasTransactionalInQuery() {
     return transactionalInQuery;
   }
 
-  public boolean hasAcidResources() {
-    return hasAcidReadWrite() || getAcidDdlDesc() != null ||
+  public boolean isRequiresOpenTransaction() {
+    return hasTransactionalInQuery() || getAcidDdlDesc() != null ||
       Stream.of(getInputs(), getOutputs()).flatMap(Collection::stream)
         .filter(entity -> entity.getType() == Entity.Type.TABLE || 
entity.getType() == Entity.Type.PARTITION)
         .flatMap(entity -> {
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
index 89942658161..1428bea874b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
@@ -225,7 +225,7 @@ public class ExecuteStatementAnalyzer extends 
SemanticAnalyzer{
 
       this.queryProperties = cachedPlan.getQueryProperties();
       this.setAutoCommitValue(cachedPlan.getAutoCommitValue());
-      this.transactionalInQuery = cachedPlan.hasAcidReadWrite();
+      this.transactionalInQuery = cachedPlan.hasTransactionalInQuery();
       this.acidFileSinks.addAll(cachedPlan.getAcidFileSinks());
       this.initCtx(cachedPlan.getCtx());
       this.ctx.setCboInfo(cachedPlan.getCboInfo());
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
index c7ff003af17..70253374d99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
@@ -129,7 +129,7 @@ public class ExportSemanticAnalyzer extends 
BaseSemanticAnalyzer {
   }
   
   @Override
-  public boolean hasAcidReadWrite() {
+  public boolean hasTransactionalInQuery() {
     return isMmExport; // Full ACID export goes through UpdateDelete analyzer.
   }
 }
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 3feb2c8a0f5..2344b908ceb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -1446,10 +1446,10 @@ public class TestTxnCommands extends 
TxnCommandsBaseForTests {
     query = "select ROW__ID, a, b" + (isVectorized ? "" : ", 
INPUT__FILE__NAME") + " from "
         + Table.NONACIDORCTBL + " order by ROW__ID";
     String[][] expected2 = new String[][] {
-        {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":0}\t1\t2", 
"nonacidorctbl/base_10000001_v0000010/bucket_00001"},
-        {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":1}\t0\t12", 
"nonacidorctbl/base_10000001_v0000010/bucket_00001"},
-        {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":2}\t1\t5", 
"nonacidorctbl/base_10000001_v0000010/bucket_00001"},
-        {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t1\t17", 
"nonacidorctbl/base_10000001_v0000010/bucket_00001"}
+        {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":0}\t1\t2", 
"nonacidorctbl/base_10000001_v0000009/bucket_00001"},
+        {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":1}\t0\t12", 
"nonacidorctbl/base_10000001_v0000009/bucket_00001"},
+        {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":2}\t1\t5", 
"nonacidorctbl/base_10000001_v0000009/bucket_00001"},
+        {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t1\t17", 
"nonacidorctbl/base_10000001_v0000009/bucket_00001"}
     };
     checkResult(expected2, query, isVectorized, "after major compact", LOG);
     //make sure they are the same before and after compaction
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 7bed9d2bc81..3386404cbf0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -727,9 +727,9 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
     boolean sawNewBase = false;
     for (int i = 0; i < status.length; i++) {
       if (status[i].getPath().getName().matches("base_.*")) {
-        //should be base_-9223372036854775808_v0000023 but 23 is a txn id not 
write id so it makes
+        //should be base_-9223372036854775808_v0000008 but 8 is a txn id not 
write id so it makes
         //the tests fragile
-        
Assert.assertTrue(status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000009"));
+        
Assert.assertTrue(status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000008"));
         sawNewBase = true;
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
         Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -781,7 +781,7 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
           Assert.assertEquals("bucket_00001_0", 
buckets[0].getPath().getName());
         }
       } else if (status[i].getPath().getName().matches("base_.*")) {
-        Assert.assertTrue("base_-9223372036854775808", 
status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000009"));//_v0000009
+        Assert.assertTrue("base_-9223372036854775808", 
status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000008"));//_v0000008
         sawNewBase = true;
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
         Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -804,7 +804,7 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
     // 5. Perform another major compaction
     runStatementOnDriver("alter table "+ Table.NONACIDORCTBL + " compact 
'MAJOR'");
     runWorker(hiveConf);
-    // There should be 1 new base directory: base_0000001
+    // There should be 1 new base directory: base_00000016
     // Original bucket files, delta directories, delete_delta directories and 
the
     // previous base directory should stay until Cleaner kicks in.
     status = fs.listStatus(new Path(getWarehouseDir() + "/" +
@@ -818,12 +818,12 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
         Arrays.sort(buckets);
         if (numBase == 1) {
-          Assert.assertEquals("base_-9223372036854775808_v0000009", 
status[i].getPath().getName());
+          Assert.assertEquals("base_-9223372036854775808_v0000008", 
status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numBase == 2) {
           // The new base dir now has two bucket files, since the delta dir 
has two bucket files
-          Assert.assertEquals("base_10000002_v0000018", 
status[i].getPath().getName());
+          Assert.assertEquals("base_10000002_v0000016", 
status[i].getPath().getName());
           Assert.assertEquals(2, buckets.length);
           Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
         }
@@ -845,12 +845,12 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
     Assert.assertEquals(7, status.length);
     runCleaner(hiveConf);
     runCleaner(hiveConf);
-    // There should be only 1 directory left: base_0000001.
+    // There should be only 1 directory left: base_00000016
     // Original bucket files, delta directories and previous base directory 
should have been cleaned up.
     status = fs.listStatus(new Path(getWarehouseDir() + "/" +
       (Table.NONACIDORCTBL).toString().toLowerCase()), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
     Assert.assertEquals(1, status.length);
-    Assert.assertEquals("base_10000002_v0000018", 
status[0].getPath().getName());
+    Assert.assertEquals("base_10000002_v0000016", 
status[0].getPath().getName());
     FileStatus[] buckets = fs.listStatus(status[0].getPath(), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
     Arrays.sort(buckets);
     Assert.assertEquals(2, buckets.length);
@@ -919,7 +919,7 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
     for (int i = 0; i < status.length; i++) {
       Path parent = status[i].getPath().getParent();
       if (parent.getName().matches("base_.*")) {
-        
Assert.assertTrue(parent.getName().startsWith("base_-9223372036854775808_v0000009"));
+        
Assert.assertTrue(parent.getName().startsWith("base_-9223372036854775808_v0000008"));
         sawNewBase = true;
         FileStatus[] buckets = fs.listStatus(parent, 
FileUtils.HIDDEN_FILES_PATH_FILTER);
         Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -970,7 +970,7 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
           Assert.assertEquals("bucket_00001_0", 
buckets[0].getPath().getName());
         }
       } else if (parent.getName().matches("base_.*")) {
-        Assert.assertTrue("base_-9223372036854775808", 
parent.getName().startsWith("base_-9223372036854775808_v0000009"));//_v0000009
+        Assert.assertTrue("base_-9223372036854775808", 
parent.getName().startsWith("base_-9223372036854775808_v0000008"));//_v0000009
         sawNewBase = true;
         FileStatus[] buckets = fs.listStatus(parent, 
FileUtils.HIDDEN_FILES_PATH_FILTER);
         Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -991,7 +991,7 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
     // 5. Perform another major compaction
     runStatementOnDriver("alter table "+ Table.NONACIDNESTEDPART + " 
partition(p='p1',q='q1') compact 'MAJOR'");
     runWorker(hiveConf);
-    // There should be 1 new base directory: base_0000001
+    // There should be 1 new base directory: base_00000016
     // Original bucket files, delta directories, delete_delta directories and 
the
     // previous base directory should stay until Cleaner kicks in.
     status = listFilesByTable(fs, Table.NONACIDNESTEDPART);
@@ -1007,12 +1007,12 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
         FileStatus[] buckets = fs.listStatus(parent, 
FileUtils.HIDDEN_FILES_PATH_FILTER);
         Arrays.sort(buckets);
         if (numBase == 1) {
-          Assert.assertEquals("base_-9223372036854775808_v0000009", 
parent.getName());
+          Assert.assertEquals("base_-9223372036854775808_v0000008", 
parent.getName());
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numBase == 2) {
           // The new base dir now has two bucket files, since the delta dir 
has two bucket files
-          Assert.assertEquals("base_10000002_v0000018", parent.getName());
+          Assert.assertEquals("base_10000002_v0000016", parent.getName());
           Assert.assertEquals(2, buckets.length);
           Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
         }
@@ -1034,11 +1034,11 @@ public class TestTxnCommands2 extends 
TxnCommandsBaseForTests {
     Assert.assertEquals(8, status.length);
     runCleaner(hiveConf);
     runCleaner(hiveConf);
-    // There should be only 1 directory left: base_0000001.
+    // There should be only 1 directory left: base_00000016
     // Original bucket files, delta directories and previous base directory 
should have been cleaned up. Only one base with 2 files.
     status = listFilesByTable(fs, Table.NONACIDNESTEDPART);
     Assert.assertEquals(2, status.length);
-    Assert.assertEquals("base_10000002_v0000018", 
status[0].getPath().getParent().getName());
+    Assert.assertEquals("base_10000002_v0000016", 
status[0].getPath().getParent().getName());
     FileStatus[] buckets = fs.listStatus(status[0].getPath().getParent(), 
FileUtils.HIDDEN_FILES_PATH_FILTER);
     Arrays.sort(buckets);
     Assert.assertEquals(2, buckets.length);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
index d3d829c0247..563b86a471f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
@@ -350,7 +350,7 @@ public class TestTxnCommands3 extends 
TxnCommandsBaseForTests {
      ├── delta_0000001_0000001_0000
      │   ├── _orc_acid_version
      │   └── bucket_00000
-     ├── delta_0000001_0000002_v0000010
+     ├── delta_0000001_0000002_v0000009
      │   ├── _orc_acid_version
      │   └── bucket_00000
      └── delta_0000002_0000002_0000
@@ -362,7 +362,7 @@ public class TestTxnCommands3 extends 
TxnCommandsBaseForTests {
         FileUtils.HIDDEN_FILES_PATH_FILTER);
 
     String[] expectedList = new String[] {
-        "/t/delta_0000001_0000002_v0000010",
+        "/t/delta_0000001_0000002_v0000009",
         "/t/delta_0000001_0000001_0000",
         "/t/delta_0000002_0000002_0000",
     };
@@ -390,13 +390,13 @@ public class TestTxnCommands3 extends 
TxnCommandsBaseForTests {
     runWorker(hiveConf);
     /*
     at this point delta_0000001_0000003_v0000012 is visible to everyone
-    so cleaner removes all files shadowed by it (which is everything in this 
case)
+    so cleaner removes all files shadowed by it
     */
     runCleaner(hiveConf);
     runCleaner(hiveConf);
 
     expectedList = new String[] {
-        "/t/delta_0000001_0000003_v0000014"
+        "/t/delta_0000001_0000003_v0000012"
     };
     actualList = fs.listStatus(new Path(warehousePath + "/t"),
         FileUtils.HIDDEN_FILES_PATH_FILTER);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
index 49fccaf3642..6636ff252bd 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
@@ -345,11 +345,11 @@ 
target/tmp/org.apache.hadoop.hive.ql.TestTxnCommands-1521148657811/
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected3 = new String[][] {
         {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
-            ".*t/delta_0000001_0000002_v000001[5-6]/bucket_00000"},
+            ".*t/delta_0000001_0000002_v000001[4-5]/bucket_00000"},
         {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
-            ".*t/delta_0000001_0000002_v000001[5-6]/bucket_00000"},
+            ".*t/delta_0000001_0000002_v000001[4-5]/bucket_00000"},
         {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t0\t6",
-            ".*t/delta_0000001_0000002_v000001[5-6]/bucket_00000"}};
+            ".*t/delta_0000001_0000002_v000001[4-5]/bucket_00000"}};
     checkResult(expected3, testQuery, isVectorized, "minor compact imported 
table");
 
   }
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index 9fdb208f60b..d5d6a330f48 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -138,8 +138,8 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'minor'");
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected3 = new String[][] {
-        {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t17", 
"t/delta_0000001_0000004_v0000019/bucket_00000"},
-        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2", 
"t/delta_0000001_0000004_v0000019/bucket_00000"}
+        {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t17", 
"t/delta_0000001_0000004_v0000018/bucket_00000"},
+        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2", 
"t/delta_0000001_0000004_v0000018/bucket_00000"}
     };
     checkResult(expected3, testQuery, isVectorized, "delete compact minor");
 
@@ -172,9 +172,9 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'major'");
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected6 = new String[][]{
-        {"{\"writeid\":7,\"bucketid\":536870913,\"rowid\":0}\t1\t17", 
"t/base_0000009_v0000035/bucket_00000"},
-        {"{\"writeid\":7,\"bucketid\":536936449,\"rowid\":0}\t1\t17", 
"t/base_0000009_v0000035/bucket_00001"},
-        {"{\"writeid\":9,\"bucketid\":536870912,\"rowid\":0}\t2\t2", 
"t/base_0000009_v0000035/bucket_00000"}
+        {"{\"writeid\":7,\"bucketid\":536870913,\"rowid\":0}\t1\t17", 
"t/base_0000009_v0000033/bucket_00000"},
+        {"{\"writeid\":7,\"bucketid\":536936449,\"rowid\":0}\t1\t17", 
"t/base_0000009_v0000033/bucket_00001"},
+        {"{\"writeid\":9,\"bucketid\":536870912,\"rowid\":0}\t2\t2", 
"t/base_0000009_v0000033/bucket_00000"}
     };
     checkResult(expected6, testQuery, isVectorized, "load data inpath compact 
major");
   }
@@ -208,10 +208,10 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'minor'");
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected1 = new String[][] {
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2", 
"t/delta_0000001_0000002_v0000011/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4", 
"t/delta_0000001_0000002_v0000011/bucket_00000"},
-        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/delta_0000001_0000002_v0000011/bucket_00000"},
-        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/delta_0000001_0000002_v0000011/bucket_00000"}
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2", 
"t/delta_0000001_0000002_v0000010/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4", 
"t/delta_0000001_0000002_v0000010/bucket_00000"},
+        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/delta_0000001_0000002_v0000010/bucket_00000"},
+        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/delta_0000001_0000002_v0000010/bucket_00000"}
     };
     checkResult(expected1, testQuery, isVectorized, "load data inpath 
(minor)");
 
@@ -220,11 +220,11 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'major'");
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected2 = new String[][] {
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2", 
"t/base_0000003_v0000017/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4", 
"t/base_0000003_v0000017/bucket_00000"},
-        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/base_0000003_v0000017/bucket_00000"},
-        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/base_0000003_v0000017/bucket_00000"},
-        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2", 
"t/base_0000003_v0000017/bucket_00000"}
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2", 
"t/base_0000003_v0000015/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4", 
"t/base_0000003_v0000015/bucket_00000"},
+        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/base_0000003_v0000015/bucket_00000"},
+        {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/base_0000003_v0000015/bucket_00000"},
+        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2", 
"t/base_0000003_v0000015/bucket_00000"}
     };
     checkResult(expected2, testQuery, isVectorized, "load data inpath 
(major)");
 
@@ -242,9 +242,9 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'major'");
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected4 = new String[][] {
-        {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t5\t6", 
"t/base_0000005_v0000026/bucket_00000"},
-        {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":1}\t7\t8", 
"t/base_0000005_v0000026/bucket_00000"},
-        {"{\"writeid\":5,\"bucketid\":536870912,\"rowid\":0}\t6\t6", 
"t/base_0000005_v0000026/bucket_00000"}};
+        {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t5\t6", 
"t/base_0000005_v0000023/bucket_00000"},
+        {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":1}\t7\t8", 
"t/base_0000005_v0000023/bucket_00000"},
+        {"{\"writeid\":5,\"bucketid\":536870912,\"rowid\":0}\t6\t6", 
"t/base_0000005_v0000023/bucket_00000"}};
     checkResult(expected4, testQuery, isVectorized, "load data inpath 
overwrite (major)");
   }
   /**
@@ -322,13 +322,13 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
 
     String[][] expected3 = new String[][] {
         {"{\"writeid\":10000002,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
-            "t/base_10000003_v0000014/bucket_00000"},
+            "t/base_10000003_v0000013/bucket_00000"},
         {"{\"writeid\":10000002,\"bucketid\":536870912,\"rowid\":1}\t7\t8",
-            "t/base_10000003_v0000014/bucket_00000"},
+            "t/base_10000003_v0000013/bucket_00000"},
         {"{\"writeid\":10000002,\"bucketid\":536936448,\"rowid\":0}\t8\t8",
-            "t/base_10000003_v0000014/bucket_00001"},
+            "t/base_10000003_v0000013/bucket_00001"},
         {"{\"writeid\":10000003,\"bucketid\":536870912,\"rowid\":0}\t9\t9",
-            "t/base_10000003_v0000014/bucket_00000"}
+            "t/base_10000003_v0000013/bucket_00000"}
     };
     checkResult(expected3, testQuery, isVectorized, "load data inpath 
overwrite (major)");
   }
@@ -451,10 +451,10 @@ public class TestTxnLoadData extends 
TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'major'");
     TestTxnCommands2.runWorker(hiveConf);
     String[][] expected2 = new String[][] {
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/base_0000001_v0000010/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/base_0000001_v0000010/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t5\t5", 
"t/base_0000001_v0000010/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t6\t6", 
"t/base_0000001_v0000010/bucket_00000"}
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2", 
"t/base_0000001_v0000009/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4", 
"t/base_0000001_v0000009/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t5\t5", 
"t/base_0000001_v0000009/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t6\t6", 
"t/base_0000001_v0000009/bucket_00000"}
     };
     checkResult(expected2, testQuery, isVectorized, "load data inpath 
(major)");
     //at lest for now, Load Data w/Overwrite is not allowed in a txn: 
HIVE-18154
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java 
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index d140868ae8c..40a0a169744 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -164,10 +164,10 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
     */
 
     String expected[][] = {
-        {"{\"writeid\":2,\"bucketid\":536936449,\"rowid\":0}\t0\t0\t17", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00001"},
-        {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t1\t17", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00001"},
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00000"}
+        {"{\"writeid\":2,\"bucketid\":536936449,\"rowid\":0}\t0\t0\t17", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00001"},
+        {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t1\t17", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00001"},
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3", 
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00000"}
     };
     checkResult(expected,
         "select ROW__ID, c1, c2, c3" + (shouldVectorize() ? "" : ", 
INPUT__FILE__NAME")
@@ -182,8 +182,8 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
     expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/delta_0000001_0000001_0000/bucket_00001_0");
     expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/delta_0000002_0000002_0001/bucket_00000_0");
     expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/delta_0000002_0000002_0001/bucket_00001_0");
-    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000012/bucket_00000");
-    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000012/bucket_00001");
+    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000011/bucket_00000");
+    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000011/bucket_00001");
     assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/" + 
NO_BUCKETS_TBL_NAME, NO_BUCKETS_TBL_NAME);
 
     TestTxnCommands2.runCleaner(hiveConf);
@@ -192,8 +192,8 @@ public class TestTxnNoBuckets extends 
TxnCommandsBaseForTests {
     Assert.assertEquals("Unexpected result after clean", 
stringifyValues(result), rs);
 
     expectedFiles.clear();
-    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000012/bucket_00000");
-    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000012/bucket_00001");
+    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000011/bucket_00000");
+    expectedFiles.add(NO_BUCKETS_TBL_NAME + 
"/base_0000002_v0000011/bucket_00001");
     assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/" + 
NO_BUCKETS_TBL_NAME, NO_BUCKETS_TBL_NAME);
   }
 
@@ -472,23 +472,23 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/Users/ekoifman/dev/hiver
     /*Compaction preserves location of rows wrt buckets/tranches (for now)*/
     String expected4[][] = {
         {"{\"writeid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2",
-            "warehouse/t/base_10000002_v0000016/bucket_00002"},
+            "warehouse/t/base_10000002_v0000015/bucket_00002"},
         {"{\"writeid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4",
-            "warehouse/t/base_10000002_v0000016/bucket_00002"},
+            "warehouse/t/base_10000002_v0000015/bucket_00002"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t9\t10",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t10\t20",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t12\t12",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t20\t40",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":5}\t50\t60",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
         {"{\"writeid\":10000001,\"bucketid\":536870913,\"rowid\":0}\t60\t88",
-            "warehouse/t/base_10000002_v0000016/bucket_00000"},
+            "warehouse/t/base_10000002_v0000015/bucket_00000"},
     };
     checkExpected(rs, expected4,"after major compact");
   }
@@ -744,15 +744,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/Users/ekoifman/dev/hiver
     rs = runStatementOnDriver(query);
     String[][] expected5 = {//the row__ids are the same after compaction
         {"{\"writeid\":10000001,\"bucketid\":536870913,\"rowid\":0}\t1\t17",
-            "warehouse/t/base_10000001_v0000018/bucket_00000"},
+            "warehouse/t/base_10000001_v0000017/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t2\t4",
-            "warehouse/t/base_10000001_v0000018/bucket_00000"},
+            "warehouse/t/base_10000001_v0000017/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6",
-            "warehouse/t/base_10000001_v0000018/bucket_00000"},
+            "warehouse/t/base_10000001_v0000017/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t6\t8",
-            "warehouse/t/base_10000001_v0000018/bucket_00000"},
+            "warehouse/t/base_10000001_v0000017/bucket_00000"},
         {"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10",
-            "warehouse/t/base_10000001_v0000018/bucket_00000"}
+            "warehouse/t/base_10000001_v0000017/bucket_00000"}
     };
     checkExpected(rs, expected5, "After major compaction");
     //vectorized because there is INPUT__FILE__NAME
@@ -816,10 +816,10 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree 
/Users/ekoifman/dev/hiver
         {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", 
"t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000_0"},
         {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", 
"t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000_0"},
         {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", 
"t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000_0"},
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", 
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"},
-        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", 
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"},
-        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", 
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"},
-        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", 
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"}
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", 
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"},
+        {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", 
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"},
+        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", 
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"},
+        {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", 
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"}
     };
     checkExpected(rs, expected2, "after major compaction");
 
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java 
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index c819c477f88..42be09529d3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -888,7 +888,6 @@ public class TestDbTxnManager2 extends 
DbTxnManagerEndToEndTestBase {
 
     conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
     HiveTxnManager txnMgr = 
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
-    txnMgr.openTxn(ctx, "T1");
     driver.compileAndRespond("insert into tab_not_acid partition(np='blah') 
values(7,8)", true);
     ((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T1", false);
     List<ShowLocksResponseElement> locks = getLocks(txnMgr);
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
index 865365722fc..e18feedbf56 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -225,6 +225,17 @@ public class DatabaseProduct implements Configurable {
         || (isDERBY() && "42X05".equalsIgnoreCase(e.getSQLState()));
   }
 
+  public String toVarChar(String column) {
+    switch (dbType) {
+      case DERBY:
+        return String.format("CAST(%s AS VARCHAR(4000))", column);
+      case ORACLE:
+        return String.format("to_char(%s)", column);
+      default:
+        return column;
+    }
+  }
+
   /**
    * Whether the RDBMS has restrictions on IN list size (explicit, or poor 
perf-based).
    */
diff --git 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index d910504a9d3..35079df61d3 100644
--- 
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ 
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -3384,7 +3384,7 @@ class MetaStoreDirectSql {
     String statement = TxnUtils.createUpdatePreparedStmt(
         "\"TABLE_PARAMS\"",
         ImmutableList.of("\"PARAM_VALUE\""),
-        ImmutableList.of("\"TBL_ID\"", "\"PARAM_KEY\"", "\"PARAM_VALUE\""));
+        ImmutableList.of("\"TBL_ID\"", "\"PARAM_KEY\"", 
dbType.toVarChar("\"PARAM_VALUE\"")));
     Query query = pm.newQuery("javax.jdo.query.SQL", statement);
     return (long) query.executeWithArray(newValue, table.getId(), key, 
expectedValue);
   }
diff --git a/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java 
b/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
index b3c83b95b06..d11f7950be4 100644
--- a/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
+++ b/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
@@ -21,6 +21,7 @@ package org.apache.hive.streaming;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.LockComponentBuilder;
 import org.apache.hadoop.hive.metastore.LockRequestBuilder;
 import org.apache.hadoop.hive.metastore.Warehouse;
@@ -435,18 +436,25 @@ public class TransactionBatch extends 
AbstractStreamingTransaction {
     }
   }
 
-  private static LockRequest createLockRequest(final HiveStreamingConnection 
connection,
+  private static LockRequest createLockRequest(final HiveStreamingConnection 
conn,
       String partNameForLock, String user, long txnId, String agentInfo) {
     LockRequestBuilder requestBuilder = new LockRequestBuilder(agentInfo);
     requestBuilder.setUser(user);
     requestBuilder.setTransactionId(txnId);
 
     LockComponentBuilder lockCompBuilder = new LockComponentBuilder()
-        .setDbName(connection.getDatabase())
-        .setTableName(connection.getTable().getTableName())
+        .setDbName(conn.getDatabase())
+        .setTableName(conn.getTable().getTableName())
         .setSharedRead()
         .setOperationType(DataOperationType.INSERT);
-    if (connection.isDynamicPartitioning()) {
+
+    boolean isLocklessReadsEnabled = 
conn.getConf().getBoolVar(HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
+    boolean sharedWrite = 
!conn.getConf().getBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK);
+
+    if (sharedWrite || isLocklessReadsEnabled) {
+      lockCompBuilder.setSharedWrite();
+    }
+    if (conn.isDynamicPartitioning()) {
       lockCompBuilder.setIsDynamicPartitionWrite(true);
     }
     if (partNameForLock != null && !partNameForLock.isEmpty()) {


Reply via email to