This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6e1f1ccf2ac HIVE-28366: Iceberg: Concurrent Insert and IOW produce
incorrect result (Denys Kuzmenko, reviewed by Ayush Saxena)
6e1f1ccf2ac is described below
commit 6e1f1ccf2aca58af4be06d6ceea1aa2b877a5d08
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Tue Sep 17 09:24:06 2024 +0200
HIVE-28366: Iceberg: Concurrent Insert and IOW produce incorrect result
(Denys Kuzmenko, reviewed by Ayush Saxena)
Closes #5386
---
.../mr/hive/HiveIcebergOutputCommitter.java | 50 +++++++++------
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 32 ++++++++-
.../iceberg/mr/hive/TestConflictingDataFiles.java | 75 +++++++++++++++++++++-
.../org/apache/iceberg/mr/hive/TestHiveShell.java | 2 +
.../org/apache/hadoop/hive/ql/TestAcidOnTez.java | 23 ++++---
.../ql/txn/compactor/TestCrudCompactorOnTez.java | 12 ++--
.../java/org/apache/hadoop/hive/ql/Compiler.java | 2 +-
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 49 ++++++--------
.../apache/hadoop/hive/ql/DriverTxnHandler.java | 33 +++++++---
.../java/org/apache/hadoop/hive/ql/Executor.java | 2 +-
.../java/org/apache/hadoop/hive/ql/QueryPlan.java | 20 +++---
.../ql/ddl/database/drop/DropDatabaseAnalyzer.java | 2 +-
.../storage/compact/AlterTableCompactAnalyzer.java | 5 +-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 11 +---
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 46 ++++++-------
.../hive/ql/metadata/HiveStorageHandler.java | 31 ++++-----
.../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 6 +-
.../hive/ql/parse/ExecuteStatementAnalyzer.java | 2 +-
.../hive/ql/parse/ExportSemanticAnalyzer.java | 2 +-
.../org/apache/hadoop/hive/ql/TestTxnCommands.java | 8 +--
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 30 ++++-----
.../apache/hadoop/hive/ql/TestTxnCommands3.java | 8 +--
.../org/apache/hadoop/hive/ql/TestTxnExIm.java | 6 +-
.../org/apache/hadoop/hive/ql/TestTxnLoadData.java | 50 +++++++--------
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 52 +++++++--------
.../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 1 -
.../hadoop/hive/metastore/DatabaseProduct.java | 11 ++++
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 2 +-
.../apache/hive/streaming/TransactionBatch.java | 16 +++--
29 files changed, 357 insertions(+), 232 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
index 189661b4736..f22e66c0623 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
@@ -508,7 +508,7 @@ public class HiveIcebergOutputCommitter extends
OutputCommitter {
commitCompaction(table, snapshotId, startTime, filesForCommit,
partitionPath);
} else {
- commitOverwrite(table, branchName, startTime, filesForCommit);
+ commitOverwrite(table, branchName, snapshotId, startTime,
filesForCommit);
}
}
}
@@ -526,6 +526,7 @@ public class HiveIcebergOutputCommitter extends
OutputCommitter {
* Creates and commits an Iceberg change with the provided data and delete
files.
* If there are no delete files then an Iceberg 'append' is created,
otherwise Iceberg 'overwrite' is created.
* @param table The table we are changing
+ * @param snapshotId The snapshot id of the table to use for validation
* @param startTime The start time of the commit - used only for logging
* @param results The object containing the new files we would like to
add to the table
* @param filterExpr Filter expression for conflict detection filter
@@ -563,6 +564,7 @@ public class HiveIcebergOutputCommitter extends
OutputCommitter {
RowDelta write = table.newRowDelta();
results.dataFiles().forEach(write::addRows);
results.deleteFiles().forEach(write::addDeletes);
+
if (StringUtils.isNotEmpty(branchName)) {
write.toBranch(HiveUtils.getTableSnapshotRef(branchName));
}
@@ -589,11 +591,11 @@ public class HiveIcebergOutputCommitter extends
OutputCommitter {
* Creates and commits an Iceberg compaction change with the provided data
files.
* Either full table or a selected partition contents is replaced with
compacted files.
*
- * @param table The table we are changing
- * @param snapshotId The snapshot id of the table to use for
validation
- * @param startTime The start time of the commit - used only for
logging
- * @param results The object containing the new files
- * @param partitionPath The path of the compacted partition
+ * @param table The table we are changing
+ * @param snapshotId The snapshot id of the table to use for validation
+ * @param startTime The start time of the commit - used only for logging
+ * @param results The object containing the new files
+ * @param partitionPath The path of the compacted partition
*/
private void commitCompaction(Table table, Long snapshotId, long startTime,
FilesForCommit results,
String partitionPath) {
@@ -601,46 +603,53 @@ public class HiveIcebergOutputCommitter extends
OutputCommitter {
List<DeleteFile> existingDeleteFiles =
IcebergTableUtil.getDeleteFiles(table, partitionPath);
RewriteFiles rewriteFiles = table.newRewrite();
- rewriteFiles.validateFromSnapshot(getSnapshotId(table, null));
- if (snapshotId != null) {
- rewriteFiles.validateFromSnapshot(snapshotId);
- }
-
existingDataFiles.forEach(rewriteFiles::deleteFile);
existingDeleteFiles.forEach(rewriteFiles::deleteFile);
results.dataFiles().forEach(rewriteFiles::addFile);
+ if (snapshotId != null) {
+ rewriteFiles.validateFromSnapshot(snapshotId);
+ }
rewriteFiles.commit();
LOG.info("Compaction commit took {} ms for table: {} partition: {} with {}
file(s)",
- System.currentTimeMillis() - startTime, table, partitionPath == null ?
"N/A" : partitionPath,
+ System.currentTimeMillis() - startTime, table,
StringUtils.defaultString(partitionPath, "N/A"),
results.dataFiles().size());
}
/**
* Creates and commits an Iceberg insert overwrite change with the provided
data files.
- * For unpartitioned tables the table content is replaced with the new data
files. If not data files are provided
- * then the unpartitioned table is truncated.
- * For partitioned tables the relevant partitions are replaced with the new
data files. If no data files are provided
- * then the unpartitioned table remains unchanged.
+ * For unpartitioned tables the table content is replaced with the new data
files. Table is truncated
+ * if no data files are provided.
+ * For partitioned tables the relevant partitions are replaced with the new
data files. Table remains unchanged
+ * unless data files are provided.
*
- * @param table The table we are changing
- * @param startTime The start time of the commit - used only
for logging
- * @param results The object containing the new files
+ * @param table The table we are changing
+ * @param snapshotId The snapshot id of the table to use for validation
+ * @param startTime The start time of the commit - used only for logging
+ * @param results The object containing the new files
*/
- private void commitOverwrite(Table table, String branchName, long startTime,
FilesForCommit results) {
+ private void commitOverwrite(Table table, String branchName, Long
snapshotId, long startTime,
+ FilesForCommit results) {
Preconditions.checkArgument(results.deleteFiles().isEmpty(), "Can not
handle deletes with overwrite");
if (!results.dataFiles().isEmpty()) {
ReplacePartitions overwrite = table.newReplacePartitions();
results.dataFiles().forEach(overwrite::addFile);
+
if (StringUtils.isNotEmpty(branchName)) {
overwrite.toBranch(HiveUtils.getTableSnapshotRef(branchName));
}
+ if (snapshotId != null) {
+ overwrite.validateFromSnapshot(snapshotId);
+ }
+ overwrite.validateNoConflictingDeletes();
+ overwrite.validateNoConflictingData();
overwrite.commit();
LOG.info("Overwrite commit took {} ms for table: {} with {} file(s)",
System.currentTimeMillis() - startTime,
table, results.dataFiles().size());
} else if (table.spec().isUnpartitioned()) {
DeleteFiles deleteFiles = table.newDelete();
deleteFiles.deleteFromRowFilter(Expressions.alwaysTrue());
+
if (StringUtils.isNotEmpty(branchName)) {
deleteFiles.toBranch(HiveUtils.getTableSnapshotRef(branchName));
}
@@ -881,7 +890,6 @@ public class HiveIcebergOutputCommitter extends
OutputCommitter {
LOG.info("Cleaning job for jobID: {}, table: {}",
jobContext.getJobID(), output);
Table table = output.getKey();
- FileSystem fileSystem = new
Path(table.location()).getFileSystem(jobConf);
String jobLocation = generateJobLocation(table.location(),
jobConf, jobContext.getJobID());
// list jobLocation to get number of forCommit files
// we do this because map/reduce num in jobConf is unreliable
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index a78ea2069f4..4b7f87f0206 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -164,10 +164,12 @@ import org.apache.iceberg.SortField;
import org.apache.iceberg.SortOrder;
import org.apache.iceberg.StatisticsFile;
import org.apache.iceberg.Table;
+import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TableScan;
import org.apache.iceberg.actions.DeleteOrphanFiles;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.expressions.Evaluator;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.ExpressionUtil;
@@ -175,6 +177,7 @@ import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Projections;
import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.expressions.StrictMetricsEvaluator;
+import org.apache.iceberg.hadoop.ConfigProperties;
import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.hive.HiveSchemaUtil;
import org.apache.iceberg.io.CloseableIterable;
@@ -640,12 +643,15 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
}
/**
- * No need for exclusive locks when writing, since Iceberg tables use
optimistic concurrency when writing
- * and only lock the table during the commit operation.
+ * Iceberg's optimistic concurrency control fails to provide means for IOW
and Insert operations isolation.
+ * Use `hive.txn.ext.locking.enabled` config to create Hive locks in order
to guarantee data consistency.
*/
@Override
public LockType getLockType(WriteEntity writeEntity) {
- return LockType.SHARED_READ;
+ if (WriteEntity.WriteType.INSERT_OVERWRITE == writeEntity.getWriteType()) {
+ return LockType.EXCL_WRITE;
+ }
+ return LockType.SHARED_WRITE;
}
@Override
@@ -1443,6 +1449,26 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
map.remove("columns.comments");
}
+ @Override
+ public void validateCurrentSnapshot(TableDesc tableDesc) {
+ if (conf.getBoolean(ConfigProperties.LOCK_HIVE_ENABLED,
TableProperties.HIVE_LOCK_ENABLED_DEFAULT) ||
+ !HiveConf.getBoolVar(conf, ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED)) {
+ return;
+ }
+ Table table = IcebergTableUtil.getTable(conf, tableDesc.getProperties());
+ if (table.currentSnapshot() != null || table instanceof BaseTable) {
+ TableMetadata currentMetadata = ((BaseTable)
table).operations().current();
+ if (currentMetadata.propertyAsBoolean(TableProperties.HIVE_LOCK_ENABLED,
false)) {
+ return;
+ }
+ TableMetadata newMetadata = ((BaseTable) table).operations().refresh();
+ ValidationException.check(
+ Objects.equals(newMetadata.metadataFileLocation(),
currentMetadata.metadataFileLocation()),
+ "Table snapshot is outdated: %s", tableDesc.getTableName()
+ );
+ }
+ }
+
/**
* Recursively replaces the ExprNodeDynamicListDesc nodes by a dummy
ExprNodeConstantDesc so we can test if we can
* convert the predicate to an Iceberg predicate when pruning the partitions
later. Also collects the column names
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
index b107041105d..a07d2d5203f 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestConflictingDataFiles.java
@@ -28,12 +28,15 @@ import java.util.concurrent.Executors;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
import org.apache.iceberg.hive.HiveTableOperations;
import org.apache.iceberg.mr.TestHelper;
import org.apache.iceberg.relocated.com.google.common.base.Throwables;
+import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.Tasks;
import org.junit.After;
import org.junit.Assert;
@@ -43,6 +46,7 @@ import org.junit.Test;
import org.mockito.MockedStatic;
import static
org.apache.iceberg.mr.hive.HiveIcebergStorageHandlerTestUtils.init;
+import static org.apache.iceberg.types.Types.NestedField.required;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
@@ -65,8 +69,9 @@ public class TestConflictingDataFiles extends
HiveIcebergStorageHandlerWithEngin
tableOps.when(() -> method.invoke(null, anyMap(), eq(true)))
.thenAnswer(invocation -> null);
// create and insert an initial batch of records
- testTables.createTable(shell, "customers",
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, fileFormat,
- HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2, 2,
Collections.emptyMap(), storageHandlerStub);
+ testTables.createTable(shell, "customers",
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec,
+ fileFormat,
HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2, 2,
Collections.emptyMap(),
+ storageHandlerStub);
}
// insert one more batch so that we have multiple data files within the
same partition
shell.executeStatement(testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_1,
@@ -259,6 +264,72 @@ public class TestConflictingDataFiles extends
HiveIcebergStorageHandlerWithEngin
.build();
HiveIcebergTestUtils.validateData(expected,
HiveIcebergTestUtils.valueForRow(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
objects), 0);
+ }
+
+ @Test
+ public void testConcurrentInsertAndInsertOverwrite() {
+ Assume.assumeTrue(formatVersion == 2);
+
+ Schema schema = new Schema(
+ required(1, "i", Types.IntegerType.get()),
+ required(2, "p", Types.IntegerType.get())
+ );
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("i",
10).build();
+
+ // create and insert an initial batch of records
+ testTables.createTable(shell, "ice_t", schema, spec, fileFormat,
+ TestHelper.RecordsBuilder.newInstance(schema)
+ .add(1, 1)
+ .add(2, 2)
+ .add(10, 10)
+ .add(20, 20)
+ .add(40, 40)
+ .add(30, 30)
+ .build(),
+ formatVersion);
+
+ String[] singleFilterQuery = new String[] { "INSERT INTO ice_t SELECT
i*100, p*100 FROM ice_t",
+ "INSERT OVERWRITE TABLE ice_t SELECT i+1, p+1 FROM ice_t" };
+
+ Tasks.range(2).executeWith(Executors.newFixedThreadPool(2)).run(i -> {
+ if (i == 1) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ init(shell, testTables, temp, executionEngine);
+ HiveConf.setBoolVar(shell.getHiveConf(),
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, isVectorized);
+ HiveConf.setVar(shell.getHiveConf(),
HiveConf.ConfVars.HIVE_FETCH_TASK_CONVERSION, "none");
+
+ HiveConf.setBoolVar(shell.getHiveConf(),
HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
+ shell.getHiveConf().setBoolean(ConfigProperties.LOCK_HIVE_ENABLED,
false);
+ HiveConf.setVar(shell.getHiveConf(),
HiveConf.ConfVars.HIVE_QUERY_REEXECUTION_STRATEGIES,
+ RETRY_STRATEGIES_WITHOUT_WRITE_CONFLICT);
+ shell.executeStatement(singleFilterQuery[i]);
+ shell.closeSession();
+ });
+
+ List<Object[]> objects =
+ shell.executeStatement("SELECT * FROM ice_t");
+ Assert.assertEquals(12, objects.size());
+ List<Record> expected = TestHelper.RecordsBuilder.newInstance(schema)
+ .add(2, 2)
+ .add(3, 3)
+ .add(11, 11)
+ .add(21, 21)
+ .add(31, 31)
+ .add(41, 41)
+ .add(101, 101)
+ .add(201, 201)
+ .add(1001, 1001)
+ .add(2001, 2001)
+ .add(3001, 3001)
+ .add(4001, 4001)
+ .build();
+ HiveIcebergTestUtils.validateData(expected,
+ HiveIcebergTestUtils.valueForRow(schema, objects), 0);
}
}
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
index 4898ce81338..43b22039baf 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -227,6 +228,7 @@ public class TestHiveShell {
// set lifecycle hooks
hiveConf.setVar(HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS,
HiveIcebergQueryLifeTimeHook.class.getName());
+ MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.TRY_DIRECT_SQL,
true);
return hiveConf;
}
}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index 0fbf6427338..a9ede32a591 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -92,7 +92,7 @@ public class TestAcidOnTez {
public TestName testName = new TestName();
private HiveConf hiveConf;
private IDriver d;
- private static enum Table {
+ private enum Table {
ACIDTBL("acidTbl"),
ACIDTBLPART("acidTblPart"),
ACIDNOBUCKET("acidNoBucket"),
@@ -166,7 +166,8 @@ public class TestAcidOnTez {
}
private void dropTables() throws Exception {
- for(Table t : Table.values()) {
+ runStatementOnDriver("drop table if exists T");
+ for (Table t : Table.values()) {
runStatementOnDriver("drop table if exists " + t);
}
}
@@ -544,7 +545,7 @@ public class TestAcidOnTez {
//check we have right delete delta files after minor compaction
status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
(Table.ACIDNOBUCKET).toString().toLowerCase()),
FileUtils.STAGING_DIR_PATH_FILTER);
- String[] expectedDelDelta2 = { "delete_delta_0000002_0000002_0000",
"delete_delta_0000003_0000003_0000", "delete_delta_0000001_0000003_v0000015"};
+ String[] expectedDelDelta2 = { "delete_delta_0000002_0000002_0000",
"delete_delta_0000003_0000003_0000", "delete_delta_0000001_0000003_v0000014"};
for(FileStatus stat : status) {
for(int i = 0; i < expectedDelDelta2.length; i++) {
if(expectedDelDelta2[i] != null &&
stat.getPath().toString().endsWith(expectedDelDelta2[i])) {
@@ -569,7 +570,7 @@ public class TestAcidOnTez {
for(int i = 0; i < expected2.length; i++) {
Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i),
rs.get(i).startsWith(expected2[i][0]));
//everything is now in base/
- Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i),
rs.get(i).endsWith("base_0000003_v0000019/bucket_00000"));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i),
rs.get(i).endsWith("base_0000003_v0000017/bucket_00000"));
}
}
/**
@@ -582,7 +583,6 @@ public class TestAcidOnTez {
@Test
public void testInsertWithRemoveUnion() throws Exception {
int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}};
- runStatementOnDriver("drop table if exists T", hiveConf);
runStatementOnDriver("create table T (a int, b int) stored as ORC
TBLPROPERTIES ('transactional'='false')", hiveConf);
/*
ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505502329802/warehouse/t/.hive-staging_hive_2017-09-15_12-07-33_224_7717909516029836949-1/
@@ -630,15 +630,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
~/dev/hiverwgit/itests/h
}
String[][] expected2 = {
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
- "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+ "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
- "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+ "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6",
- "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+ "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t7\t8",
- "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"},
+ "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10",
- "warehouse/t/base_-9223372036854775808_v0000013/bucket_00000"}
+ "warehouse/t/base_-9223372036854775808_v0000011/bucket_00000"}
};
Assert.assertEquals("Unexpected row count after major compact",
expected2.length, rs.size());
for(int i = 0; i < expected2.length; i++) {
@@ -655,7 +655,6 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
~/dev/hiverwgit/itests/h
*/
@Test
public void testAcidInsertWithRemoveUnion() throws Exception {
- runStatementOnDriver("drop table if exists T", hiveConf);
runStatementOnDriver("create table T (a int, b int) stored as ORC
TBLPROPERTIES ('transactional'='true')", hiveConf);
/*On Tez, below (T is transactional), we get the following layout
ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505500035574/warehouse/t/.hive-staging_hive_2017-09-15_11-28-33_960_9111484239090506828-1/
@@ -698,13 +697,13 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
~/dev/hiverwgit/itests/h
Assert.assertTrue("Actual line(file) " + i + " ac: " + rs.get(i),
rs.get(i).endsWith(expected2[i][1]));
}
}
+
@Test
public void testBucketedAcidInsertWithRemoveUnion() throws Exception {
int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}};
runStatementOnDriver("delete from " + Table.ACIDTBL, hiveConf);
//make sure both buckets are not empty
runStatementOnDriver("insert into " + Table.ACIDTBL +
makeValuesClause(values), hiveConf);
- runStatementOnDriver("drop table if exists T", hiveConf);
/*
With bucketed target table Union All is not removed
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index 148a7cb3225..b8d1827de0a 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -147,7 +147,7 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
}
aborted = true;
Assert.assertEquals(LockException.class, e.getCause().getClass());
- Assert.assertEquals( "Transaction manager has aborted the transaction
txnid:20. Reason: Aborting [txnid:20,20] due to a write conflict on
default/rebalance_test committed by [txnid:19,20] d/u", e.getCauseMessage());
+ Assert.assertEquals( "Transaction manager has aborted the transaction
txnid:19. Reason: Aborting [txnid:19,19] due to a write conflict on
default/rebalance_test committed by [txnid:18,19] d/u", e.getCauseMessage());
// Delete the record, so the rest of the test can be the same in both
cases
executeStatementOnDriver("DELETE FROM " + tableName + " WHERE b = 12",
driver);
} finally {
@@ -197,7 +197,7 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
},
};
verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
- new String[] {"bucket_00000", "bucket_00001", "bucket_00002",
"bucket_00003"}, "base_0000007_v0000019");
+ new String[] {"bucket_00000", "bucket_00001", "bucket_00002",
"bucket_00003"}, "base_0000007_v0000018");
}
@Test
@@ -253,7 +253,7 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
},
};
verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
- new String[] {"bucket_00000", "bucket_00001",
"bucket_00002","bucket_00003"}, "base_0000007_v0000019");
+ new String[] {"bucket_00000", "bucket_00001",
"bucket_00002","bucket_00003"}, "base_0000007_v0000018");
}
@Test
@@ -307,7 +307,7 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
},
};
verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
- new String[] {"bucket_00000", "bucket_00001",
"bucket_00002","bucket_00003"}, "base_0000007_v0000019");
+ new String[] {"bucket_00000", "bucket_00001",
"bucket_00002","bucket_00003"}, "base_0000007_v0000018");
}
@Test
@@ -416,7 +416,7 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
},
};
verifyRebalance(testDataProvider, tableName, "ds=tomorrow",
expectedBuckets,
- new String[] {"bucket_00000", "bucket_00001", "bucket_00002"},
"base_0000007_v0000015");
+ new String[] {"bucket_00000", "bucket_00001", "bucket_00002"},
"base_0000007_v0000014");
}
@Test
@@ -490,7 +490,7 @@ public class TestCrudCompactorOnTez extends
CompactorOnTezTest {
},
};
verifyRebalance(testDataProvider, tableName, null, expectedBuckets,
- new String[] {"bucket_00000", "bucket_00001", "bucket_00002",
"bucket_00003"}, "base_0000007_v0000019");
+ new String[] {"bucket_00000", "bucket_00001", "bucket_00002",
"bucket_00003"}, "base_0000007_v0000018");
}
private TestDataProvider prepareRebalanceTestData(String tableName) throws
Exception {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
b/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
index fabe05dbee8..5bacbf3e406 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
@@ -113,7 +113,7 @@ public class Compiler {
plan = createPlan(sem);
if (HiveOperation.START_TRANSACTION == queryState.getHiveOperation()
- || plan.hasAcidResources()) {
+ || plan.isRequiresOpenTransaction()) {
openTxnAndGetValidTxnList();
}
verifyTxnState();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 2241baf3f5b..392a1ffbc00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.reexec.ReCompileException;
import org.apache.hadoop.hive.ql.session.LineageState;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -173,6 +174,11 @@ public class Driver implements IDriver {
driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
}
+ DriverUtils.checkInterrupted(driverState, driverContext, "at acquiring
the lock.", null, null);
+
+ lockAndRespond();
+ validateCurrentSnapshot();
+
// Reset the PerfLogger so that it doesn't retain any previous values.
// Any value from compilation phase can be obtained through the map set
in queryDisplay during compilation.
PerfLogger perfLogger = SessionState.getPerfLogger(true);
@@ -180,17 +186,7 @@ public class Driver implements IDriver {
// the reason that we set the txn manager for the cxt here is because
each query has its own ctx object.
// The txn mgr is shared across the same instance of Driver, which can
run multiple queries.
context.setHiveTxnManager(driverContext.getTxnManager());
-
- DriverUtils.checkInterrupted(driverState, driverContext, "at acquiring
the lock.", null, null);
-
- lockAndRespond();
-
- if (validateTxnList()) {
- // the reason that we set the txn manager for the cxt here is because
each query has its own ctx object.
- // The txn mgr is shared across the same instance of Driver, which can
run multiple queries.
- context.setHiveTxnManager(driverContext.getTxnManager());
- perfLogger = SessionState.getPerfLogger(true);
- }
+
execute();
FetchTask fetchTask = driverContext.getPlan().getFetchTask();
@@ -224,20 +220,21 @@ public class Driver implements IDriver {
}
/**
- * @return If the txn manager should be set.
+ * Checks if the recorded snapshot is up-to-date
*/
- private boolean validateTxnList() throws CommandProcessorException {
+ private void validateCurrentSnapshot() throws CommandProcessorException {
int retryShapshotCount = 0;
+
int maxRetrySnapshotCount = HiveConf.getIntVar(driverContext.getConf(),
HiveConf.ConfVars.HIVE_TXN_MAX_RETRYSNAPSHOT_COUNT);
- boolean shouldSet = false;
-
try {
do {
driverContext.setOutdatedTxn(false);
// Inserts will not invalidate the snapshot, that could cause
duplicates.
+
if (!driverTxnHandler.isValidTxnListState()) {
LOG.info("Re-compiling after acquiring locks, attempt #" +
retryShapshotCount);
+ HiveTxnManager txnMgr = driverContext.getTxnManager();
// Snapshot was outdated when locks were acquired, hence regenerate
context, txn list and retry.
// TODO: Lock acquisition should be moved before analyze, this is a
bit hackish.
// Currently, we acquire a snapshot, compile the query with that
snapshot, and then - acquire locks.
@@ -246,10 +243,10 @@ public class Driver implements IDriver {
if (driverContext.isOutdatedTxn()) {
// Later transaction invalidated the snapshot, a new transaction
is required
LOG.info("Snapshot is outdated, re-initiating transaction ...");
- driverContext.getTxnManager().rollbackTxn();
+ txnMgr.rollbackTxn();
String userFromUGI = DriverUtils.getUserFromUGI(driverContext);
- driverContext.getTxnManager().openTxn(context, userFromUGI,
driverContext.getTxnType());
+ txnMgr.openTxn(context, userFromUGI, driverContext.getTxnType());
lockAndRespond();
} else {
// We need to clear the possibly cached writeIds for the prior
transaction, so new writeIds
@@ -265,42 +262,36 @@ public class Driver implements IDriver {
// The scan basically does last writer wins for a given row which
is determined by
// max(committingWriteId) for a given ROW__ID(originalWriteId,
bucketId, rowId). So the
// data add ends up being > than the data delete.
- driverContext.getTxnManager().clearCaches();
+ txnMgr.clearCaches();
}
driverContext.getConf().unset(ValidTxnList.VALID_TXNS_KEY);
driverContext.setRetrial(true);
- if (driverContext.getPlan().hasAcidReadWrite()) {
- compileInternal(context.getCmd(), true);
+ compileInternal(context.getCmd(), true);
+
+ if (driverContext.getPlan().hasAcidResourcesInQuery()) {
driverTxnHandler.recordValidWriteIds();
driverTxnHandler.setWriteIdForAcidFileSinks();
}
// Since we're reusing the compiled plan, we need to update its
start time for current run
-
driverContext.getPlan().setQueryStartTime(driverContext.getQueryDisplay().getQueryStartTime());
+ driverContext.getPlan().setQueryStartTime(
+ driverContext.getQueryDisplay().getQueryStartTime());
driverContext.setRetrial(false);
}
// Re-check snapshot only in case we had to release locks and open a
new transaction,
// otherwise exclusive locks should protect output tables/partitions
in snapshot from concurrent writes.
} while (driverContext.isOutdatedTxn() && ++retryShapshotCount <=
maxRetrySnapshotCount);
- shouldSet = shouldSetTxnManager(retryShapshotCount,
maxRetrySnapshotCount);
} catch (LockException | SemanticException e) {
DriverUtils.handleHiveException(driverContext, e, 13, null);
}
- return shouldSet;
- }
-
- private boolean shouldSetTxnManager(int retryShapshotCount, int
maxRetrySnapshotCount)
- throws CommandProcessorException {
if (retryShapshotCount > maxRetrySnapshotCount) {
// Throw exception
HiveException e = new HiveException(
"Operation could not be executed, " +
SNAPSHOT_WAS_OUTDATED_WHEN_LOCKS_WERE_ACQUIRED + ".");
DriverUtils.handleHiveException(driverContext, e, 14, null);
}
-
- return retryShapshotCount != 0;
}
private void setInitialStateForRun(boolean alreadyCompiled) throws
CommandProcessorException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
index 05c96d9452c..5f10cd129ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverTxnHandler.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.lockmgr.LockException;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.HiveTableName;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -64,6 +65,7 @@ import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
+import org.apache.hadoop.hive.ql.reexec.ReCompileException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.util.StringUtils;
@@ -218,6 +220,8 @@ class DriverTxnHandler {
perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS);
if (!driverContext.getTxnManager().isTxnOpen() &&
driverContext.getTxnManager().supportsAcid()
+ && (driverContext.getPlan().isRequiresOpenTransaction()
+ || !HiveConf.getBoolVar(driverContext.getConf(),
HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED))
&& !SessionState.get().isCompaction()) {
/* non acid txn managers don't support txns but fwd lock requests to
lock managers
acid txn manager requires all locks to be associated with a txn so if
we end up here w/o an open txn
@@ -235,7 +239,7 @@ class DriverTxnHandler {
if (!SessionState.get().isCompaction()) {
acquireLocksInternal();
}
- if (driverContext.getPlan().hasAcidReadWrite() || hasAcidDdl) {
+ if (driverContext.getPlan().hasAcidResourcesInQuery() || hasAcidDdl) {
recordValidWriteIds();
}
} catch (Exception e) {
@@ -336,8 +340,8 @@ class DriverTxnHandler {
LOG.info("Operation {} obtained {} locks",
driverContext.getPlan().getOperation(),
((locks == null) ? 0 : locks.size()));
// This check is for controlling the correctness of the current state
- if (driverContext.getTxnManager().recordSnapshot(driverContext.getPlan())
&&
- !driverContext.isValidTxnListsGenerated()) {
+ if (driverContext.getTxnManager().recordSnapshot(driverContext.getPlan())
+ && !driverContext.isValidTxnListsGenerated()) {
throw new IllegalStateException("Need to record valid WriteID list but
there is no valid TxnID list (" +
JavaUtils.txnIdToString(driverContext.getTxnManager().getCurrentTxnId()) +
", queryId: " + driverContext.getPlan().getQueryId() + ")");
@@ -418,6 +422,18 @@ class DriverTxnHandler {
* on the table over which the lock is required.
*/
boolean isValidTxnListState() throws LockException {
+ try {
+ context.getLoadTableOutputMap().forEach(
+ (ltd, we) -> {
+ HiveStorageHandler handler = we.getTable().getStorageHandler();
+ if (handler != null) {
+ handler.validateCurrentSnapshot(ltd.getTable());
+ }
+ });
+ } catch (Exception e) {
+ LOG.warn(e.getMessage());
+ return false;
+ }
// 1) Get valid txn list.
String txnString =
driverContext.getConf().get(ValidTxnList.VALID_TXNS_KEY);
if (txnString == null) {
@@ -516,17 +532,18 @@ class DriverTxnHandler {
void handleTransactionAfterExecution() throws CommandProcessorException {
try {
//since set autocommit starts an implicit txn, close it
- if (driverContext.getTxnManager().isImplicitTransactionOpen(context) ||
- driverContext.getPlan().getOperation() == HiveOperation.COMMIT) {
+ if (driverContext.getTxnManager().isImplicitTransactionOpen(context)
+ || driverContext.getPlan().getOperation() == HiveOperation.COMMIT) {
endTransactionAndCleanup(true);
} else if (driverContext.getPlan().getOperation() ==
HiveOperation.ROLLBACK) {
endTransactionAndCleanup(false);
- } else if (!driverContext.getTxnManager().isTxnOpen() &&
- driverContext.getQueryState().getHiveOperation() ==
HiveOperation.REPLLOAD) {
+ } else if (!driverContext.getTxnManager().isTxnOpen()
+ && (driverContext.getQueryState().getHiveOperation() ==
HiveOperation.REPLLOAD
+ || !SessionState.get().isCompaction())) {
// repl load during migration, commits the explicit txn and start some
internal txns. Call
// releaseLocksAndCommitOrRollback to do the clean up.
endTransactionAndCleanup(false);
- }
+ }
// if none of the above is true, then txn (if there is one started) is
not finished
} catch (LockException e) {
DriverUtils.handleHiveException(driverContext, e, 12, null);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
b/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
index 9d0ce22da32..708e3870efa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
@@ -219,7 +219,7 @@ public class Executor {
if (driverContext.getCacheUsage().getStatus() ==
CacheUsage.CacheStatus.CAN_CACHE_QUERY_RESULTS &&
driverContext.getPlan().getFetchTask() != null) {
ValidTxnWriteIdList txnWriteIdList = null;
- if (driverContext.getPlan().hasAcidReadWrite()) {
+ if (driverContext.getPlan().hasAcidResourcesInQuery()) {
txnWriteIdList =
AcidUtils.getValidTxnWriteIdList(driverContext.getConf());
}
// The results of this query execution might be cacheable.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
b/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
index 92d395bfefe..6c47d3bc169 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryPlan.java
@@ -116,8 +116,8 @@ public class QueryPlan implements Serializable {
private transient Long queryStartTime;
private final HiveOperation operation;
- private final boolean hasAcidReadWrite;
- private final boolean hasAcidResources;
+ private final boolean acidResourcesInQuery;
+ private final boolean requiresOpenTransaction;
private final Set<FileSinkDesc> acidSinks; // Note: both full-ACID and
insert-only sinks.
private final WriteEntity acidAnalyzeTable;
private final DDLDescWithWriteId acidDdlDesc;
@@ -132,8 +132,8 @@ public class QueryPlan implements Serializable {
protected QueryPlan(HiveOperation command) {
this.reducerTimeStatsPerJobList = new ArrayList<>();
this.operation = command;
- this.hasAcidReadWrite = false;
- this.hasAcidResources = false;
+ this.acidResourcesInQuery = false;
+ this.requiresOpenTransaction = false;
this.acidSinks = Collections.emptySet();
this.acidDdlDesc = null;
this.acidAnalyzeTable = null;
@@ -165,8 +165,8 @@ public class QueryPlan implements Serializable {
this.autoCommitValue = sem.getAutoCommitValue();
this.resultSchema = resultSchema;
// TODO: all this ACID stuff should be in some sub-object
- this.hasAcidReadWrite = sem.hasAcidReadWrite();
- this.hasAcidResources = sem.hasAcidResources();
+ this.acidResourcesInQuery = sem.hasTransactionalInQuery();
+ this.requiresOpenTransaction = sem.isRequiresOpenTransaction();
this.acidSinks = sem.getAcidFileSinks();
this.acidDdlDesc = sem.getAcidDdlDesc();
this.acidAnalyzeTable = sem.getAcidAnalyzeTable();
@@ -177,12 +177,12 @@ public class QueryPlan implements Serializable {
/**
* @return true if any acid resources are read/written
*/
- public boolean hasAcidReadWrite() {
- return hasAcidReadWrite;
+ public boolean hasAcidResourcesInQuery() {
+ return acidResourcesInQuery;
}
- public boolean hasAcidResources() {
- return hasAcidResources;
+ public boolean isRequiresOpenTransaction() {
+ return requiresOpenTransaction;
}
public WriteEntity getAcidAnalyzeTable() {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
index 09d94383dd9..6d01c2e88b3 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java
@@ -116,7 +116,7 @@ public class DropDatabaseAnalyzer extends
BaseSemanticAnalyzer {
}
@Override
- public boolean hasAcidResources() {
+ public boolean isRequiresOpenTransaction() {
// check DB tags once supported (i.e. ICEBERG_ONLY, ACID_ONLY,
EXTERNAL_ONLY)
return true;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
index 309d356209d..69e0a77a2d5 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/compact/AlterTableCompactAnalyzer.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
@@ -120,7 +119,7 @@ public class AlterTableCompactAnalyzer extends
AbstractAlterTableAnalyzer {
}
@Override
- protected void setAcidDdlDesc(DDLDescWithWriteId desc) {
- // doesn't need an open txn
+ public boolean isRequiresOpenTransaction() {
+ return false; // doesn't need an open txn
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index ff125a30740..cbd99aa5994 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -3057,15 +3057,10 @@ public class AcidUtils {
boolean isExclMergeInsert =
conf.getBoolVar(ConfVars.TXN_MERGE_INSERT_X_LOCK) && isMerge;
compBuilder.setSharedRead();
- if (sharedWrite) {
+ if (sharedWrite || !isExclMergeInsert && isLocklessReadsEnabled) {
compBuilder.setSharedWrite();
- } else {
- if (isExclMergeInsert) {
- compBuilder.setExclWrite();
-
- } else if (isLocklessReadsEnabled) {
- compBuilder.setSharedWrite();
- }
+ } else if (isExclMergeInsert) {
+ compBuilder.setExclWrite();
}
if (isExclMergeInsert) {
compBuilder.setOperationType(DataOperationType.UPDATE);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 3a7eb675256..d9593e514e4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -251,7 +251,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
contention.*/
init();
getLockManager();
- if(isTxnOpen()) {
+ if (isTxnOpen()) {
throw new LockException("Transaction already opened. " +
JavaUtils.txnIdToString(txnId));
}
try {
@@ -295,7 +295,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
acquireLocksWithHeartbeatDelay(plan, ctx, username, 0);
}
catch(LockException e) {
- if(e.getCause() instanceof TxnAbortedException) {
+ if (e.getCause() instanceof TxnAbortedException) {
resetTxnInfo();
}
throw e;
@@ -313,7 +313,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
private void markExplicitTransaction(QueryPlan queryPlan) throws
LockException {
isExplicitTransaction = true;
- if(++startTransactionCount > 1) {
+ if (++startTransactionCount > 1) {
throw new LockException(null, ErrorMsg.OP_NOT_ALLOWED_IN_TXN,
queryPlan.getOperationName(),
JavaUtils.txnIdToString(getCurrentTxnId()), queryPlan.getQueryId());
}
@@ -328,11 +328,11 @@ public final class DbTxnManager extends
HiveTxnManagerImpl {
* @throws LockException
*/
private void verifyState(QueryPlan queryPlan) throws LockException {
- if(!isTxnOpen() && queryPlan.hasAcidResources()) {
+ if (!isTxnOpen() && queryPlan.isRequiresOpenTransaction()) {
throw new LockException("No transaction context for operation: " +
queryPlan.getOperationName() +
" for " + getQueryIdWaterMark(queryPlan));
}
- if(queryPlan.getOperation() == null) {
+ if (queryPlan.getOperation() == null) {
throw new IllegalStateException("Unknown HiveOperation(null) for " +
getQueryIdWaterMark(queryPlan));
}
numStatements++;
@@ -342,13 +342,13 @@ public final class DbTxnManager extends
HiveTxnManagerImpl {
break;
case COMMIT:
case ROLLBACK:
- if(!isExplicitTransaction) {
+ if (!isExplicitTransaction) {
throw new LockException(null,
ErrorMsg.OP_NOT_ALLOWED_IN_IMPLICIT_TXN, queryPlan.getOperationName());
}
break;
default:
- if(!queryPlan.getOperation().isAllowedInTransaction() &&
isExplicitTransaction) {
- if(allowOperationInATransaction(queryPlan)) {
+ if (!queryPlan.getOperation().isAllowedInTransaction() &&
isExplicitTransaction) {
+ if (allowOperationInATransaction(queryPlan)) {
break;
}
//look at queryPlan.outputs(WriteEntity.t - that's the table)
@@ -374,9 +374,9 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
private boolean allowOperationInATransaction(QueryPlan queryPlan) {
//Acid and MM tables support Load Data with transactional semantics. This
will allow Load Data
//in a txn assuming we can determine the target is a suitable table type.
- if(queryPlan.getOperation() == HiveOperation.LOAD &&
queryPlan.getOutputs() != null && queryPlan.getOutputs().size() == 1) {
+ if (queryPlan.getOperation() == HiveOperation.LOAD &&
queryPlan.getOutputs() != null && queryPlan.getOutputs().size() == 1) {
WriteEntity writeEntity = queryPlan.getOutputs().iterator().next();
- if(AcidUtils.isTransactionalTable(writeEntity.getTable())) {
+ if (AcidUtils.isTransactionalTable(writeEntity.getTable())) {
switch (writeEntity.getWriteType()) {
case INSERT:
//allow operation in a txn
@@ -645,7 +645,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
@Override
public void heartbeat() throws LockException {
List<HiveLock> locks;
- if(isTxnOpen()) {
+ if (isTxnOpen()) {
// Create one dummy lock so we can go through the loop below, though we
only
//really need txnId
locks = Collections.singletonList(new DbLockManager.DbHiveLock(0L));
@@ -653,7 +653,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
else {
locks = lockMgr.getLocks(false, false);
}
- if(LOG.isInfoEnabled()) {
+ if (LOG.isInfoEnabled()) {
StringBuilder sb = new StringBuilder("Sending heartbeat for ")
.append(JavaUtils.txnIdToString(txnId)).append(" and");
for(HiveLock lock : locks) {
@@ -661,7 +661,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
}
LOG.info(sb.toString());
}
- if(!isTxnOpen() && locks.isEmpty()) {
+ if (!isTxnOpen() && locks.isEmpty()) {
// No locks, no txn, we outta here.
LOG.debug("No need to send heartbeat as there is no transaction and no
locks.");
return;
@@ -716,7 +716,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
private ScheduledFuture<?> startHeartbeat(long initialDelay, long
heartbeatInterval, Runnable heartbeater) {
// For negative testing purpose..
- if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) &&
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) &&
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
initialDelay = 0;
} else if (initialDelay == 0) {
/*make initialDelay a random number in [0, 0.75*heartbeatInterval] so
that if a lot
@@ -852,17 +852,19 @@ public final class DbTxnManager extends
HiveTxnManagerImpl {
*/
@Override
public boolean recordSnapshot(QueryPlan queryPlan) {
- assert isTxnOpen();
+ if (!isTxnOpen()) {
+ return false;
+ }
assert numStatements > 0 : "was acquireLocks() called already?";
- if(queryPlan.getOperation() == HiveOperation.START_TRANSACTION) {
+ if (queryPlan.getOperation() == HiveOperation.START_TRANSACTION) {
//here if start of explicit txn
assert isExplicitTransaction;
assert numStatements == 1;
return true;
}
- else if(!isExplicitTransaction) {
+ else if (!isExplicitTransaction) {
assert numStatements == 1 : "numStatements=" + numStatements + " in
implicit txn";
- if (queryPlan.hasAcidReadWrite()) {
+ if (queryPlan.hasAcidResourcesInQuery()) {
//1st and only stmt in implicit txn and uses acid resource
return true;
}
@@ -877,11 +879,11 @@ public final class DbTxnManager extends
HiveTxnManagerImpl {
@Override
public boolean isImplicitTransactionOpen(Context ctx) {
- if(!isTxnOpen()) {
+ if (!isTxnOpen()) {
//some commands like "show databases" don't start implicit transactions
return false;
}
- if(!isExplicitTransaction) {
+ if (!isExplicitTransaction) {
if (ctx == null || !ctx.isExplainSkipExecution()) {
assert numStatements == 1 : "numStatements=" + numStatements;
}
@@ -915,7 +917,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
}
private synchronized static void initHeartbeatExecutorService(int
corePoolSize) {
- if(heartbeatExecutorService != null) {
+ if (heartbeatExecutorService != null) {
return;
}
// The following code will be executed only once when the service is not
initialized
@@ -1109,7 +1111,7 @@ public final class DbTxnManager extends
HiveTxnManagerImpl {
public void run() {
try {
// For negative testing purpose..
- if(conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) &&
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) &&
conf.getBoolVar(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER)) {
throw new
LockException(HiveConf.ConfVars.HIVE_TEST_MODE_FAIL_HEARTBEATER.name() +
"=true");
}
LOG.debug("Heartbeating...for currentUser: " + currentUser);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index 6ff715cdf44..2b05837a884 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -105,23 +105,23 @@ public interface HiveStorageHandler extends Configurable {
/**
* @return Class providing an implementation of {@link InputFormat}
*/
- public Class<? extends InputFormat> getInputFormatClass();
+ Class<? extends InputFormat> getInputFormatClass();
/**
* @return Class providing an implementation of {@link OutputFormat}
*/
- public Class<? extends OutputFormat> getOutputFormatClass();
+ Class<? extends OutputFormat> getOutputFormatClass();
/**
* @return Class providing an implementation of {@link AbstractSerDe}
*/
- public Class<? extends AbstractSerDe> getSerDeClass();
+ Class<? extends AbstractSerDe> getSerDeClass();
/**
* @return metadata hook implementation, or null if this
* storage handler does not need any metadata notifications
*/
- public HiveMetaHook getMetaHook();
+ HiveMetaHook getMetaHook();
/**
* Returns the implementation specific authorization provider
@@ -129,8 +129,7 @@ public interface HiveStorageHandler extends Configurable {
* @return authorization provider
* @throws HiveException
*/
- public HiveAuthorizationProvider getAuthorizationProvider()
- throws HiveException;
+ HiveAuthorizationProvider getAuthorizationProvider() throws HiveException;
/**
* This method is called to allow the StorageHandlers the chance
@@ -149,14 +148,13 @@ public interface HiveStorageHandler extends Configurable {
* @param jobProperties receives properties copied or transformed
* from the table properties
*/
- public abstract void configureInputJobProperties(TableDesc tableDesc,
- Map<String, String> jobProperties);
+ void configureInputJobProperties(TableDesc tableDesc, Map<String, String>
jobProperties);
/**
* This method is called to allow the StorageHandlers the chance to
* populate secret keys into the job's credentials.
*/
- public abstract void configureInputJobCredentials(TableDesc tableDesc,
Map<String, String> secrets);
+ void configureInputJobCredentials(TableDesc tableDesc, Map<String, String>
secrets);
/**
* This method is called to allow the StorageHandlers the chance
@@ -175,8 +173,7 @@ public interface HiveStorageHandler extends Configurable {
* @param jobProperties receives properties copied or transformed
* from the table properties
*/
- public abstract void configureOutputJobProperties(TableDesc tableDesc,
- Map<String, String> jobProperties);
+ void configureOutputJobProperties(TableDesc tableDesc, Map<String, String>
jobProperties);
/**
* Deprecated use configureInputJobProperties/configureOutputJobProperties
@@ -191,9 +188,7 @@ public interface HiveStorageHandler extends Configurable {
* from the table properties
*/
@Deprecated
- public void configureTableJobProperties(
- TableDesc tableDesc,
- Map<String, String> jobProperties);
+ void configureTableJobProperties(TableDesc tableDesc, Map<String, String>
jobProperties);
/**
* Called just before submitting MapReduce job.
@@ -201,7 +196,7 @@ public interface HiveStorageHandler extends Configurable {
* @param tableDesc descriptor for the table being accessed
* @param jobConf jobConf for MapReduce job
*/
- public void configureJobConf(TableDesc tableDesc, JobConf jobConf);
+ void configureJobConf(TableDesc tableDesc, JobConf jobConf);
/**
* Used to fetch runtime information about storage handler during DESCRIBE
EXTENDED statement
@@ -210,8 +205,7 @@ public interface HiveStorageHandler extends Configurable {
* @return StorageHandlerInfo containing runtime information about storage
handler
* OR `null` if the storage handler choose to not provide any runtime
information.
*/
- public default StorageHandlerInfo getStorageHandlerInfo(Table table) throws
MetaException
- {
+ default StorageHandlerInfo getStorageHandlerInfo(Table table) throws
MetaException {
return null;
}
@@ -689,6 +683,9 @@ public interface HiveStorageHandler extends Configurable {
default SnapshotContext
getCurrentSnapshotContext(org.apache.hadoop.hive.ql.metadata.Table table) {
return null;
}
+
+ default void validateCurrentSnapshot(TableDesc tableDesc) {
+ }
/**
* Return snapshot metadata of table snapshots which are newer than the
specified.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index fc87e9d736c..f528d67716a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -1571,12 +1571,12 @@ public abstract class BaseSemanticAnalyzer {
return acidFileSinks;
}
- public boolean hasAcidReadWrite() {
+ public boolean hasTransactionalInQuery() {
return transactionalInQuery;
}
- public boolean hasAcidResources() {
- return hasAcidReadWrite() || getAcidDdlDesc() != null ||
+ public boolean isRequiresOpenTransaction() {
+ return hasTransactionalInQuery() || getAcidDdlDesc() != null ||
Stream.of(getInputs(), getOutputs()).flatMap(Collection::stream)
.filter(entity -> entity.getType() == Entity.Type.TABLE ||
entity.getType() == Entity.Type.PARTITION)
.flatMap(entity -> {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
index 89942658161..1428bea874b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExecuteStatementAnalyzer.java
@@ -225,7 +225,7 @@ public class ExecuteStatementAnalyzer extends
SemanticAnalyzer{
this.queryProperties = cachedPlan.getQueryProperties();
this.setAutoCommitValue(cachedPlan.getAutoCommitValue());
- this.transactionalInQuery = cachedPlan.hasAcidReadWrite();
+ this.transactionalInQuery = cachedPlan.hasTransactionalInQuery();
this.acidFileSinks.addAll(cachedPlan.getAcidFileSinks());
this.initCtx(cachedPlan.getCtx());
this.ctx.setCboInfo(cachedPlan.getCboInfo());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
index c7ff003af17..70253374d99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
@@ -129,7 +129,7 @@ public class ExportSemanticAnalyzer extends
BaseSemanticAnalyzer {
}
@Override
- public boolean hasAcidReadWrite() {
+ public boolean hasTransactionalInQuery() {
return isMmExport; // Full ACID export goes through UpdateDelete analyzer.
}
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 3feb2c8a0f5..2344b908ceb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -1446,10 +1446,10 @@ public class TestTxnCommands extends
TxnCommandsBaseForTests {
query = "select ROW__ID, a, b" + (isVectorized ? "" : ",
INPUT__FILE__NAME") + " from "
+ Table.NONACIDORCTBL + " order by ROW__ID";
String[][] expected2 = new String[][] {
- {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":0}\t1\t2",
"nonacidorctbl/base_10000001_v0000010/bucket_00001"},
- {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":1}\t0\t12",
"nonacidorctbl/base_10000001_v0000010/bucket_00001"},
- {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":2}\t1\t5",
"nonacidorctbl/base_10000001_v0000010/bucket_00001"},
- {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t1\t17",
"nonacidorctbl/base_10000001_v0000010/bucket_00001"}
+ {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":0}\t1\t2",
"nonacidorctbl/base_10000001_v0000009/bucket_00001"},
+ {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":1}\t0\t12",
"nonacidorctbl/base_10000001_v0000009/bucket_00001"},
+ {"{\"writeid\":0,\"bucketid\":536936448,\"rowid\":2}\t1\t5",
"nonacidorctbl/base_10000001_v0000009/bucket_00001"},
+ {"{\"writeid\":10000001,\"bucketid\":536936448,\"rowid\":0}\t1\t17",
"nonacidorctbl/base_10000001_v0000009/bucket_00001"}
};
checkResult(expected2, query, isVectorized, "after major compact", LOG);
//make sure they are the same before and after compaction
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 7bed9d2bc81..3386404cbf0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -727,9 +727,9 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
boolean sawNewBase = false;
for (int i = 0; i < status.length; i++) {
if (status[i].getPath().getName().matches("base_.*")) {
- //should be base_-9223372036854775808_v0000023 but 23 is a txn id not
write id so it makes
+ //should be base_-9223372036854775808_v0000008 but 8 is a txn id not
write id so it makes
//the tests fragile
-
Assert.assertTrue(status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000009"));
+
Assert.assertTrue(status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000008"));
sawNewBase = true;
FileStatus[] buckets = fs.listStatus(status[i].getPath(),
FileUtils.HIDDEN_FILES_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -781,7 +781,7 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
Assert.assertEquals("bucket_00001_0",
buckets[0].getPath().getName());
}
} else if (status[i].getPath().getName().matches("base_.*")) {
- Assert.assertTrue("base_-9223372036854775808",
status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000009"));//_v0000009
+ Assert.assertTrue("base_-9223372036854775808",
status[i].getPath().getName().startsWith("base_-9223372036854775808_v0000008"));//_v0000008
sawNewBase = true;
FileStatus[] buckets = fs.listStatus(status[i].getPath(),
FileUtils.HIDDEN_FILES_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -804,7 +804,7 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
// 5. Perform another major compaction
runStatementOnDriver("alter table "+ Table.NONACIDORCTBL + " compact
'MAJOR'");
runWorker(hiveConf);
- // There should be 1 new base directory: base_0000001
+ // There should be 1 new base directory: base_00000016
// Original bucket files, delta directories, delete_delta directories and
the
// previous base directory should stay until Cleaner kicks in.
status = fs.listStatus(new Path(getWarehouseDir() + "/" +
@@ -818,12 +818,12 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
FileStatus[] buckets = fs.listStatus(status[i].getPath(),
FileUtils.HIDDEN_FILES_PATH_FILTER);
Arrays.sort(buckets);
if (numBase == 1) {
- Assert.assertEquals("base_-9223372036854775808_v0000009",
status[i].getPath().getName());
+ Assert.assertEquals("base_-9223372036854775808_v0000008",
status[i].getPath().getName());
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
} else if (numBase == 2) {
// The new base dir now has two bucket files, since the delta dir
has two bucket files
- Assert.assertEquals("base_10000002_v0000018",
status[i].getPath().getName());
+ Assert.assertEquals("base_10000002_v0000016",
status[i].getPath().getName());
Assert.assertEquals(2, buckets.length);
Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
}
@@ -845,12 +845,12 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
Assert.assertEquals(7, status.length);
runCleaner(hiveConf);
runCleaner(hiveConf);
- // There should be only 1 directory left: base_0000001.
+ // There should be only 1 directory left: base_00000016
// Original bucket files, delta directories and previous base directory
should have been cleaned up.
status = fs.listStatus(new Path(getWarehouseDir() + "/" +
(Table.NONACIDORCTBL).toString().toLowerCase()),
FileUtils.HIDDEN_FILES_PATH_FILTER);
Assert.assertEquals(1, status.length);
- Assert.assertEquals("base_10000002_v0000018",
status[0].getPath().getName());
+ Assert.assertEquals("base_10000002_v0000016",
status[0].getPath().getName());
FileStatus[] buckets = fs.listStatus(status[0].getPath(),
FileUtils.HIDDEN_FILES_PATH_FILTER);
Arrays.sort(buckets);
Assert.assertEquals(2, buckets.length);
@@ -919,7 +919,7 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
for (int i = 0; i < status.length; i++) {
Path parent = status[i].getPath().getParent();
if (parent.getName().matches("base_.*")) {
-
Assert.assertTrue(parent.getName().startsWith("base_-9223372036854775808_v0000009"));
+
Assert.assertTrue(parent.getName().startsWith("base_-9223372036854775808_v0000008"));
sawNewBase = true;
FileStatus[] buckets = fs.listStatus(parent,
FileUtils.HIDDEN_FILES_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -970,7 +970,7 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
Assert.assertEquals("bucket_00001_0",
buckets[0].getPath().getName());
}
} else if (parent.getName().matches("base_.*")) {
- Assert.assertTrue("base_-9223372036854775808",
parent.getName().startsWith("base_-9223372036854775808_v0000009"));//_v0000009
+ Assert.assertTrue("base_-9223372036854775808",
parent.getName().startsWith("base_-9223372036854775808_v0000008"));//_v0000009
sawNewBase = true;
FileStatus[] buckets = fs.listStatus(parent,
FileUtils.HIDDEN_FILES_PATH_FILTER);
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
@@ -991,7 +991,7 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
// 5. Perform another major compaction
runStatementOnDriver("alter table "+ Table.NONACIDNESTEDPART + "
partition(p='p1',q='q1') compact 'MAJOR'");
runWorker(hiveConf);
- // There should be 1 new base directory: base_0000001
+ // There should be 1 new base directory: base_00000016
// Original bucket files, delta directories, delete_delta directories and
the
// previous base directory should stay until Cleaner kicks in.
status = listFilesByTable(fs, Table.NONACIDNESTEDPART);
@@ -1007,12 +1007,12 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
FileStatus[] buckets = fs.listStatus(parent,
FileUtils.HIDDEN_FILES_PATH_FILTER);
Arrays.sort(buckets);
if (numBase == 1) {
- Assert.assertEquals("base_-9223372036854775808_v0000009",
parent.getName());
+ Assert.assertEquals("base_-9223372036854775808_v0000008",
parent.getName());
Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
} else if (numBase == 2) {
// The new base dir now has two bucket files, since the delta dir
has two bucket files
- Assert.assertEquals("base_10000002_v0000018", parent.getName());
+ Assert.assertEquals("base_10000002_v0000016", parent.getName());
Assert.assertEquals(2, buckets.length);
Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
}
@@ -1034,11 +1034,11 @@ public class TestTxnCommands2 extends
TxnCommandsBaseForTests {
Assert.assertEquals(8, status.length);
runCleaner(hiveConf);
runCleaner(hiveConf);
- // There should be only 1 directory left: base_0000001.
+ // There should be only 1 directory left: base_00000016
// Original bucket files, delta directories and previous base directory
should have been cleaned up. Only one base with 2 files.
status = listFilesByTable(fs, Table.NONACIDNESTEDPART);
Assert.assertEquals(2, status.length);
- Assert.assertEquals("base_10000002_v0000018",
status[0].getPath().getParent().getName());
+ Assert.assertEquals("base_10000002_v0000016",
status[0].getPath().getParent().getName());
FileStatus[] buckets = fs.listStatus(status[0].getPath().getParent(),
FileUtils.HIDDEN_FILES_PATH_FILTER);
Arrays.sort(buckets);
Assert.assertEquals(2, buckets.length);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
index d3d829c0247..563b86a471f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
@@ -350,7 +350,7 @@ public class TestTxnCommands3 extends
TxnCommandsBaseForTests {
├── delta_0000001_0000001_0000
│ ├── _orc_acid_version
│ └── bucket_00000
- ├── delta_0000001_0000002_v0000010
+ ├── delta_0000001_0000002_v0000009
│ ├── _orc_acid_version
│ └── bucket_00000
└── delta_0000002_0000002_0000
@@ -362,7 +362,7 @@ public class TestTxnCommands3 extends
TxnCommandsBaseForTests {
FileUtils.HIDDEN_FILES_PATH_FILTER);
String[] expectedList = new String[] {
- "/t/delta_0000001_0000002_v0000010",
+ "/t/delta_0000001_0000002_v0000009",
"/t/delta_0000001_0000001_0000",
"/t/delta_0000002_0000002_0000",
};
@@ -390,13 +390,13 @@ public class TestTxnCommands3 extends
TxnCommandsBaseForTests {
runWorker(hiveConf);
/*
at this point delta_0000001_0000003_v0000012 is visible to everyone
- so cleaner removes all files shadowed by it (which is everything in this
case)
+ so cleaner removes all files shadowed by it
*/
runCleaner(hiveConf);
runCleaner(hiveConf);
expectedList = new String[] {
- "/t/delta_0000001_0000003_v0000014"
+ "/t/delta_0000001_0000003_v0000012"
};
actualList = fs.listStatus(new Path(warehousePath + "/t"),
FileUtils.HIDDEN_FILES_PATH_FILTER);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
index 49fccaf3642..6636ff252bd 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnExIm.java
@@ -345,11 +345,11 @@
target/tmp/org.apache.hadoop.hive.ql.TestTxnCommands-1521148657811/
TestTxnCommands2.runWorker(hiveConf);
String[][] expected3 = new String[][] {
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
- ".*t/delta_0000001_0000002_v000001[5-6]/bucket_00000"},
+ ".*t/delta_0000001_0000002_v000001[4-5]/bucket_00000"},
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
- ".*t/delta_0000001_0000002_v000001[5-6]/bucket_00000"},
+ ".*t/delta_0000001_0000002_v000001[4-5]/bucket_00000"},
{"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t0\t6",
- ".*t/delta_0000001_0000002_v000001[5-6]/bucket_00000"}};
+ ".*t/delta_0000001_0000002_v000001[4-5]/bucket_00000"}};
checkResult(expected3, testQuery, isVectorized, "minor compact imported
table");
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index 9fdb208f60b..d5d6a330f48 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -138,8 +138,8 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'minor'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected3 = new String[][] {
- {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t17",
"t/delta_0000001_0000004_v0000019/bucket_00000"},
- {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2",
"t/delta_0000001_0000004_v0000019/bucket_00000"}
+ {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t17",
"t/delta_0000001_0000004_v0000018/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2",
"t/delta_0000001_0000004_v0000018/bucket_00000"}
};
checkResult(expected3, testQuery, isVectorized, "delete compact minor");
@@ -172,9 +172,9 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected6 = new String[][]{
- {"{\"writeid\":7,\"bucketid\":536870913,\"rowid\":0}\t1\t17",
"t/base_0000009_v0000035/bucket_00000"},
- {"{\"writeid\":7,\"bucketid\":536936449,\"rowid\":0}\t1\t17",
"t/base_0000009_v0000035/bucket_00001"},
- {"{\"writeid\":9,\"bucketid\":536870912,\"rowid\":0}\t2\t2",
"t/base_0000009_v0000035/bucket_00000"}
+ {"{\"writeid\":7,\"bucketid\":536870913,\"rowid\":0}\t1\t17",
"t/base_0000009_v0000033/bucket_00000"},
+ {"{\"writeid\":7,\"bucketid\":536936449,\"rowid\":0}\t1\t17",
"t/base_0000009_v0000033/bucket_00001"},
+ {"{\"writeid\":9,\"bucketid\":536870912,\"rowid\":0}\t2\t2",
"t/base_0000009_v0000033/bucket_00000"}
};
checkResult(expected6, testQuery, isVectorized, "load data inpath compact
major");
}
@@ -208,10 +208,10 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'minor'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected1 = new String[][] {
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2",
"t/delta_0000001_0000002_v0000011/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4",
"t/delta_0000001_0000002_v0000011/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
"t/delta_0000001_0000002_v0000011/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
"t/delta_0000001_0000002_v0000011/bucket_00000"}
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2",
"t/delta_0000001_0000002_v0000010/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4",
"t/delta_0000001_0000002_v0000010/bucket_00000"},
+ {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
"t/delta_0000001_0000002_v0000010/bucket_00000"},
+ {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
"t/delta_0000001_0000002_v0000010/bucket_00000"}
};
checkResult(expected1, testQuery, isVectorized, "load data inpath
(minor)");
@@ -220,11 +220,11 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected2 = new String[][] {
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2",
"t/base_0000003_v0000017/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4",
"t/base_0000003_v0000017/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
"t/base_0000003_v0000017/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
"t/base_0000003_v0000017/bucket_00000"},
- {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2",
"t/base_0000003_v0000017/bucket_00000"}
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2",
"t/base_0000003_v0000015/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4",
"t/base_0000003_v0000015/bucket_00000"},
+ {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
"t/base_0000003_v0000015/bucket_00000"},
+ {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
"t/base_0000003_v0000015/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2",
"t/base_0000003_v0000015/bucket_00000"}
};
checkResult(expected2, testQuery, isVectorized, "load data inpath
(major)");
@@ -242,9 +242,9 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected4 = new String[][] {
- {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
"t/base_0000005_v0000026/bucket_00000"},
- {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":1}\t7\t8",
"t/base_0000005_v0000026/bucket_00000"},
- {"{\"writeid\":5,\"bucketid\":536870912,\"rowid\":0}\t6\t6",
"t/base_0000005_v0000026/bucket_00000"}};
+ {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
"t/base_0000005_v0000023/bucket_00000"},
+ {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":1}\t7\t8",
"t/base_0000005_v0000023/bucket_00000"},
+ {"{\"writeid\":5,\"bucketid\":536870912,\"rowid\":0}\t6\t6",
"t/base_0000005_v0000023/bucket_00000"}};
checkResult(expected4, testQuery, isVectorized, "load data inpath
overwrite (major)");
}
/**
@@ -322,13 +322,13 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
String[][] expected3 = new String[][] {
{"{\"writeid\":10000002,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
- "t/base_10000003_v0000014/bucket_00000"},
+ "t/base_10000003_v0000013/bucket_00000"},
{"{\"writeid\":10000002,\"bucketid\":536870912,\"rowid\":1}\t7\t8",
- "t/base_10000003_v0000014/bucket_00000"},
+ "t/base_10000003_v0000013/bucket_00000"},
{"{\"writeid\":10000002,\"bucketid\":536936448,\"rowid\":0}\t8\t8",
- "t/base_10000003_v0000014/bucket_00001"},
+ "t/base_10000003_v0000013/bucket_00001"},
{"{\"writeid\":10000003,\"bucketid\":536870912,\"rowid\":0}\t9\t9",
- "t/base_10000003_v0000014/bucket_00000"}
+ "t/base_10000003_v0000013/bucket_00000"}
};
checkResult(expected3, testQuery, isVectorized, "load data inpath
overwrite (major)");
}
@@ -451,10 +451,10 @@ public class TestTxnLoadData extends
TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected2 = new String[][] {
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
"t/base_0000001_v0000010/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
"t/base_0000001_v0000010/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t5\t5",
"t/base_0000001_v0000010/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t6\t6",
"t/base_0000001_v0000010/bucket_00000"}
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2",
"t/base_0000001_v0000009/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4",
"t/base_0000001_v0000009/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":0}\t5\t5",
"t/base_0000001_v0000009/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870913,\"rowid\":1}\t6\t6",
"t/base_0000001_v0000009/bucket_00000"}
};
checkResult(expected2, testQuery, isVectorized, "load data inpath
(major)");
//at lest for now, Load Data w/Overwrite is not allowed in a txn:
HIVE-18154
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index d140868ae8c..40a0a169744 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -164,10 +164,10 @@ public class TestTxnNoBuckets extends
TxnCommandsBaseForTests {
*/
String expected[][] = {
- {"{\"writeid\":2,\"bucketid\":536936449,\"rowid\":0}\t0\t0\t17",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00001"},
- {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t1\t17",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00001"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000012/bucket_00000"}
+ {"{\"writeid\":2,\"bucketid\":536936449,\"rowid\":0}\t0\t0\t17",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00001"},
+ {"{\"writeid\":2,\"bucketid\":536870913,\"rowid\":0}\t1\t1\t17",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00001"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3",
NO_BUCKETS_TBL_NAME + "/base_0000002_v0000011/bucket_00000"}
};
checkResult(expected,
"select ROW__ID, c1, c2, c3" + (shouldVectorize() ? "" : ",
INPUT__FILE__NAME")
@@ -182,8 +182,8 @@ public class TestTxnNoBuckets extends
TxnCommandsBaseForTests {
expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/delta_0000001_0000001_0000/bucket_00001_0");
expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/delta_0000002_0000002_0001/bucket_00000_0");
expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/delta_0000002_0000002_0001/bucket_00001_0");
- expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000012/bucket_00000");
- expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000012/bucket_00001");
+ expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000011/bucket_00000");
+ expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000011/bucket_00001");
assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/" +
NO_BUCKETS_TBL_NAME, NO_BUCKETS_TBL_NAME);
TestTxnCommands2.runCleaner(hiveConf);
@@ -192,8 +192,8 @@ public class TestTxnNoBuckets extends
TxnCommandsBaseForTests {
Assert.assertEquals("Unexpected result after clean",
stringifyValues(result), rs);
expectedFiles.clear();
- expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000012/bucket_00000");
- expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000012/bucket_00001");
+ expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000011/bucket_00000");
+ expectedFiles.add(NO_BUCKETS_TBL_NAME +
"/base_0000002_v0000011/bucket_00001");
assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/" +
NO_BUCKETS_TBL_NAME, NO_BUCKETS_TBL_NAME);
}
@@ -472,23 +472,23 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
/Users/ekoifman/dev/hiver
/*Compaction preserves location of rows wrt buckets/tranches (for now)*/
String expected4[][] = {
{"{\"writeid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2",
- "warehouse/t/base_10000002_v0000016/bucket_00002"},
+ "warehouse/t/base_10000002_v0000015/bucket_00002"},
{"{\"writeid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4",
- "warehouse/t/base_10000002_v0000016/bucket_00002"},
+ "warehouse/t/base_10000002_v0000015/bucket_00002"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":0}\t5\t6",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t9\t10",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t10\t20",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t12\t12",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t20\t40",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":5}\t50\t60",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
{"{\"writeid\":10000001,\"bucketid\":536870913,\"rowid\":0}\t60\t88",
- "warehouse/t/base_10000002_v0000016/bucket_00000"},
+ "warehouse/t/base_10000002_v0000015/bucket_00000"},
};
checkExpected(rs, expected4,"after major compact");
}
@@ -744,15 +744,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
/Users/ekoifman/dev/hiver
rs = runStatementOnDriver(query);
String[][] expected5 = {//the row__ids are the same after compaction
{"{\"writeid\":10000001,\"bucketid\":536870913,\"rowid\":0}\t1\t17",
- "warehouse/t/base_10000001_v0000018/bucket_00000"},
+ "warehouse/t/base_10000001_v0000017/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":1}\t2\t4",
- "warehouse/t/base_10000001_v0000018/bucket_00000"},
+ "warehouse/t/base_10000001_v0000017/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6",
- "warehouse/t/base_10000001_v0000018/bucket_00000"},
+ "warehouse/t/base_10000001_v0000017/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":3}\t6\t8",
- "warehouse/t/base_10000001_v0000018/bucket_00000"},
+ "warehouse/t/base_10000001_v0000017/bucket_00000"},
{"{\"writeid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10",
- "warehouse/t/base_10000001_v0000018/bucket_00000"}
+ "warehouse/t/base_10000001_v0000017/bucket_00000"}
};
checkExpected(rs, expected5, "After major compaction");
//vectorized because there is INPUT__FILE__NAME
@@ -816,10 +816,10 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree
/Users/ekoifman/dev/hiver
{"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3",
"t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000_0"},
{"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1",
"t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000_0"},
{"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3",
"t/p=1/q=1/delta_0000003_0000003_0000/bucket_00000_0"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2",
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4",
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"},
- {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2",
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"},
- {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4",
"t/p=1/q=2/base_0000003_v0000011/bucket_00000"}
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2",
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4",
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2",
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4",
"t/p=1/q=2/base_0000003_v0000010/bucket_00000"}
};
checkExpected(rs, expected2, "after major compaction");
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index c819c477f88..42be09529d3 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -888,7 +888,6 @@ public class TestDbTxnManager2 extends
DbTxnManagerEndToEndTestBase {
conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
HiveTxnManager txnMgr =
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
- txnMgr.openTxn(ctx, "T1");
driver.compileAndRespond("insert into tab_not_acid partition(np='blah')
values(7,8)", true);
((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T1", false);
List<ShowLocksResponseElement> locks = getLocks(txnMgr);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
index 865365722fc..e18feedbf56 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -225,6 +225,17 @@ public class DatabaseProduct implements Configurable {
|| (isDERBY() && "42X05".equalsIgnoreCase(e.getSQLState()));
}
+ public String toVarChar(String column) {
+ switch (dbType) {
+ case DERBY:
+ return String.format("CAST(%s AS VARCHAR(4000))", column);
+ case ORACLE:
+ return String.format("to_char(%s)", column);
+ default:
+ return column;
+ }
+ }
+
/**
* Whether the RDBMS has restrictions on IN list size (explicit, or poor
perf-based).
*/
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index d910504a9d3..35079df61d3 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -3384,7 +3384,7 @@ class MetaStoreDirectSql {
String statement = TxnUtils.createUpdatePreparedStmt(
"\"TABLE_PARAMS\"",
ImmutableList.of("\"PARAM_VALUE\""),
- ImmutableList.of("\"TBL_ID\"", "\"PARAM_KEY\"", "\"PARAM_VALUE\""));
+ ImmutableList.of("\"TBL_ID\"", "\"PARAM_KEY\"",
dbType.toVarChar("\"PARAM_VALUE\"")));
Query query = pm.newQuery("javax.jdo.query.SQL", statement);
return (long) query.executeWithArray(newValue, table.getId(), key,
expectedValue);
}
diff --git a/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
b/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
index b3c83b95b06..d11f7950be4 100644
--- a/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
+++ b/streaming/src/java/org/apache/hive/streaming/TransactionBatch.java
@@ -21,6 +21,7 @@ package org.apache.hive.streaming;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.LockComponentBuilder;
import org.apache.hadoop.hive.metastore.LockRequestBuilder;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -435,18 +436,25 @@ public class TransactionBatch extends
AbstractStreamingTransaction {
}
}
- private static LockRequest createLockRequest(final HiveStreamingConnection
connection,
+ private static LockRequest createLockRequest(final HiveStreamingConnection
conn,
String partNameForLock, String user, long txnId, String agentInfo) {
LockRequestBuilder requestBuilder = new LockRequestBuilder(agentInfo);
requestBuilder.setUser(user);
requestBuilder.setTransactionId(txnId);
LockComponentBuilder lockCompBuilder = new LockComponentBuilder()
- .setDbName(connection.getDatabase())
- .setTableName(connection.getTable().getTableName())
+ .setDbName(conn.getDatabase())
+ .setTableName(conn.getTable().getTableName())
.setSharedRead()
.setOperationType(DataOperationType.INSERT);
- if (connection.isDynamicPartitioning()) {
+
+ boolean isLocklessReadsEnabled =
conn.getConf().getBoolVar(HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
+ boolean sharedWrite =
!conn.getConf().getBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK);
+
+ if (sharedWrite || isLocklessReadsEnabled) {
+ lockCompBuilder.setSharedWrite();
+ }
+ if (conn.isDynamicPartitioning()) {
lockCompBuilder.setIsDynamicPartitionWrite(true);
}
if (partNameForLock != null && !partNameForLock.isEmpty()) {