kasakrisz commented on code in PR #4575:
URL: https://github.com/apache/hive/pull/4575#discussion_r1294188752
##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java:
##########
@@ -405,45 +380,63 @@ private Set<FileStatus> listForCommits(JobConf jobConf,
String jobLocation) thro
* @param executor The executor used to read the forCommit files
* @param outputTable The table used for loading from the catalog
*/
- private void commitTable(FileIO io, ExecutorService executor, OutputTable
outputTable) {
+ private void commitTable(FileIO io, ExecutorService executor, OutputTable
outputTable,
+ List<JobContext> jobContextList) {
String name = outputTable.tableName;
- JobContext jobContext = outputTable.jobContext;
- JobConf conf = jobContext.getJobConf();
Properties catalogProperties = new Properties();
catalogProperties.put(Catalogs.NAME, name);
catalogProperties.put(Catalogs.LOCATION, outputTable.table.location());
if (outputTable.catalogName != null) {
catalogProperties.put(InputFormatConfig.CATALOG_NAME,
outputTable.catalogName);
}
- Table table = Catalogs.loadTable(conf, catalogProperties);
+ Collection<DataFile> dataFiles = Lists.newArrayList();
+ Collection<DeleteFile> deleteFiles = Lists.newArrayList();
+
+ Table table = null;
+ String branchName = null;
+ boolean isOverWrite = false;
+
+ for (JobContext jobContext : jobContextList) {
+ JobConf conf = jobContext.getJobConf();
+ table = Optional.ofNullable(table).orElse(Catalogs.loadTable(conf,
catalogProperties));
+
+ branchName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
+ isOverWrite = conf.getBoolean(InputFormatConfig.IS_OVERWRITE, false);
Review Comment:
Is it possible that some `jobContext` has `isOverwrite` `true` and some of
them is `false`? If the answer is yes then this local variable has the last
`jobContext`'s value. Is this expected?
##########
iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergV2.java:
##########
@@ -646,6 +652,33 @@ public void testDMLFailsForCopyOnMergeDeleteMode() {
shell.executeStatement("select count(*) from customers");
}
+ @Test
+ public void testConcurrent2Updates() {
+ Assume.assumeTrue(fileFormat == FileFormat.PARQUET && isVectorized &&
+ testTableType == TestTables.TestTableType.HIVE_CATALOG);
+
+ testTables.createTable(shell, "customers",
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), fileFormat,
HiveIcebergStorageHandlerTestUtils.OTHER_CUSTOMER_RECORDS_2, 2);
+ String sql = "UPDATE customers SET last_name='Changed' WHERE customer_id=3
or first_name='Joanna'";
+ try {
+ Tasks.range(2)
+ .executeWith(Executors.newFixedThreadPool(2))
+ .run(i -> {
+ init(shell, testTables, temp, executionEngine);
+ HiveConf.setBoolVar(shell.getHiveConf(),
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, isVectorized);
+ HiveConf.setVar(shell.getHiveConf(),
HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "none");
+ shell.executeStatement(sql);
+ shell.closeSession();
+ });
+ } catch (Throwable ex) {
+ Throwable cause = Throwables.getRootCause(ex);
+ Assert.assertTrue(cause instanceof ValidationException);
+ Assert.assertTrue(cause.getMessage().startsWith("Found new conflicting
delete files"));
+ }
+ List<Object[]> res = shell.executeStatement("SELECT * FROM customers WHERE
last_name='Changed'");
+ Assert.assertEquals(5, res.size());
+ }
Review Comment:
Is this test flaky?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
