This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 88c8cc45116 HIVE-29436: Iceberg materialized view rebuild locking
issue (#6330)
88c8cc45116 is described below
commit 88c8cc4511689f1c3d55303c23d60495002b6b69
Author: Krisztian Kasa <[email protected]>
AuthorDate: Tue Apr 14 06:11:36 2026 +0200
HIVE-29436: Iceberg materialized view rebuild locking issue (#6330)
---
.../src/test/queries/positive/mv_iceberg_orc2.q | 13 ++
.../src/test/queries/positive/mv_iceberg_orc6.q | 7 +
.../test/results/positive/mv_iceberg_orc2.q.out | 74 +++++++++
.../test/results/positive/mv_iceberg_orc6.q.out | 33 ++++
.../TestMaterializedViewRebuildWithMiniHS2.java | 174 +++++++++++++++++++++
.../AlterMaterializedViewRebuildAnalyzer.java | 3 +-
.../apache/hadoop/hive/ql/plan/HiveOperation.java | 2 +-
7 files changed, 303 insertions(+), 3 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc2.q
b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc2.q
index f0fd54ffbfe..8c4c3142de9 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc2.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc2.q
@@ -19,7 +19,11 @@ select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c >
52;
create materialized view mat1 stored by iceberg stored as orc tblproperties
('format-version'='1') as
select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c > 52;
+create materialized view mat2 stored by iceberg stored as orc tblproperties
('format-version'='1') as
+select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c <= 52;
+
select * from mat1;
+select * from mat2;
alter materialized view mat1 disable rewrite;
@@ -38,6 +42,15 @@ insert into tbl_ice values (10, 'ten', 60);
explain cbo
alter materialized view mat1 rebuild;
+alter materialized view mat1 rebuild;
+alter materialized view mat2 rebuild;
+
+select * from mat1;
+select * from mat2;
+
+-- test subsequent MV rebuilds
+insert into tbl_ice values (20, 'twenty', 80);
+
alter materialized view mat1 rebuild;
select * from mat1;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc6.q
b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc6.q
index 70fdc943575..65ca8ce6e29 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc6.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc6.q
@@ -30,3 +30,10 @@ alter materialized view mat1 rebuild;
alter materialized view mat1 rebuild;
select * from mat1;
+
+-- test subsequent MV rebuilds
+insert into tbl_ice values (1, 'twenty', 80);
+
+alter materialized view mat1 rebuild;
+
+select * from mat1;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out
b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out
index e7ea0e62530..66dcca6edcf 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc2.q.out
@@ -168,6 +168,22 @@ POSTHOOK: Output: default@mat1
POSTHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: Lineage: mat1.b SIMPLE [(tbl_ice)tbl_ice.FieldSchema(name:b,
type:string, comment:null), ]
POSTHOOK: Lineage: mat1.c SIMPLE [(tbl_ice)tbl_ice.FieldSchema(name:c,
type:int, comment:null), ]
+PREHOOK: query: create materialized view mat2 stored by iceberg stored as orc
tblproperties ('format-version'='1') as
+select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c <= 52
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@tbl_ice
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mat2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: create materialized view mat2 stored by iceberg stored as orc
tblproperties ('format-version'='1') as
+select tbl_ice.b, tbl_ice.c from tbl_ice where tbl_ice.c <= 52
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@tbl_ice
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mat2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: Lineage: mat2.b SIMPLE [(tbl_ice)tbl_ice.FieldSchema(name:b,
type:string, comment:null), ]
+POSTHOOK: Lineage: mat2.c SIMPLE [(tbl_ice)tbl_ice.FieldSchema(name:c,
type:int, comment:null), ]
PREHOOK: query: select * from mat1
PREHOOK: type: QUERY
PREHOOK: Input: default@mat1
@@ -178,6 +194,17 @@ POSTHOOK: Input: default@mat1
POSTHOOK: Output: hdfs://### HDFS PATH ###
five 54
four 53
+PREHOOK: query: select * from mat2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mat2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from mat2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mat2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+one 50
+three 52
+two 51
PREHOOK: query: alter materialized view mat1 disable rewrite
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
PREHOOK: Input: default@mat1
@@ -248,6 +275,52 @@ HiveProject(tbl_ice.b=[$1], tbl_ice.c=[$2])
HiveFilter(condition=[>($2, 52)])
HiveTableScan(table=[[default, tbl_ice]], table:alias=[tbl_ice],
fromVersion=[#Masked#])
+PREHOOK: query: alter materialized view mat1 rebuild
+PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
+PREHOOK: Input: default@tbl_ice
+PREHOOK: Output: default@mat1
+POSTHOOK: query: alter materialized view mat1 rebuild
+POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
+POSTHOOK: Input: default@tbl_ice
+POSTHOOK: Output: default@mat1
+PREHOOK: query: alter materialized view mat2 rebuild
+PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
+PREHOOK: Input: default@tbl_ice
+PREHOOK: Output: default@mat2
+POSTHOOK: query: alter materialized view mat2 rebuild
+POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
+POSTHOOK: Input: default@tbl_ice
+POSTHOOK: Output: default@mat2
+PREHOOK: query: select * from mat1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mat1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from mat1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mat1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+five 54
+four 53
+ten 60
+PREHOOK: query: select * from mat2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mat2
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from mat2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mat2
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+one 50
+three 52
+two 51
+PREHOOK: query: insert into tbl_ice values (20, 'twenty', 80)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tbl_ice
+POSTHOOK: query: insert into tbl_ice values (20, 'twenty', 80)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tbl_ice
PREHOOK: query: alter materialized view mat1 rebuild
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
PREHOOK: Input: default@tbl_ice
@@ -267,3 +340,4 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
five 54
four 53
ten 60
+twenty 80
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out
b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out
index 35904d2a958..e81eace0ee8 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc6.q.out
@@ -112,3 +112,36 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@mat1
POSTHOOK: Output: hdfs://### HDFS PATH ###
five 54 54
+PREHOOK: query: insert into tbl_ice values (1, 'twenty', 80)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tbl_ice
+POSTHOOK: query: insert into tbl_ice values (1, 'twenty', 80)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tbl_ice
+PREHOOK: query: alter materialized view mat1 rebuild
+PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
+PREHOOK: Input: default@mat1
+PREHOOK: Input: default@tbl_ice
+PREHOOK: Input: default@tbl_ice_v2
+PREHOOK: Output: default@mat1
+POSTHOOK: query: alter materialized view mat1 rebuild
+POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
+POSTHOOK: Input: default@mat1
+POSTHOOK: Input: default@tbl_ice
+POSTHOOK: Input: default@tbl_ice_v2
+POSTHOOK: Output: default@mat1
+POSTHOOK: Lineage: mat1._c2 EXPRESSION
[(tbl_ice_v2)tbl_ice_v2.FieldSchema(name:f, type:int, comment:null),
(mat1)default.mat1.FieldSchema(name:_c2, type:bigint, comment:null), ]
+POSTHOOK: Lineage: mat1.b EXPRESSION [(tbl_ice)tbl_ice.FieldSchema(name:b,
type:string, comment:null), (mat1)default.mat1.FieldSchema(name:b, type:string,
comment:null), ]
+POSTHOOK: Lineage: mat1.c EXPRESSION [(tbl_ice)tbl_ice.FieldSchema(name:c,
type:int, comment:null), (mat1)default.mat1.FieldSchema(name:c, type:int,
comment:null), ]
+PREHOOK: query: select * from mat1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mat1
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from mat1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mat1
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+five 54 54
+twenty 80 50
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuildWithMiniHS2.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuildWithMiniHS2.java
new file mode 100644
index 00000000000..f3511a916ee
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMaterializedViewRebuildWithMiniHS2.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn.compactor;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConfForTest;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+public class TestMaterializedViewRebuildWithMiniHS2 {
+
+ private static MiniHS2 miniHS2 = null;
+
+ private static final String TABLE1 = "t1";
+ private static final String MV1 = "mat1";
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ HiveConfForTest hiveConf = new
HiveConfForTest(TestMaterializedViewRebuildWithMiniHS2.class);
+
+ String testWarehouseDir = hiveConf.getTestDataDir() + "/warehouse";
+ File f = new File(testWarehouseDir);
+ if (f.exists()) {
+ FileUtil.fullyDelete(f);
+ }
+ if (!(new File(testWarehouseDir).mkdirs())) {
+ throw new RuntimeException("Could not create " + testWarehouseDir);
+ }
+ MetastoreConf.setVar(hiveConf, MetastoreConf.ConfVars.WAREHOUSE,
testWarehouseDir);
+
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
+ hiveConf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER,
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+
+
+ miniHS2 = new MiniHS2.Builder()
+ .withConf(hiveConf)
+ .withRemoteMetastore(true)
+ .build();
+
+ miniHS2.start(new HashMap<>());
+ hiveConf.set("iceberg.engine.hive.lock-enabled", "true");
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (miniHS2 != null && miniHS2.isStarted()) {
+ miniHS2.stop();
+ }
+ }
+
+ @Test
+ public void testConcurrentMVRebuildWhenRollback() throws Exception {
+ String connectionUrl = miniHS2.getJdbcURL();
+
+ try (Connection connection = DriverManager.getConnection(connectionUrl,
"user1", "pass1");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("drop materialized view if exists " + MV1);
+ statement.execute("drop table if exists " + TABLE1);
+
+ statement.execute("create table " + TABLE1 + " (a int, b int) stored by
iceberg stored as orc");
+ statement.execute("insert into " + TABLE1 + " values (0, 1), (2, 20)");
+ statement.execute("create materialized view " + MV1 +
+ " stored by iceberg stored as orc as select a, b from " + TABLE1
+ " where a > 1");
+ statement.execute("insert into " + TABLE1 + " values (3, 3)");
+ }
+
+ assertResults(connectionUrl, MV1, Set.of("2 20"));
+
+ try (Connection connection = DriverManager.getConnection(connectionUrl,
"user1", "pass1");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("start transaction");
+ statement.execute("alter materialized view " + MV1 + " rebuild");
+
+ try (Connection connection2 = DriverManager.getConnection(connectionUrl,
"user2", "pass1");
+ Statement statement2 = connection2.createStatement()) {
+ statement2.execute("start transaction");
+ statement2.execute("alter materialized view " + MV1 + " rebuild");
+ statement2.execute("commit");
+ }
+
+ statement.execute("rollback");
+ }
+
+ assertResults(connectionUrl, MV1, Set.of("2 20"));
+ }
+
+ @Test
+ public void testConcurrentMVRebuildWhenCommit() throws Exception {
+ String connectionUrl = miniHS2.getJdbcURL();
+
+ try (Connection connection = DriverManager.getConnection(connectionUrl,
"user1", "pass1");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("drop materialized view if exists " + MV1);
+ statement.execute("drop table if exists " + TABLE1);
+
+ statement.execute("create table " + TABLE1 + " (a int, b int) stored by
iceberg stored as orc");
+ statement.execute("insert into " + TABLE1 + " values (0, 1), (2, 20)");
+ statement.execute("create materialized view " + MV1 +
+ " stored by iceberg stored as orc as select a, b from " + TABLE1
+ " where a > 1");
+ statement.execute("insert into " + TABLE1 + " values (3, 3)");
+ }
+
+ assertResults(connectionUrl, MV1, Set.of("2 20"));
+
+ try (Connection connection = DriverManager.getConnection(connectionUrl,
"user1", "pass1");
+ Statement statement = connection.createStatement()) {
+
+ statement.execute("start transaction");
+ statement.execute("alter materialized view " + MV1 + " rebuild");
+
+ try (Connection connection2 = DriverManager.getConnection(connectionUrl,
"user2", "pass1");
+ Statement statement2 = connection2.createStatement()) {
+ statement2.execute("start transaction");
+ statement2.execute("alter materialized view " + MV1 + " rebuild");
+ statement2.execute("commit");
+ }
+
+ statement.execute("commit");
+ }
+
+ assertResults(connectionUrl, MV1, Set.of("2 20", "3 3"));
+ }
+
+ private void assertResults(String connectionUrl, String tableName,
Set<String> expected) throws SQLException {
+ try (Connection connection = DriverManager.getConnection(connectionUrl,
"user1", "pass1");
+ Statement statement = connection.createStatement()) {
+
+ Set<String> expectedCopy = new HashSet<>(expected);
+
+ ResultSet resultSet = statement.executeQuery("select * from " +
tableName);
+ while (resultSet.next()) {
+ String line = resultSet.getInt(1) + " " + resultSet.getInt(2);
+ if (!expectedCopy.remove(line)) {
+ throw new AssertionError("Actual record not expected: " + line);
+ }
+ }
+ if (!expectedCopy.isEmpty()) {
+ throw new AssertionError("Expected record(s) not found: " +
String.join(System.lineSeparator(), expectedCopy));
+ }
+ }
+ }
+}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
index 26022004627..a0cf25fbbaf 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
@@ -207,8 +207,7 @@ private ASTNode getRewrittenAST(TableName tableName) throws
SemanticException {
rewrittenAST = ParseUtils.parse(rewrittenInsertStatement, ctx);
this.ctx.addSubContext(ctx);
- if (!this.ctx.isExplainPlan() && (AcidUtils.isTransactionalTable(table)
||
- table.isNonNative() &&
table.getStorageHandler().areSnapshotsSupported())) {
+ if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(table)) {
// Acquire lock for the given materialized view. Only one rebuild per
materialized view can be triggered at a
// given time, as otherwise we might produce incorrect results if
incremental maintenance is triggered.
HiveTxnManager txnManager = getTxnMgr();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
index 25114a46a6e..fcc8da5021b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
@@ -145,7 +145,7 @@ public enum HiveOperation {
ALTER_MATERIALIZED_VIEW_REWRITE("ALTER_MATERIALIZED_VIEW_REWRITE",
HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REWRITE,
new Privilege[]{Privilege.ALTER_METADATA}, null),
ALTER_MATERIALIZED_VIEW_REBUILD("ALTER_MATERIALIZED_VIEW_REBUILD",
HiveParser.TOK_ALTER_MATERIALIZED_VIEW_REBUILD,
- new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.CREATE,
Privilege.DROP}),
+ new Privilege[]{Privilege.SELECT}, new Privilege[]{Privilege.CREATE,
Privilege.DROP},true, false),
ALTERVIEW_PROPERTIES("ALTERVIEW_PROPERTIES",
new int[] {HiveParser.TOK_ALTERVIEW_PROPERTIES,
HiveParser.TOK_ALTERVIEW_DROPPROPERTIES}, null, null),
LOCKTABLE("LOCKTABLE", HiveParser.TOK_LOCKTABLE, new
Privilege[]{Privilege.LOCK}, null),
