[hive] branch master updated: HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed by Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
tchoi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ac1cfe2294d HIVE-26555 Read-only mode for Hive database (#3614) (Teddy
Choi, reviewed by Laszlo Bodor)
ac1cfe2294d is described below
commit ac1cfe2294d1b7b8c02b60899d69868920ede0f5
Author: Teddy Choi
AuthorDate: Wed Jan 4 13:35:57 2023 +0900
HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed
by Laszlo Bodor)
---
.../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 1 +
.../hive/ql/hooks/EnforceReadOnlyDatabaseHook.java | 196 +
.../test/queries/clientnegative/read_only_delete.q | 23 +++
.../test/queries/clientnegative/read_only_insert.q | 23 +++
.../test/queries/clientpositive/read_only_hook.q | 27 +++
.../results/clientnegative/read_only_delete.q.out | 34
.../results/clientnegative/read_only_insert.q.out | 35
.../clientpositive/llap/read_only_hook.q.out | 71
8 files changed, 410 insertions(+)
diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 1a3ad38acef..5e4fbf8e642 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -485,6 +485,7 @@ public enum ErrorMsg {
CTLF_UNSUPPORTED_FORMAT(10434, "CREATE TABLE LIKE FILE is not supported by
the ''{0}'' file format", true),
NON_NATIVE_ACID_UPDATE(10435, "Update and Merge into non-native ACID table
is only supported when " +
HiveConf.ConfVars.SPLIT_UPDATE.varname + " is true."),
+ READ_ONLY_DATABASE(10436, "Database {0} is read-only", true),
//== 2 range starts here
//
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java
new file mode 100644
index 000..c343fc7b14f
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
+import org.apache.hadoop.hive.ql.ddl.DDLWork;
+import
org.apache.hadoop.hive.ql.ddl.database.alter.poperties.AlterDatabaseSetPropertiesDesc;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * EnforceReadOnlyDatabaseHook is a hook that disallows write operations on
read-only databases.
+ * It's enforced when "hive.exec.pre.hooks" has
"org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyDatabaseHook" and
+ * a database has 'readonly'='true' property.
+ */
+public class EnforceReadOnlyDatabaseHook implements ExecuteWithHookContext {
+ public static final String READONLY = "readonly";
+ private static final Logger LOG =
LoggerFactory.getLogger(EnforceReadOnlyDatabaseHook.class);
+
+ @Override
+ public void run(HookContext hookContext) throws Exception {
+assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK);
+final QueryState queryState = hookContext.getQueryState();
+final HiveOperation hiveOperation = queryState.getHiveOperation();
+
+// Allow read-only type operations, excluding query.
+// A query can be EXPLAIN, SELECT, INSERT, UPDATE, or DELETE.
+if (isReadOnlyOperation(hiveOperation)) {
+ return;
+}
+
+// Allow
[hive] branch master updated (67906e855da -> 62893c0db6c)
This is an automated email from the ASF dual-hosted git repository. dengzh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git from 67906e855da HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795) add 62893c0db6c HIVE-26790: Update protobuf/grpc versions (#3814) (Henri Biestro, reviewed by Zhihua Deng and Chris Nauroth) No new revisions were added by this update. Summary of changes: pom.xml | 2 +- standalone-metastore/metastore-common/pom.xml | 2 +- standalone-metastore/pom.xml | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-)
[hive] branch master updated: HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795)
This is an automated email from the ASF dual-hosted git repository.
rameshkumar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 67906e855da HIVE-23559: Optimise Hive::moveAcidFiles for cloud
storage. (#3795)
67906e855da is described below
commit 67906e855da46835fe3c4bce22378670efc7b411
Author: Dmitriy Fingerman
AuthorDate: Tue Jan 3 13:18:56 2023 -0500
HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795)
Co-authored-by: Dmitriy Fingerman
---
.../org/apache/hadoop/hive/ql/exec/MoveTask.java | 2 +-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 156 ++---
2 files changed, 105 insertions(+), 53 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index a5a5ea793d6..555bd7fb5b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -388,7 +388,7 @@ public class MoveTask extends Task implements
Serializable {
FileSystem srcFs = sourcePath.getFileSystem(conf);
FileStatus[] srcs = srcFs.globStatus(sourcePath);
if(srcs != null) {
- Hive.moveAcidFiles(srcFs, srcs, targetPath, null);
+ Hive.moveAcidFiles(srcFs, srcs, targetPath, null, conf);
} else {
LOG.debug("No files found to move from " + sourcePath + " to " +
targetPath);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 238b24d2641..d9301c3ea4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -5102,7 +5102,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
// If we're moving files around for an ACID write then the rules and paths
are all different.
// You can blame this on Owen.
if (isAcidIUD) {
- moveAcidFiles(srcFs, srcs, destf, newFiles);
+ moveAcidFiles(srcFs, srcs, destf, newFiles, conf);
} else {
// For ACID non-bucketed case, the filenames have to be in the format
consistent with INSERT/UPDATE/DELETE Ops,
// i.e, like 00_0, 01_0_copy_1, 02_0.gz etc.
@@ -5124,7 +5124,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
public static void moveAcidFiles(FileSystem fs, FileStatus[] stats, Path dst,
-List newFiles) throws HiveException {
+List newFiles, HiveConf conf) throws
HiveException {
// The layout for ACID files is
table|partname/base|delta|delete_delta/bucket
// We will always only be writing delta files ( except IOW which writes
base_X/ ).
// In the buckets created by FileSinkOperator
@@ -5185,78 +5185,130 @@ private void constructOneLBLocationMap(FileStatus fSta,
for (FileStatus origBucketStat : origBucketStats) {
Path origBucketPath = origBucketStat.getPath();
moveAcidFiles(AcidUtils.DELTA_PREFIX, AcidUtils.deltaFileFilter,
-fs, dst, origBucketPath, createdDeltaDirs, newFiles);
+fs, dst, origBucketPath, createdDeltaDirs, newFiles, conf);
moveAcidFiles(AcidUtils.DELETE_DELTA_PREFIX,
AcidUtils.deleteEventDeltaDirFilter,
-fs, dst,origBucketPath, createdDeltaDirs, newFiles);
+fs, dst,origBucketPath, createdDeltaDirs, newFiles, conf);
moveAcidFiles(AcidUtils.BASE_PREFIX, AcidUtils.baseFileFilter,//for
Insert Overwrite
-fs, dst, origBucketPath, createdDeltaDirs, newFiles);
+fs, dst, origBucketPath, createdDeltaDirs, newFiles, conf);
}
}
}
private static void moveAcidFiles(String deltaFileType, PathFilter
pathFilter, FileSystem fs,
Path dst, Path origBucketPath, Set
createdDeltaDirs,
-List newFiles) throws HiveException {
-LOG.debug("Acid move looking for " + deltaFileType + " files in bucket " +
origBucketPath);
+List newFiles, HiveConf conf) throws
HiveException {
-FileStatus[] deltaStats = null;
-try {
- deltaStats = fs.listStatus(origBucketPath, pathFilter);
-} catch (IOException e) {
- throw new HiveException("Unable to look for " + deltaFileType + " files
in original bucket " +
- origBucketPath.toUri().toString(), e);
-}
-LOG.debug("Acid move found " + deltaStats.length + " " + deltaFileType + "
files");
-
-for (FileStatus deltaStat : deltaStats) {
- Path deltaPath = deltaStat.getPath();
- // Create the delta directory. Don't worry if it already exists,
- // as that likely means another task got to it first.
