[hive] branch master updated: HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed by Laszlo Bodor)

2023-01-03 Thread tchoi
This is an automated email from the ASF dual-hosted git repository.

tchoi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new ac1cfe2294d HIVE-26555 Read-only mode for Hive database (#3614) (Teddy 
Choi, reviewed by Laszlo Bodor)
ac1cfe2294d is described below

commit ac1cfe2294d1b7b8c02b60899d69868920ede0f5
Author: Teddy Choi 
AuthorDate: Wed Jan 4 13:35:57 2023 +0900

HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed 
by Laszlo Bodor)
---
 .../java/org/apache/hadoop/hive/ql/ErrorMsg.java   |   1 +
 .../hive/ql/hooks/EnforceReadOnlyDatabaseHook.java | 196 +
 .../test/queries/clientnegative/read_only_delete.q |  23 +++
 .../test/queries/clientnegative/read_only_insert.q |  23 +++
 .../test/queries/clientpositive/read_only_hook.q   |  27 +++
 .../results/clientnegative/read_only_delete.q.out  |  34 
 .../results/clientnegative/read_only_insert.q.out  |  35 
 .../clientpositive/llap/read_only_hook.q.out   |  71 
 8 files changed, 410 insertions(+)

diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java 
b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 1a3ad38acef..5e4fbf8e642 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -485,6 +485,7 @@ public enum ErrorMsg {
   CTLF_UNSUPPORTED_FORMAT(10434, "CREATE TABLE LIKE FILE is not supported by 
the ''{0}'' file format", true),
   NON_NATIVE_ACID_UPDATE(10435, "Update and Merge into non-native ACID table 
is only supported when " +
   HiveConf.ConfVars.SPLIT_UPDATE.varname + " is true."),
+  READ_ONLY_DATABASE(10436, "Database {0} is read-only", true),
 
   //== 2 range starts here 
//
 
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java
new file mode 100644
index 000..c343fc7b14f
--- /dev/null
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
+import org.apache.hadoop.hive.ql.ddl.DDLWork;
+import 
org.apache.hadoop.hive.ql.ddl.database.alter.poperties.AlterDatabaseSetPropertiesDesc;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * EnforceReadOnlyDatabaseHook is a hook that disallows write operations on 
read-only databases.
+ * It's enforced when "hive.exec.pre.hooks" has 
"org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyDatabaseHook" and
+ * a database has 'readonly'='true' property.
+ */
+public class EnforceReadOnlyDatabaseHook implements ExecuteWithHookContext {
+  public static final String READONLY = "readonly";
+  private static final Logger LOG = 
LoggerFactory.getLogger(EnforceReadOnlyDatabaseHook.class);
+
+  @Override
+  public void run(HookContext hookContext) throws Exception {
+assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK);
+final QueryState queryState = hookContext.getQueryState();
+final HiveOperation hiveOperation = queryState.getHiveOperation();
+
+// Allow read-only type operations, excluding query.
+// A query can be EXPLAIN, SELECT, INSERT, UPDATE, or DELETE.
+if (isReadOnlyOperation(hiveOperation)) {
+  return;
+}
+
+// Allow 

[hive] branch master updated (67906e855da -> 62893c0db6c)

2023-01-03 Thread dengzh
This is an automated email from the ASF dual-hosted git repository.

dengzh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


from 67906e855da HIVE-23559: Optimise Hive::moveAcidFiles for cloud 
storage. (#3795)
 add 62893c0db6c HIVE-26790: Update protobuf/grpc versions (#3814) (Henri 
Biestro, reviewed by Zhihua Deng and Chris Nauroth)

No new revisions were added by this update.

Summary of changes:
 pom.xml   | 2 +-
 standalone-metastore/metastore-common/pom.xml | 2 +-
 standalone-metastore/pom.xml  | 5 ++---
 3 files changed, 4 insertions(+), 5 deletions(-)



[hive] branch master updated: HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795)

2023-01-03 Thread rameshkumar
This is an automated email from the ASF dual-hosted git repository.

rameshkumar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
 new 67906e855da HIVE-23559: Optimise Hive::moveAcidFiles for cloud 
storage. (#3795)
67906e855da is described below

commit 67906e855da46835fe3c4bce22378670efc7b411
Author: Dmitriy Fingerman 
AuthorDate: Tue Jan 3 13:18:56 2023 -0500

HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795)

Co-authored-by: Dmitriy Fingerman 
---
 .../org/apache/hadoop/hive/ql/exec/MoveTask.java   |   2 +-
 .../org/apache/hadoop/hive/ql/metadata/Hive.java   | 156 ++---
 2 files changed, 105 insertions(+), 53 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index a5a5ea793d6..555bd7fb5b6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -388,7 +388,7 @@ public class MoveTask extends Task implements 
Serializable {
 FileSystem srcFs = sourcePath.getFileSystem(conf);
 FileStatus[] srcs = srcFs.globStatus(sourcePath);
 if(srcs != null) {
-  Hive.moveAcidFiles(srcFs, srcs, targetPath, null);
+  Hive.moveAcidFiles(srcFs, srcs, targetPath, null, conf);
 } else {
   LOG.debug("No files found to move from " + sourcePath + " to " + 
targetPath);
 }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 238b24d2641..d9301c3ea4b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -5102,7 +5102,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
 // If we're moving files around for an ACID write then the rules and paths 
are all different.
 // You can blame this on Owen.
 if (isAcidIUD) {
-  moveAcidFiles(srcFs, srcs, destf, newFiles);
+  moveAcidFiles(srcFs, srcs, destf, newFiles, conf);
 } else {
   // For ACID non-bucketed case, the filenames have to be in the format 
consistent with INSERT/UPDATE/DELETE Ops,
   // i.e, like 00_0, 01_0_copy_1, 02_0.gz etc.
@@ -5124,7 +5124,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
   }
 
   public static void moveAcidFiles(FileSystem fs, FileStatus[] stats, Path dst,
-List newFiles) throws HiveException {
+List newFiles, HiveConf conf) throws 
HiveException {
 // The layout for ACID files is 
table|partname/base|delta|delete_delta/bucket
 // We will always only be writing delta files ( except IOW which writes 
base_X/ ).
 // In the buckets created by FileSinkOperator
@@ -5185,78 +5185,130 @@ private void constructOneLBLocationMap(FileStatus fSta,
   for (FileStatus origBucketStat : origBucketStats) {
 Path origBucketPath = origBucketStat.getPath();
 moveAcidFiles(AcidUtils.DELTA_PREFIX, AcidUtils.deltaFileFilter,
-fs, dst, origBucketPath, createdDeltaDirs, newFiles);
+fs, dst, origBucketPath, createdDeltaDirs, newFiles, conf);
 moveAcidFiles(AcidUtils.DELETE_DELTA_PREFIX, 
AcidUtils.deleteEventDeltaDirFilter,
-fs, dst,origBucketPath, createdDeltaDirs, newFiles);
+fs, dst,origBucketPath, createdDeltaDirs, newFiles, conf);
 moveAcidFiles(AcidUtils.BASE_PREFIX, AcidUtils.baseFileFilter,//for 
Insert Overwrite
-fs, dst, origBucketPath, createdDeltaDirs, newFiles);
+fs, dst, origBucketPath, createdDeltaDirs, newFiles, conf);
   }
 }
   }
 
   private static void moveAcidFiles(String deltaFileType, PathFilter 
pathFilter, FileSystem fs,
 Path dst, Path origBucketPath, Set 
createdDeltaDirs,
-List newFiles) throws HiveException {
-LOG.debug("Acid move looking for " + deltaFileType + " files in bucket " + 
origBucketPath);
+List newFiles, HiveConf conf) throws 
HiveException {
 
-FileStatus[] deltaStats = null;
-try {
-  deltaStats = fs.listStatus(origBucketPath, pathFilter);
-} catch (IOException e) {
-  throw new HiveException("Unable to look for " + deltaFileType + " files 
in original bucket " +
-  origBucketPath.toUri().toString(), e);
-}
-LOG.debug("Acid move found " + deltaStats.length + " " + deltaFileType + " 
files");
-
-for (FileStatus deltaStat : deltaStats) {
-  Path deltaPath = deltaStat.getPath();
-  // Create the delta directory.  Don't worry if it already exists,
-  // as that likely means another task got to it first.