[hive] branch master updated: HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed by Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository. tchoi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new ac1cfe2294d HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed by Laszlo Bodor) ac1cfe2294d is described below commit ac1cfe2294d1b7b8c02b60899d69868920ede0f5 Author: Teddy Choi AuthorDate: Wed Jan 4 13:35:57 2023 +0900 HIVE-26555 Read-only mode for Hive database (#3614) (Teddy Choi, reviewed by Laszlo Bodor) --- .../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 1 + .../hive/ql/hooks/EnforceReadOnlyDatabaseHook.java | 196 + .../test/queries/clientnegative/read_only_delete.q | 23 +++ .../test/queries/clientnegative/read_only_insert.q | 23 +++ .../test/queries/clientpositive/read_only_hook.q | 27 +++ .../results/clientnegative/read_only_delete.q.out | 34 .../results/clientnegative/read_only_insert.q.out | 35 .../clientpositive/llap/read_only_hook.q.out | 71 8 files changed, 410 insertions(+) diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 1a3ad38acef..5e4fbf8e642 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -485,6 +485,7 @@ public enum ErrorMsg { CTLF_UNSUPPORTED_FORMAT(10434, "CREATE TABLE LIKE FILE is not supported by the ''{0}'' file format", true), NON_NATIVE_ACID_UPDATE(10435, "Update and Merge into non-native ACID table is only supported when " + HiveConf.ConfVars.SPLIT_UPDATE.varname + " is true."), + READ_ONLY_DATABASE(10436, "Database {0} is read-only", true), //== 2 range starts here // diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java new file mode 100644 index 000..c343fc7b14f --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyDatabaseHook.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.hooks; + +import com.google.common.collect.Sets; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLWork; +import org.apache.hadoop.hive.ql.ddl.database.alter.poperties.AlterDatabaseSetPropertiesDesc; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.HiveOperation; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * EnforceReadOnlyDatabaseHook is a hook that disallows write operations on read-only databases. + * It's enforced when "hive.exec.pre.hooks" has "org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyDatabaseHook" and + * a database has 'readonly'='true' property. + */ +public class EnforceReadOnlyDatabaseHook implements ExecuteWithHookContext { + public static final String READONLY = "readonly"; + private static final Logger LOG = LoggerFactory.getLogger(EnforceReadOnlyDatabaseHook.class); + + @Override + public void run(HookContext hookContext) throws Exception { +assert(hookContext.getHookType() == HookContext.HookType.PRE_EXEC_HOOK); +final QueryState queryState = hookContext.getQueryState(); +final HiveOperation hiveOperation = queryState.getHiveOperation(); + +// Allow read-only type operations, excluding query. +// A query can be EXPLAIN, SELECT, INSERT, UPDATE, or DELETE. +if (isReadOnlyOperation(hiveOperation)) { + return; +} + +// Allow
[hive] branch master updated (67906e855da -> 62893c0db6c)
This is an automated email from the ASF dual-hosted git repository. dengzh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git from 67906e855da HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795) add 62893c0db6c HIVE-26790: Update protobuf/grpc versions (#3814) (Henri Biestro, reviewed by Zhihua Deng and Chris Nauroth) No new revisions were added by this update. Summary of changes: pom.xml | 2 +- standalone-metastore/metastore-common/pom.xml | 2 +- standalone-metastore/pom.xml | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-)
[hive] branch master updated: HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795)
This is an automated email from the ASF dual-hosted git repository. rameshkumar pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 67906e855da HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795) 67906e855da is described below commit 67906e855da46835fe3c4bce22378670efc7b411 Author: Dmitriy Fingerman AuthorDate: Tue Jan 3 13:18:56 2023 -0500 HIVE-23559: Optimise Hive::moveAcidFiles for cloud storage. (#3795) Co-authored-by: Dmitriy Fingerman --- .../org/apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 156 ++--- 2 files changed, 105 insertions(+), 53 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index a5a5ea793d6..555bd7fb5b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@ -388,7 +388,7 @@ public class MoveTask extends Task implements Serializable { FileSystem srcFs = sourcePath.getFileSystem(conf); FileStatus[] srcs = srcFs.globStatus(sourcePath); if(srcs != null) { - Hive.moveAcidFiles(srcFs, srcs, targetPath, null); + Hive.moveAcidFiles(srcFs, srcs, targetPath, null, conf); } else { LOG.debug("No files found to move from " + sourcePath + " to " + targetPath); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 238b24d2641..d9301c3ea4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -5102,7 +5102,7 @@ private void constructOneLBLocationMap(FileStatus fSta, // If we're moving files around for an ACID write then the rules and paths are all different. // You can blame this on Owen. if (isAcidIUD) { - moveAcidFiles(srcFs, srcs, destf, newFiles); + moveAcidFiles(srcFs, srcs, destf, newFiles, conf); } else { // For ACID non-bucketed case, the filenames have to be in the format consistent with INSERT/UPDATE/DELETE Ops, // i.e, like 00_0, 01_0_copy_1, 02_0.gz etc. @@ -5124,7 +5124,7 @@ private void constructOneLBLocationMap(FileStatus fSta, } public static void moveAcidFiles(FileSystem fs, FileStatus[] stats, Path dst, -List newFiles) throws HiveException { +List newFiles, HiveConf conf) throws HiveException { // The layout for ACID files is table|partname/base|delta|delete_delta/bucket // We will always only be writing delta files ( except IOW which writes base_X/ ). // In the buckets created by FileSinkOperator @@ -5185,78 +5185,130 @@ private void constructOneLBLocationMap(FileStatus fSta, for (FileStatus origBucketStat : origBucketStats) { Path origBucketPath = origBucketStat.getPath(); moveAcidFiles(AcidUtils.DELTA_PREFIX, AcidUtils.deltaFileFilter, -fs, dst, origBucketPath, createdDeltaDirs, newFiles); +fs, dst, origBucketPath, createdDeltaDirs, newFiles, conf); moveAcidFiles(AcidUtils.DELETE_DELTA_PREFIX, AcidUtils.deleteEventDeltaDirFilter, -fs, dst,origBucketPath, createdDeltaDirs, newFiles); +fs, dst,origBucketPath, createdDeltaDirs, newFiles, conf); moveAcidFiles(AcidUtils.BASE_PREFIX, AcidUtils.baseFileFilter,//for Insert Overwrite -fs, dst, origBucketPath, createdDeltaDirs, newFiles); +fs, dst, origBucketPath, createdDeltaDirs, newFiles, conf); } } } private static void moveAcidFiles(String deltaFileType, PathFilter pathFilter, FileSystem fs, Path dst, Path origBucketPath, Set createdDeltaDirs, -List newFiles) throws HiveException { -LOG.debug("Acid move looking for " + deltaFileType + " files in bucket " + origBucketPath); +List newFiles, HiveConf conf) throws HiveException { -FileStatus[] deltaStats = null; -try { - deltaStats = fs.listStatus(origBucketPath, pathFilter); -} catch (IOException e) { - throw new HiveException("Unable to look for " + deltaFileType + " files in original bucket " + - origBucketPath.toUri().toString(), e); -} -LOG.debug("Acid move found " + deltaStats.length + " " + deltaFileType + " files"); - -for (FileStatus deltaStat : deltaStats) { - Path deltaPath = deltaStat.getPath(); - // Create the delta directory. Don't worry if it already exists, - // as that likely means another task got to it first.