This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 63e204d36e2 HIVE-28173: Fixed staging dir issues on materialized views
on HDFS encrypted tables (Stephen Carlin reviewed by Krisztian Kasa, Denys
Kuzmenko)
63e204d36e2 is described below
commit 63e204d36e2b829b033d6c5323004a4753e4d827
Author: scarlin-cloudera <[email protected]>
AuthorDate: Thu May 2 03:55:55 2024 -0700
HIVE-28173: Fixed staging dir issues on materialized views on HDFS
encrypted tables (Stephen Carlin reviewed by Krisztian Kasa, Denys Kuzmenko)
(cherry picked from commit 636b0d3abf00afbe2cf71dc89f762acca48867ca)
---
.../hive/ql/parse/TestParseUtilsStagingDir.java | 150 +++++++++++++++++++++
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 6 +-
.../ql/metadata/HiveMaterializedViewsRegistry.java | 10 +-
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 6 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 20 +--
5 files changed, 175 insertions(+), 17 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java
new file mode 100644
index 00000000000..5ab19c0100d
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test ParseUtils to see if a staging dir is created.
+ */
+public class TestParseUtilsStagingDir {
+ protected static final Logger LOG =
LoggerFactory.getLogger(TestParseUtilsStagingDir.class.getName());
+
+ private static String jksFile = System.getProperty("java.io.tmpdir") +
"/test.jks";
+
+ private static WarehouseInstance primary;
+ private static HiveConf conf;
+ private static MiniDFSCluster miniDFSCluster;
+
+ private static final String TABLE = "tbl";
+
+ @BeforeClass
+ public static void beforeClassSetup() throws Exception {
+ System.setProperty("jceks.key.serialFilter",
"java.lang.Enum;java.security.KeyRep;" +
+ "java.security.KeyRep$Type;javax.crypto.spec.SecretKeySpec;" +
+ "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;!*");
+ conf = new HiveConf();
+ conf.set("hadoop.security.key.provider.path", "jceks://file" + jksFile);
+
+ miniDFSCluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
+
+ DFSTestUtil.createKey("test_key", miniDFSCluster, conf);
+ primary = new WarehouseInstance(LOG, miniDFSCluster, new HashMap<String,
String>(),
+ "test_key");
+ SessionState.start((HiveConf) conf);
+ }
+
+ @AfterClass
+ public static void classLevelTearDown() throws IOException {
+ SessionState.get().close();
+ primary.close();
+ FileUtils.deleteQuietly(new File(jksFile));
+ }
+
+ /**
+ * Test to make sure staging directory is not created when the
+ * isLoadingMaterializedView flag is true and the file system
+ * is encrypted.
+ */
+ @Test
+ public void testGetStagingDirEncryptedWithMV() throws Exception {
+ Table table = createTable(TABLE, primary.warehouseRoot);
+ QB qb = createQB(table);
+ Context ctx = new Context(conf);
+ ctx.setIsLoadingMaterializedView(true);
+ Path path = SemanticAnalyzer.getStagingDirectoryPathname(qb, conf, ctx);
+
Assert.assertFalse(miniDFSCluster.getFileSystem().exists(path.getParent()));
+ }
+
+ /**
+ * Test to make sure staging directory is created when with no flags set when
+ * the file system is encrypted.
+ *
+ * Note: This might not be the correct behavior, but this was the behavior
+ * HIVE-28173 was fixed. It is too risky to change behavior for
+ * the default path.
+ */
+ @Test
+ public void testGetStagingDirEncrypted() throws Exception {
+ Table table = createTable(TABLE, primary.warehouseRoot);
+ QB qb = createQB(table);
+ Context ctx = new Context(conf);
+ Path path = SemanticAnalyzer.getStagingDirectoryPathname(qb, conf, ctx);
+ Assert.assertTrue(miniDFSCluster.getFileSystem().exists(path.getParent()));
+ }
+
+ /**
+ * Test that no staging directory is created for default behavior where the
+ * directory location is on the local file system.
+ */
+ @Test
+ public void testGetStagingDir() throws Exception {
+ Context ctx = new Context(conf);
+ QB qb = new QB("", "", false);
+ Path path = SemanticAnalyzer.getStagingDirectoryPathname(qb, conf, ctx);
+ FileSystem fs = FileSystem.getLocal(conf);
+ Assert.assertFalse(fs.exists(path.getParent()));
+ }
+
+ private static QB createQB(Table table) {
+ String tableName = table.getTTable().getTableName();
+ QB qb = new QB("", "", false);
+ qb.setTabAlias(tableName, tableName);
+ qb.getMetaData().setSrcForAlias(tableName, table);
+ return qb;
+ }
+
+ private static Table createTable(String tableName, Path location) {
+ Table table = new Table();
+ table.setTTable(createTTableObject(tableName));
+ table.setDataLocation(location);
+ return table;
+ }
+
+ private static org.apache.hadoop.hive.metastore.api.Table
createTTableObject(String tableName) {
+ org.apache.hadoop.hive.metastore.api.Table tTable =
+ new org.apache.hadoop.hive.metastore.api.Table();
+ tTable.setSd(new StorageDescriptor());
+ tTable.setTableName(tableName);
+ return tTable;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index c7515276770..fd0638737df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -674,7 +674,7 @@ public class Context {
*
*/
public Path getMRScratchDir() {
- return getMRScratchDir(!isExplainSkipExecution());
+ return getMRScratchDir(!isExplainSkipExecution() &&
!isLoadingMaterializedView());
}
/**
@@ -850,7 +850,9 @@ public class Context {
}
public Path getMRTmpPath(URI uri) {
- return new Path(getStagingDir(new Path(uri), !isExplainSkipExecution()),
MR_PREFIX + nextPathId());
+ return new Path(getStagingDir(new Path(uri),
+ !isExplainSkipExecution() && !isLoadingMaterializedView()),
+ MR_PREFIX + nextPathId());
}
public Path getMRTmpPath(boolean mkDir) {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 32ffac3a03f..19e3759ec87 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
@@ -227,7 +228,7 @@ public final class HiveMaterializedViewsRegistry {
}
final CBOPlan plan;
try {
- plan = ParseUtils.parseQuery(conf, viewQuery);
+ plan = ParseUtils.parseQuery(createContext(conf), viewQuery);
} catch (Exception e) {
LOG.warn("Materialized view " + materializedViewTable.getCompleteName() +
" ignored; error parsing original query; " + e);
@@ -362,6 +363,13 @@ public final class HiveMaterializedViewsRegistry {
return materializedViewsCache.get(ast);
}
+ private Context createContext(HiveConf conf) {
+ Context ctx = new Context(conf);
+ ctx.setIsLoadingMaterializedView(true);
+ ctx.setHDFSCleanup(true);
+ return ctx;
+ }
+
public boolean isEmpty() {
return materializedViewsCache.isEmpty();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 3feb18b9202..a4280ce2398 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -534,12 +534,10 @@ public final class ParseUtils {
return sb.toString();
}
- public static CBOPlan parseQuery(HiveConf conf, String viewQuery)
+ public static CBOPlan parseQuery(Context ctx, String viewQuery)
throws SemanticException, ParseException {
- final Context ctx = new Context(conf);
- ctx.setIsLoadingMaterializedView(true);
final ASTNode ast = parse(viewQuery, ctx);
- final CalcitePlanner analyzer = getAnalyzer(conf, ctx);
+ final CalcitePlanner analyzer = getAnalyzer((HiveConf) ctx.getConf(), ctx);
RelNode logicalPlan = analyzer.genLogicalPlan(ast);
return new CBOPlan(
ast, logicalPlan,
analyzer.getMaterializationValidationResult().getSupportedRewriteAlgorithms());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 95193d7dea6..802191d497f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2646,7 +2646,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
} else {
// This is the only place where isQuery is set to true; it
defaults to false.
qb.setIsQuery(true);
- Path stagingPath = getStagingDirectoryPathname(qb);
+ Path stagingPath = getStagingDirectoryPathname(qb, conf, ctx);
fname = stagingPath.toString();
ctx.setResDir(stagingPath);
}
@@ -2714,7 +2714,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
* @return True if the path is encrypted; False if it is not encrypted
* @throws HiveException If an error occurs while checking for encryption
*/
- private boolean isPathEncrypted(Path path) throws HiveException {
+ private static boolean isPathEncrypted(Path path, HiveConf conf) throws
HiveException {
try {
HadoopShims.HdfsEncryptionShim hdfsEncryptionShim =
@@ -2739,7 +2739,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
* @return -1 if strength is weak; 0 if is equals; 1 if it is stronger
* @throws HiveException If an error occurs while comparing key strengths.
*/
- private int comparePathKeyStrength(Path p1, Path p2) throws HiveException {
+ private static int comparePathKeyStrength(Path p1, Path p2, HiveConf conf)
throws HiveException {
try {
HadoopShims.HdfsEncryptionShim hdfsEncryptionShim1 =
SessionState.get().getHdfsEncryptionShim(p1.getFileSystem(conf), conf);
HadoopShims.HdfsEncryptionShim hdfsEncryptionShim2 =
SessionState.get().getHdfsEncryptionShim(p2.getFileSystem(conf), conf);
@@ -2761,7 +2761,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
* @return True if the path is read-only; False otherwise.
* @throws HiveException If an error occurs while checking file permissions.
*/
- private boolean isPathReadOnly(Path path) throws HiveException {
+ private static boolean isPathReadOnly(Path path) throws HiveException {
HiveConf conf = SessionState.get().getConf();
try {
FileSystem fs = path.getFileSystem(conf);
@@ -2790,7 +2790,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
* @return The strongest encrypted path. It may return NULL if there are not
tables encrypted, or are not HDFS tables.
* @throws HiveException if an error occurred attempting to compare the
encryption strength
*/
- private Path getStrongestEncryptedTablePath(QB qb) throws HiveException {
+ private static Path getStrongestEncryptedTablePath(QB qb, HiveConf conf)
throws HiveException {
List<String> tabAliases = new ArrayList<String>(qb.getTabAliases());
Path strongestPath = null;
@@ -2801,10 +2801,10 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
Path tablePath = tab.getDataLocation();
if (tablePath != null) {
if ("hdfs".equalsIgnoreCase(tablePath.toUri().getScheme())) {
- if (isPathEncrypted(tablePath)) {
+ if (isPathEncrypted(tablePath, conf)) {
if (strongestPath == null) {
strongestPath = tablePath;
- } else if (comparePathKeyStrength(tablePath, strongestPath) > 0)
{
+ } else if (comparePathKeyStrength(tablePath, strongestPath,
conf) > 0) {
strongestPath = tablePath;
}
}
@@ -2828,19 +2828,19 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
* @return The path to the staging directory.
* @throws HiveException If an error occurs while identifying the correct
staging location.
*/
- private Path getStagingDirectoryPathname(QB qb) throws HiveException {
+ static Path getStagingDirectoryPathname(QB qb, HiveConf conf, Context ctx)
throws HiveException {
Path stagingPath = null, tablePath = null;
if (DFSUtilClient.isHDFSEncryptionEnabled(conf)) {
// Looks for the most encrypted table location
// It may return null if there are not tables encrypted, or are not part
of HDFS
- tablePath = getStrongestEncryptedTablePath(qb);
+ tablePath = getStrongestEncryptedTablePath(qb, conf);
}
if (tablePath != null) {
// At this point, tablePath is part of HDFS and it is encrypted
if (isPathReadOnly(tablePath)) {
Path tmpPath = ctx.getMRTmpPath();
- if (comparePathKeyStrength(tablePath, tmpPath) < 0) {
+ if (comparePathKeyStrength(tablePath, tmpPath, conf) < 0) {
throw new HiveException("Read-only encrypted tables cannot be read "
+
"if the scratch directory is not encrypted (or encryption is
weak)");
} else {