This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new 63e204d36e2 HIVE-28173: Fixed staging dir issues on materialized views 
on HDFS encrypted tables (Stephen Carlin reviewed by Krisztian Kasa, Denys 
Kuzmenko)
63e204d36e2 is described below

commit 63e204d36e2b829b033d6c5323004a4753e4d827
Author: scarlin-cloudera <[email protected]>
AuthorDate: Thu May 2 03:55:55 2024 -0700

    HIVE-28173: Fixed staging dir issues on materialized views on HDFS 
encrypted tables (Stephen Carlin reviewed by Krisztian Kasa, Denys Kuzmenko)
    
    (cherry picked from commit 636b0d3abf00afbe2cf71dc89f762acca48867ca)
---
 .../hive/ql/parse/TestParseUtilsStagingDir.java    | 150 +++++++++++++++++++++
 ql/src/java/org/apache/hadoop/hive/ql/Context.java |   6 +-
 .../ql/metadata/HiveMaterializedViewsRegistry.java |  10 +-
 .../apache/hadoop/hive/ql/parse/ParseUtils.java    |   6 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java     |  20 +--
 5 files changed, 175 insertions(+), 17 deletions(-)

diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java
new file mode 100644
index 00000000000..5ab19c0100d
--- /dev/null
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestParseUtilsStagingDir.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test ParseUtils to see if a staging dir is created.
+ */
+public class TestParseUtilsStagingDir {
+  protected static final Logger LOG = 
LoggerFactory.getLogger(TestParseUtilsStagingDir.class.getName());
+
+  private static String jksFile = System.getProperty("java.io.tmpdir") + 
"/test.jks";
+
+  private static WarehouseInstance primary;
+  private static HiveConf conf;
+  private static MiniDFSCluster miniDFSCluster;
+
+  private static final String TABLE = "tbl";
+
+  @BeforeClass
+  public static void beforeClassSetup() throws Exception {
+    System.setProperty("jceks.key.serialFilter", 
"java.lang.Enum;java.security.KeyRep;" +
+        "java.security.KeyRep$Type;javax.crypto.spec.SecretKeySpec;" +
+        "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;!*");
+    conf = new HiveConf();
+    conf.set("hadoop.security.key.provider.path", "jceks://file" + jksFile);
+
+    miniDFSCluster =
+        new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
+
+    DFSTestUtil.createKey("test_key", miniDFSCluster, conf);
+    primary = new WarehouseInstance(LOG, miniDFSCluster, new HashMap<String, 
String>(),
+        "test_key");
+    SessionState.start((HiveConf) conf);
+  }
+
+  @AfterClass
+  public static void classLevelTearDown() throws IOException {
+    SessionState.get().close();
+    primary.close();
+    FileUtils.deleteQuietly(new File(jksFile));
+  }
+
+  /**
+   * Test to make sure staging directory is not created when the
+   * isLoadingMaterializedView flag is true and the file system
+   * is encrypted.
+   */
+  @Test
+  public void testGetStagingDirEncryptedWithMV() throws Exception {
+    Table table = createTable(TABLE, primary.warehouseRoot);
+    QB qb = createQB(table);
+    Context ctx = new Context(conf);
+    ctx.setIsLoadingMaterializedView(true);
+    Path path = SemanticAnalyzer.getStagingDirectoryPathname(qb, conf, ctx);
+    
Assert.assertFalse(miniDFSCluster.getFileSystem().exists(path.getParent()));
+  }
+
+  /**
+   * Test to make sure staging directory is created when with no flags set when
+   * the file system is encrypted.
+   *
+   * Note: This might not be the correct behavior, but this was the behavior
+   * HIVE-28173 was fixed. It is too risky to change behavior for
+   * the default path.
+   */
+  @Test
+  public void testGetStagingDirEncrypted() throws Exception {
+    Table table = createTable(TABLE, primary.warehouseRoot);
+    QB qb = createQB(table);
+    Context ctx = new Context(conf);
+    Path path = SemanticAnalyzer.getStagingDirectoryPathname(qb, conf, ctx);
+    Assert.assertTrue(miniDFSCluster.getFileSystem().exists(path.getParent()));
+  }
+
+  /**
+   * Test that no staging directory is created for default behavior where the
+   * directory location is on the local file system.
+   */
+  @Test
+  public void testGetStagingDir() throws Exception {
+    Context ctx = new Context(conf);
+    QB qb = new QB("", "", false);
+    Path path = SemanticAnalyzer.getStagingDirectoryPathname(qb, conf, ctx);
+    FileSystem fs = FileSystem.getLocal(conf);
+    Assert.assertFalse(fs.exists(path.getParent()));
+  }
+
+  private static QB createQB(Table table) {
+    String tableName = table.getTTable().getTableName();
+    QB qb = new QB("", "", false);
+    qb.setTabAlias(tableName, tableName);
+    qb.getMetaData().setSrcForAlias(tableName, table);
+    return qb;
+  }
+
+  private static Table createTable(String tableName, Path location) {
+    Table table = new Table();
+    table.setTTable(createTTableObject(tableName));
+    table.setDataLocation(location);
+    return table;
+  }
+
+  private static org.apache.hadoop.hive.metastore.api.Table 
createTTableObject(String tableName) {
+    org.apache.hadoop.hive.metastore.api.Table tTable =
+        new org.apache.hadoop.hive.metastore.api.Table();
+    tTable.setSd(new StorageDescriptor());
+    tTable.setTableName(tableName);
+    return tTable;
+  }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index c7515276770..fd0638737df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -674,7 +674,7 @@ public class Context {
    *
    */
   public Path getMRScratchDir() {
-    return getMRScratchDir(!isExplainSkipExecution());
+    return getMRScratchDir(!isExplainSkipExecution() && 
!isLoadingMaterializedView());
   }
 
   /**
@@ -850,7 +850,9 @@ public class Context {
   }
 
   public Path getMRTmpPath(URI uri) {
-    return new Path(getStagingDir(new Path(uri), !isExplainSkipExecution()), 
MR_PREFIX + nextPathId());
+    return new Path(getStagingDir(new Path(uri),
+        !isExplainSkipExecution() && !isLoadingMaterializedView()),
+        MR_PREFIX + nextPathId());
   }
 
   public Path getMRTmpPath(boolean mkDir) {
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 32ffac3a03f..19e3759ec87 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
@@ -227,7 +228,7 @@ public final class HiveMaterializedViewsRegistry {
     }
     final CBOPlan plan;
     try {
-      plan = ParseUtils.parseQuery(conf, viewQuery);
+      plan = ParseUtils.parseQuery(createContext(conf), viewQuery);
     } catch (Exception e) {
       LOG.warn("Materialized view " + materializedViewTable.getCompleteName() +
           " ignored; error parsing original query; " + e);
@@ -362,6 +363,13 @@ public final class HiveMaterializedViewsRegistry {
     return materializedViewsCache.get(ast);
   }
 
+  private Context createContext(HiveConf conf) {
+    Context ctx = new Context(conf);
+    ctx.setIsLoadingMaterializedView(true);
+    ctx.setHDFSCleanup(true);
+    return ctx;
+  }
+
   public boolean isEmpty() {
     return materializedViewsCache.isEmpty();
   }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 3feb18b9202..a4280ce2398 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -534,12 +534,10 @@ public final class ParseUtils {
     return sb.toString();
   }
 
-  public static CBOPlan parseQuery(HiveConf conf, String viewQuery)
+  public static CBOPlan parseQuery(Context ctx, String viewQuery)
       throws SemanticException, ParseException {
-    final Context ctx = new Context(conf);
-    ctx.setIsLoadingMaterializedView(true);
     final ASTNode ast = parse(viewQuery, ctx);
-    final CalcitePlanner analyzer = getAnalyzer(conf, ctx);
+    final CalcitePlanner analyzer = getAnalyzer((HiveConf) ctx.getConf(), ctx);
     RelNode logicalPlan = analyzer.genLogicalPlan(ast);
     return new CBOPlan(
         ast, logicalPlan, 
analyzer.getMaterializationValidationResult().getSupportedRewriteAlgorithms());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 95193d7dea6..802191d497f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2646,7 +2646,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
           } else {
             // This is the only place where isQuery is set to true; it 
defaults to false.
             qb.setIsQuery(true);
-            Path stagingPath = getStagingDirectoryPathname(qb);
+            Path stagingPath = getStagingDirectoryPathname(qb, conf, ctx);
             fname = stagingPath.toString();
             ctx.setResDir(stagingPath);
           }
@@ -2714,7 +2714,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
    * @return True if the path is encrypted; False if it is not encrypted
    * @throws HiveException If an error occurs while checking for encryption
    */
-  private boolean isPathEncrypted(Path path) throws HiveException {
+  private static boolean isPathEncrypted(Path path, HiveConf conf) throws 
HiveException {
 
     try {
       HadoopShims.HdfsEncryptionShim hdfsEncryptionShim =
@@ -2739,7 +2739,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
    * @return -1 if strength is weak; 0 if is equals; 1 if it is stronger
    * @throws HiveException If an error occurs while comparing key strengths.
    */
-  private int comparePathKeyStrength(Path p1, Path p2) throws HiveException {
+  private static int comparePathKeyStrength(Path p1, Path p2, HiveConf conf) 
throws HiveException {
     try {
       HadoopShims.HdfsEncryptionShim hdfsEncryptionShim1 = 
SessionState.get().getHdfsEncryptionShim(p1.getFileSystem(conf), conf);
       HadoopShims.HdfsEncryptionShim hdfsEncryptionShim2 = 
SessionState.get().getHdfsEncryptionShim(p2.getFileSystem(conf), conf);
@@ -2761,7 +2761,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
    * @return True if the path is read-only; False otherwise.
    * @throws HiveException If an error occurs while checking file permissions.
    */
-  private boolean isPathReadOnly(Path path) throws HiveException {
+  private static boolean isPathReadOnly(Path path) throws HiveException {
     HiveConf conf = SessionState.get().getConf();
     try {
       FileSystem fs = path.getFileSystem(conf);
@@ -2790,7 +2790,7 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
    * @return The strongest encrypted path. It may return NULL if there are not 
tables encrypted, or are not HDFS tables.
    * @throws HiveException if an error occurred attempting to compare the 
encryption strength
    */
-  private Path getStrongestEncryptedTablePath(QB qb) throws HiveException {
+  private static Path getStrongestEncryptedTablePath(QB qb, HiveConf conf) 
throws HiveException {
     List<String> tabAliases = new ArrayList<String>(qb.getTabAliases());
     Path strongestPath = null;
 
@@ -2801,10 +2801,10 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
         Path tablePath = tab.getDataLocation();
         if (tablePath != null) {
           if ("hdfs".equalsIgnoreCase(tablePath.toUri().getScheme())) {
-            if (isPathEncrypted(tablePath)) {
+            if (isPathEncrypted(tablePath, conf)) {
               if (strongestPath == null) {
                 strongestPath = tablePath;
-              } else if (comparePathKeyStrength(tablePath, strongestPath) > 0) 
{
+              } else if (comparePathKeyStrength(tablePath, strongestPath, 
conf) > 0) {
                 strongestPath = tablePath;
               }
             }
@@ -2828,19 +2828,19 @@ public class SemanticAnalyzer extends 
BaseSemanticAnalyzer {
    * @return The path to the staging directory.
    * @throws HiveException If an error occurs while identifying the correct 
staging location.
    */
-  private Path getStagingDirectoryPathname(QB qb) throws HiveException {
+  static Path getStagingDirectoryPathname(QB qb, HiveConf conf, Context ctx) 
throws HiveException {
     Path stagingPath = null, tablePath = null;
 
     if (DFSUtilClient.isHDFSEncryptionEnabled(conf)) {
       // Looks for the most encrypted table location
       // It may return null if there are not tables encrypted, or are not part 
of HDFS
-      tablePath = getStrongestEncryptedTablePath(qb);
+      tablePath = getStrongestEncryptedTablePath(qb, conf);
     }
     if (tablePath != null) {
       // At this point, tablePath is part of HDFS and it is encrypted
       if (isPathReadOnly(tablePath)) {
         Path tmpPath = ctx.getMRTmpPath();
-        if (comparePathKeyStrength(tablePath, tmpPath) < 0) {
+        if (comparePathKeyStrength(tablePath, tmpPath, conf) < 0) {
           throw new HiveException("Read-only encrypted tables cannot be read " 
+
               "if the scratch directory is not encrypted (or encryption is 
weak)");
         } else {

Reply via email to