This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 8c90ec0ce57 HIVE-28287: Attempt make the scratch directory writable 
before failing. (#5268). (Ayush Saxena, reviewed by Simhadri Govindappa)
8c90ec0ce57 is described below

commit 8c90ec0ce576d6319470f7dc4dd27daebb654dec
Author: Ayush Saxena <[email protected]>
AuthorDate: Thu May 30 21:59:54 2024 +0530

    HIVE-28287: Attempt make the scratch directory writable before failing. 
(#5268). (Ayush Saxena, reviewed by Simhadri Govindappa)
---
 .../org/apache/hadoop/hive/ql/exec/Utilities.java  | 24 ++++++++++++++++++----
 .../apache/hadoop/hive/ql/exec/TestUtilities.java  | 20 ++++++++++++++++++
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index b247e276cb1..cfb21669f49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -4961,13 +4961,29 @@ public final class Utilities {
         "HDFS dir: " + rootHDFSDirPath + ", permission: " + 
currentHDFSDirPermission);
     }
     // If the root HDFS scratch dir already exists, make sure it is writeable.
-    if (!((currentHDFSDirPermission.toShort() & writableHDFSDirPermission
-        .toShort()) == writableHDFSDirPermission.toShort())) {
-      throw new RuntimeException("The dir: " + rootHDFSDirPath
-          + " on HDFS should be writable. Current permissions are: " + 
currentHDFSDirPermission);
+    if (!isWritable(currentHDFSDirPermission, writableHDFSDirPermission) &&
+        !attemptMakePathWritable(fs, rootHDFSDirPath,
+            FsPermission.createImmutable((short) 
(currentHDFSDirPermission.toShort() | writableHDFSDirPermission.toShort())))) {
+      throw new RuntimeException(
+          "The dir: " + rootHDFSDirPath + " should be writable. Current 
permissions are: " + currentHDFSDirPermission);
     }
   }
 
+  private static boolean attemptMakePathWritable(FileSystem fs, Path path, 
FsPermission perm) {
+    try {
+      LOG.info("Attempting to set {} permissions on path {}", perm, path);
+      fs.setPermission(path, perm);
+      return isWritable(fs.getFileStatus(path).getPermission(), perm);
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
+  private static boolean isWritable(FsPermission currentHDFSDirPermission, 
FsPermission writableHDFSDirPermission) {
+    return (currentHDFSDirPermission.toShort() & 
writableHDFSDirPermission.toShort())
+        == writableHDFSDirPermission.toShort();
+  }
+
   // Get the bucketing version stored in the string format
   public static int getBucketingVersion(final String versionStr) {
     int bucketingVersion = 1;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index f87d6c40f17..7d47fa34e8c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.common.type.Timestamp;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.Context;
@@ -75,6 +76,8 @@ import 
org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.MRJobConfig;
+
+import com.google.common.io.Files;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -815,6 +818,23 @@ public class TestUtilities {
     assertEquals(expectedPathes, resultPathes);
   }
 
+  @Test
+  public void testSetPermissionsOnExistingDir() throws IOException {
+    File tmpDir = Files.createTempDir();
+    Path path = new Path(tmpDir.getPath());
+    HiveConf conf = new HiveConf(this.getClass());
+    FileSystem fs = path.getFileSystem(conf);
+    fs.setPermission(path, new FsPermission((short) 00700));
+    Utilities.ensurePathIsWritable(path, conf);
+    Assert.assertEquals((short) 0733, 
fs.getFileStatus(path).getPermission().toShort());
+
+    // Test with more open permissions than required, but still not writable,
+    // it should just make the directory writable without restricting the 
existing permissions
+    fs.setPermission(path, new FsPermission((short) 00755));
+    Utilities.ensurePathIsWritable(path, conf);
+    Assert.assertEquals((short) 0777, 
fs.getFileStatus(path).getPermission().toShort());
+  }
+
   private FileStatus[] generateTestNotEmptyFileStatuses(String... fileNames) {
     return generateTestNotEmptyFileStatuses(null, fileNames);
   }

Reply via email to