This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8c90ec0ce57 HIVE-28287: Attempt make the scratch directory writable
before failing. (#5268). (Ayush Saxena, reviewed by Simhadri Govindappa)
8c90ec0ce57 is described below
commit 8c90ec0ce576d6319470f7dc4dd27daebb654dec
Author: Ayush Saxena <[email protected]>
AuthorDate: Thu May 30 21:59:54 2024 +0530
HIVE-28287: Attempt make the scratch directory writable before failing.
(#5268). (Ayush Saxena, reviewed by Simhadri Govindappa)
---
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 24 ++++++++++++++++++----
.../apache/hadoop/hive/ql/exec/TestUtilities.java | 20 ++++++++++++++++++
2 files changed, 40 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index b247e276cb1..cfb21669f49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -4961,13 +4961,29 @@ public final class Utilities {
"HDFS dir: " + rootHDFSDirPath + ", permission: " +
currentHDFSDirPermission);
}
// If the root HDFS scratch dir already exists, make sure it is writeable.
- if (!((currentHDFSDirPermission.toShort() & writableHDFSDirPermission
- .toShort()) == writableHDFSDirPermission.toShort())) {
- throw new RuntimeException("The dir: " + rootHDFSDirPath
- + " on HDFS should be writable. Current permissions are: " +
currentHDFSDirPermission);
+ if (!isWritable(currentHDFSDirPermission, writableHDFSDirPermission) &&
+ !attemptMakePathWritable(fs, rootHDFSDirPath,
+ FsPermission.createImmutable((short)
(currentHDFSDirPermission.toShort() | writableHDFSDirPermission.toShort())))) {
+ throw new RuntimeException(
+ "The dir: " + rootHDFSDirPath + " should be writable. Current
permissions are: " + currentHDFSDirPermission);
}
}
+ private static boolean attemptMakePathWritable(FileSystem fs, Path path,
FsPermission perm) {
+ try {
+ LOG.info("Attempting to set {} permissions on path {}", perm, path);
+ fs.setPermission(path, perm);
+ return isWritable(fs.getFileStatus(path).getPermission(), perm);
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ private static boolean isWritable(FsPermission currentHDFSDirPermission,
FsPermission writableHDFSDirPermission) {
+ return (currentHDFSDirPermission.toShort() &
writableHDFSDirPermission.toShort())
+ == writableHDFSDirPermission.toShort();
+ }
+
// Get the bucketing version stored in the string format
public static int getBucketingVersion(final String versionStr) {
int bucketingVersion = 1;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index f87d6c40f17..7d47fa34e8c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
@@ -75,6 +76,8 @@ import
org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRJobConfig;
+
+import com.google.common.io.Files;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -815,6 +818,23 @@ public class TestUtilities {
assertEquals(expectedPathes, resultPathes);
}
+ @Test
+ public void testSetPermissionsOnExistingDir() throws IOException {
+ File tmpDir = Files.createTempDir();
+ Path path = new Path(tmpDir.getPath());
+ HiveConf conf = new HiveConf(this.getClass());
+ FileSystem fs = path.getFileSystem(conf);
+ fs.setPermission(path, new FsPermission((short) 00700));
+ Utilities.ensurePathIsWritable(path, conf);
+ Assert.assertEquals((short) 0733,
fs.getFileStatus(path).getPermission().toShort());
+
+ // Test with more open permissions than required, but still not writable,
+ // it should just make the directory writable without restricting the
existing permissions
+ fs.setPermission(path, new FsPermission((short) 00755));
+ Utilities.ensurePathIsWritable(path, conf);
+ Assert.assertEquals((short) 0777,
fs.getFileStatus(path).getPermission().toShort());
+ }
+
private FileStatus[] generateTestNotEmptyFileStatuses(String... fileNames) {
return generateTestNotEmptyFileStatuses(null, fileNames);
}