xiarixiaoyao commented on code in PR #6284:
URL: https://github.com/apache/hudi/pull/6284#discussion_r993215377
##########
hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java:
##########
@@ -204,4 +204,46 @@ public static Option<byte[]> readDataFromPath(FileSystem
fileSystem, org.apache.
public static Option<byte[]> readDataFromPath(FileSystem fileSystem,
org.apache.hadoop.fs.Path detailPath) {
return readDataFromPath(fileSystem, detailPath, false);
}
+
+ /**
+ * Return the configured local directories where hudi can write files. This
+ * method does not create any directories on its own, it only encapsulates
the
+ * logic of locating the local directories according to deployment mode.
+ */
+ public static String[] getConfiguredLocalDirs() {
+ if (isRunningInYarnContainer()) {
+ // If we are in yarn mode, systems can have different disk layouts so we
must set it
+ // to what Yarn on this system said was available. Note this assumes
that Yarn has
+ // created the directories already, and that they are secured so that
only the
+ // user has access to them.
+ return getYarnLocalDirs().split(",");
+ } else if (System.getenv("SPARK_EXECUTOR_DIRS") != null) {
+ return System.getenv("SPARK_EXECUTOR_DIRS").split(File.pathSeparator);
+ } else if (System.getenv("SPARK_LOCAL_DIRS") != null) {
+ return System.getenv("SPARK_LOCAL_DIRS").split(",");
+ } else {
+ // In non-Yarn mode (or for the driver in yarn-client mode), we cannot
trust the user
+ // configuration to point to a secure directory. So create a
subdirectory with restricted
+ // permissions under each listed directory.
+ return Option.of(System.getProperty("spark.local.dir"))
+ .orElse(System.getProperty("java.io.tmpdir")).split(",");
+ }
+ }
+
+ public static Boolean isRunningInYarnContainer() {
Review Comment:
Boolean -> boolean
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
