This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push: new eb4bd1f714a HIVE-27607: Backport of HIVE-21182: Skip setting up hive scratch dir during planning (Vineet Garg, reviewed by Ashutosh Chauhan) eb4bd1f714a is described below commit eb4bd1f714aac89cd5dfab566d4313c7b0cde897 Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com> AuthorDate: Thu Aug 24 15:48:59 2023 +0530 HIVE-27607: Backport of HIVE-21182: Skip setting up hive scratch dir during planning (Vineet Garg, reviewed by Ashutosh Chauhan) Signed-off-by: Sankar Hariappan <sank...@apache.org> Closes (#4586) --- ql/src/java/org/apache/hadoop/hive/ql/Context.java | 21 ++++++++++++++++----- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index b4d5806d4ed..4a47f4c5f4f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -531,14 +531,14 @@ public class Context { /** * Create a map-reduce scratch directory on demand and return it. + * @param mkDir flag to indicate if scratch dir is to be created or not * */ - public Path getMRScratchDir() { - + public Path getMRScratchDir(boolean mkDir) { // if we are executing entirely on the client side - then // just (re)use the local scratch directory if(isLocalOnlyExecutionMode()) { - return getLocalScratchDir(!isExplainSkipExecution()); + return getLocalScratchDir(mkDir); } try { @@ -546,16 +546,23 @@ public class Context { URI uri = dir.toUri(); Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(), - !isExplainSkipExecution(), uri.getPath()); + mkDir, uri.getPath()); LOG.info("New scratch dir is " + newScratchDir); return newScratchDir; } catch (IOException e) { throw new RuntimeException(e); } catch (IllegalArgumentException e) { throw new RuntimeException("Error while making MR scratch " - + "directory - check filesystem config (" + e.getCause() + ")", e); + + "directory - check filesystem config (" + e.getCause() + ")", e); } } + /** + * Create a map-reduce scratch directory on demand and return it. + * + */ + public Path getMRScratchDir() { + return getMRScratchDir(!isExplainSkipExecution()); + } /** * Create a temporary directory depending of the path specified. @@ -674,6 +681,10 @@ public class Context { return new Path(getStagingDir(new Path(uri), !isExplainSkipExecution()), MR_PREFIX + nextPathId()); } + public Path getMRTmpPath(boolean mkDir) { + return new Path(getMRScratchDir(mkDir), MR_PREFIX + + nextPathId()); + } /** * Get a path to store map-reduce intermediate data in. * diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0f1577353b9..b28fa1c0727 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2585,7 +2585,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { stagingPath = ctx.getMRTmpPath(tablePath.toUri()); } } else { - stagingPath = ctx.getMRTmpPath(); + stagingPath = ctx.getMRTmpPath(false); } return stagingPath;