This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e65875159dc HIVE-28836: Hive SessionState Resource directories
persists if SessionState is not started (Indhumathi Muthumurugesh, reviewed by
Denys Kuzmenko, Shohei Okumiya)
e65875159dc is described below
commit e65875159dce311feafaa708a0250b2b9a547626
Author: Indhumathi <[email protected]>
AuthorDate: Thu Apr 3 13:21:11 2025 +0530
HIVE-28836: Hive SessionState Resource directories persists if SessionState
is not started (Indhumathi Muthumurugesh, reviewed by Denys Kuzmenko, Shohei
Okumiya)
Closes #5708
---
.../llap/cli/service/AsyncTaskCreateUdfFile.java | 2 +-
.../hive/llap/daemon/impl/FunctionLocalizer.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/session/SessionState.java | 30 +++++-----------------
.../hadoop/hive/ql/util/ResourceDownloader.java | 16 +++++-------
.../hadoop/hive/ql/session/TestAddResource.java | 18 ++++++-------
6 files changed, 26 insertions(+), 44 deletions(-)
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCreateUdfFile.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCreateUdfFile.java
index 01fd8b6e009..80b91eba5f9 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCreateUdfFile.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCreateUdfFile.java
@@ -115,7 +115,7 @@ private Set<String> downloadPermanentFunctions() throws
HiveException, URISyntax
}
}
for (URI srcUri : srcUris) {
- List<URI> localUris = resourceDownloader.downloadExternal(srcUri, null,
false);
+ List<URI> localUris = resourceDownloader.downloadExternal(srcUri, null);
for(URI dst : localUris) {
LOG.warn("Downloaded " + dst + " from " + srcUri);
}
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
index e50c11a610e..785b1971600 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java
@@ -289,7 +289,7 @@ private void localizeOneResource(String fqfn, URI srcUri,
ResourceType rt, FnRes
return;
}
rcr = new RefCountedResource();
- List<URI> localUris = resourceDownloader.downloadExternal(srcUri, fqfn,
false);
+ List<URI> localUris = resourceDownloader.downloadExternal(srcUri, fqfn);
if (localUris == null || localUris.isEmpty()) {
LOG.error("Cannot download " + srcUri + " for " + fqfn);
return;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 4be3a32b0f0..e3372f620c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -4157,7 +4157,7 @@ private String fetchFilesNotInLocalFilesystem(String cmd)
{
String progName = getScriptProgName(cmd);
if (!ResourceDownloader.isFileUri(progName)) {
- String filePath = ss.add_resource(ResourceType.FILE, progName, true);
+ String filePath = ss.add_resource(ResourceType.FILE, progName);
Path p = new Path(filePath);
String fileName = p.getName();
String scriptArgs = getScriptArgs(cmd);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 5801ffba4e7..09bf5eecd2a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -1623,29 +1623,16 @@ public static ResourceType find_resource_type(String s)
{
return null;
}
-
-
- public String add_resource(ResourceType t, String value) throws
RuntimeException {
- return add_resource(t, value, false);
- }
-
- public String add_resource(ResourceType t, String value, boolean
convertToUnix)
+ public String add_resource(ResourceType t, String value)
throws RuntimeException {
- List<String> added = add_resources(t, Arrays.asList(value), convertToUnix);
+ List<String> added = add_resources(t, Arrays.asList(value));
if (added == null || added.isEmpty()) {
return null;
}
return added.get(0);
}
- public List<String> add_resources(ResourceType t, Collection<String> values)
- throws RuntimeException {
- // By default don't convert to unix
- return add_resources(t, values, false);
- }
-
- public List<String> add_resources(ResourceType t, Collection<String> values,
boolean convertToUnix)
- throws RuntimeException {
+ public List<String> add_resources(ResourceType t, Collection<String> values)
throws RuntimeException {
Set<String> resourceSet = resourceMaps.getResourceSet(t);
Map<String, Set<String>> resourcePathMap =
resourceMaps.getResourcePathMap(t);
Map<String, Set<String>> reverseResourcePathMap =
resourceMaps.getReverseResourcePathMap(t);
@@ -1655,7 +1642,7 @@ public List<String> add_resources(ResourceType t,
Collection<String> values, boo
String key;
//get the local path of downloaded jars.
- List<URI> downloadedURLs = resolveAndDownload(t, value, convertToUnix);
+ List<URI> downloadedURLs = resolveAndDownload(t, value);
if (ResourceDownloader.isIvyUri(value)) {
// get the key to store in map
@@ -1688,10 +1675,7 @@ public List<String> add_resources(ResourceType t,
Collection<String> values, boo
} catch (RuntimeException e) {
getConsole().printError(e.getMessage(), "\n" +
org.apache.hadoop.util.StringUtils.stringifyException(e));
throw e;
- } catch (URISyntaxException e) {
- getConsole().printError(e.getMessage());
- throw new RuntimeException(e);
- } catch (IOException e) {
+ } catch (URISyntaxException | IOException e) {
getConsole().printError(e.getMessage());
throw new RuntimeException(e);
}
@@ -1701,9 +1685,9 @@ public List<String> add_resources(ResourceType t,
Collection<String> values, boo
}
@VisibleForTesting
- protected List<URI> resolveAndDownload(ResourceType resourceType, String
value, boolean convertToUnix)
+ protected List<URI> resolveAndDownload(ResourceType resourceType, String
value)
throws URISyntaxException, IOException {
- List<URI> uris = resourceDownloader.resolveAndDownload(value,
convertToUnix);
+ List<URI> uris = resourceDownloader.resolveAndDownload(value);
if (ResourceDownloader.isHdfsUri(value)) {
assert uris.size() == 1 : "There should only be one URI
localized-resource.";
resourceMaps.getLocalHdfsLocationMap(resourceType).put(uris.get(0).toString(),
value);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
b/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
index 07dfc050436..0c4c282ebd8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/ResourceDownloader.java
@@ -44,7 +44,6 @@ public ResourceDownloader(Configuration conf, String
resourceDirPath) {
this.dependencyResolver = new DependencyResolver();
this.conf = conf;
this.resourceDir = new File(resourceDirPath);
- ensureDirectory(resourceDir);
}
/**
@@ -71,30 +70,29 @@ public static boolean isFileUri(String value) {
}
}
- public List<URI> resolveAndDownload(String source, boolean convertToUnix)
+ public List<URI> resolveAndDownload(String source)
throws URISyntaxException, IOException {
- return resolveAndDownloadInternal(createURI(source), null, convertToUnix,
true);
+ return resolveAndDownloadInternal(createURI(source), null, true);
}
- public List<URI> downloadExternal(URI source, String subDir, boolean
convertToUnix)
+ public List<URI> downloadExternal(URI source, String subDir)
throws URISyntaxException, IOException {
- return resolveAndDownloadInternal(source, subDir, convertToUnix, false);
+ return resolveAndDownloadInternal(source, subDir, false);
}
private List<URI> resolveAndDownloadInternal(URI source, String subDir,
- boolean convertToUnix, boolean isLocalAllowed) throws
URISyntaxException, IOException {
+ boolean isLocalAllowed) throws URISyntaxException, IOException {
switch (getURLType(source)) {
case FILE: return isLocalAllowed ? Collections.singletonList(source) :
null;
case IVY: return dependencyResolver.downloadDependencies(source);
case HDFS:
case OTHER:
- return Collections.singletonList(createURI(downloadResource(source,
subDir, convertToUnix)));
+ return Collections.singletonList(createURI(downloadResource(source,
subDir)));
default: throw new AssertionError(getURLType(source));
}
}
- private String downloadResource(URI srcUri, String subDir, boolean
convertToUnix)
- throws IOException, URISyntaxException {
+ private String downloadResource(URI srcUri, String subDir) throws
IOException {
LOG.debug("Converting to local {}", srcUri);
File destinationDir = (subDir == null) ? resourceDir : new
File(resourceDir, subDir);
ensureDirectory(destinationDir);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
index 54da0eb2bcc..68af29389d5 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
@@ -111,7 +111,7 @@ public void testSanity() throws URISyntaxException,
IOException {
list.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
//return all the dependency urls
- Mockito.when(ss.resolveAndDownload(t, query, false)).thenReturn(list);
+ Mockito.when(ss.resolveAndDownload(t, query)).thenReturn(list);
addList.add(query);
ss.add_resources(t, addList);
Set<String> dependencies = ss.list_resource(t, null);
@@ -147,7 +147,7 @@ public void testDuplicateAdds() throws URISyntaxException,
IOException {
Collections.sort(list);
- Mockito.when(ss.resolveAndDownload(t, query, false)).thenReturn(list);
+ Mockito.when(ss.resolveAndDownload(t, query)).thenReturn(list);
for (int i = 0; i < 10; i++) {
addList.add(query);
}
@@ -183,8 +183,8 @@ public void testUnion() throws URISyntaxException,
IOException {
list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
- Mockito.when(ss.resolveAndDownload(t, query1, false)).thenReturn(list1);
- Mockito.when(ss.resolveAndDownload(t, query2, false)).thenReturn(list2);
+ Mockito.when(ss.resolveAndDownload(t, query1)).thenReturn(list1);
+ Mockito.when(ss.resolveAndDownload(t, query2)).thenReturn(list2);
addList.add(query1);
addList.add(query2);
ss.add_resources(t, addList);
@@ -234,8 +234,8 @@ public void testDeleteJar() throws URISyntaxException,
IOException {
Collections.sort(list1);
Collections.sort(list2);
- Mockito.when(ss.resolveAndDownload(t, query1, false)).thenReturn(list1);
- Mockito.when(ss.resolveAndDownload(t, query2, false)).thenReturn(list2);
+ Mockito.when(ss.resolveAndDownload(t, query1)).thenReturn(list1);
+ Mockito.when(ss.resolveAndDownload(t, query2)).thenReturn(list2);
addList.add(query1);
addList.add(query2);
ss.add_resources(t, addList);
@@ -293,9 +293,9 @@ public void testDeleteJarMultiple() throws
URISyntaxException, IOException {
Collections.sort(list2);
Collections.sort(list3);
- Mockito.when(ss.resolveAndDownload(t, query1, false)).thenReturn(list1);
- Mockito.when(ss.resolveAndDownload(t, query2, false)).thenReturn(list2);
- Mockito.when(ss.resolveAndDownload(t, query3, false)).thenReturn(list3);
+ Mockito.when(ss.resolveAndDownload(t, query1)).thenReturn(list1);
+ Mockito.when(ss.resolveAndDownload(t, query2)).thenReturn(list2);
+ Mockito.when(ss.resolveAndDownload(t, query3)).thenReturn(list3);
addList.add(query1);
addList.add(query2);
addList.add(query3);
