This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 16255d28391472459e84a9bde4010da59ca61941 Author: Jibing-Li <[email protected]> AuthorDate: Thu Aug 3 23:53:25 2023 +0800 [Fix](multi catalog)Fix hive partition contains special character bug (#22541) Hive partition path may contain special characters, need to encode it before creating a URI object based on the file path. --- .../main/java/org/apache/doris/common/util/S3Util.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index 623e699fb6..fe59dcee33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -39,8 +39,12 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; +import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; @@ -85,15 +89,21 @@ public class S3Util { return normalizedHdfsPath(location, props); } return location; - } catch (URISyntaxException e) { + } catch (URISyntaxException | UnsupportedEncodingException e) { throw new RuntimeException(e.getMessage(), e); } } - private static String normalizedHdfsPath(String location, Map<String, String> props) throws URISyntaxException { + private static String normalizedHdfsPath(String location, Map<String, String> props) + throws URISyntaxException, UnsupportedEncodingException { + // Hive partition may contain special characters such as ' ', '<', '>' and so on. + // Need to encode these characters before creating URI. + // But doesn't encode '/' and ':' so that we can get the correct uri host. + location = URLEncoder.encode(location, StandardCharsets.UTF_8.name()).replace("%2F", "/").replace("%3A", ":"); URI normalizedUri = new URI(location); // compatible with 'hdfs:///' or 'hdfs:/' if (StringUtils.isEmpty(normalizedUri.getHost())) { + location = URLDecoder.decode(location, StandardCharsets.UTF_8.name()); String normalizedPrefix = HdfsResource.HDFS_PREFIX + "//"; String brokenPrefix = HdfsResource.HDFS_PREFIX + "/"; if (location.startsWith(brokenPrefix) && !location.startsWith(normalizedPrefix)) { @@ -116,7 +126,7 @@ public class S3Util { } } } - return location; + return URLDecoder.decode(location, StandardCharsets.UTF_8.name()); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
