This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 112893bd6e Fixing the metadata tar gz file path (#10149)
112893bd6e is described below
commit 112893bd6ee1357d08b9f889535c64a4b654ea1a
Author: Xiang Fu <[email protected]>
AuthorDate: Thu Jan 19 03:48:09 2023 -0800
Fixing the metadata tar gz file path (#10149)
---
.../segment/local/utils/SegmentPushUtils.java | 20 +++++++++++++++--
.../segment/local/utils/SegmentPushUtilsTest.java | 25 +++++++++++++++++++---
2 files changed, 40 insertions(+), 5 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
index b2ed31821c..dabee5dcb6 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPushUtils.java
@@ -290,8 +290,9 @@ public class SegmentPushUtils implements Serializable {
// Check if there is a segment metadata tar gz file named
`segmentName.metadata.tar.gz`, already in the remote
// directory. This is to avoid generating a new segment metadata tar gz
file every time we push a segment,
// which requires downloading the entire segment tar gz file.
- URI metadataTarGzFilePath = URI.create(
- new File(tarFilePath).getParentFile() + File.separator + segmentName
+ Constants.METADATA_TAR_GZ_FILE_EXT);
+
+ URI metadataTarGzFilePath = generateSegmentMetadataURI(tarFilePath,
segmentName);
+ LOGGER.info("Checking if metadata tar gz file {} exists",
metadataTarGzFilePath);
if (spec.getPushJobSpec().isPreferMetadataTarGz() &&
fileSystem.exists(metadataTarGzFilePath)) {
segmentMetadataFile = new File(FileUtils.getTempDirectory(),
"segmentMetadata-" + UUID.randomUUID() +
TarGzCompressionUtils.TAR_GZ_FILE_EXTENSION);
@@ -442,4 +443,19 @@ public class SegmentPushUtils implements Serializable {
FileUtils.deleteQuietly(segmentMetadataDir);
}
}
+
+ public static URI generateSegmentMetadataURI(String segmentTarPath, String
segmentName)
+ throws URISyntaxException {
+ URI segmentTarURI = URI.create(segmentTarPath);
+ URI metadataTarGzFilePath = new URI(
+ segmentTarURI.getScheme(),
+ segmentTarURI.getUserInfo(),
+ segmentTarURI.getHost(),
+ segmentTarURI.getPort(),
+ new File(segmentTarURI.getPath()).getParentFile() + File.separator +
segmentName
+ + Constants.METADATA_TAR_GZ_FILE_EXT,
+ segmentTarURI.getQuery(),
+ segmentTarURI.getFragment());
+ return metadataTarGzFilePath;
+ }
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
index dd3a09a7ed..20121a14b1 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPushUtilsTest.java
@@ -20,21 +20,24 @@ package org.apache.pinot.segment.local.utils;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.Map;
import org.apache.pinot.spi.ingestion.batch.spec.PushJobSpec;
import org.testng.annotations.Test;
-import static org.testng.Assert.*;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
public class SegmentPushUtilsTest {
@Test
- public void testGetSegmentUriToTarPathMap() throws IOException {
+ public void testGetSegmentUriToTarPathMap()
+ throws IOException {
URI outputDirURI = Files.createTempDirectory("test").toUri();
- String[] segmentFiles = new String[] {
+ String[] segmentFiles = new String[]{
outputDirURI.resolve("segment.tar.gz").toString(),
outputDirURI.resolve("stats_202201.tar.gz").toString(),
outputDirURI.resolve("/2022/segment.tar.gz").toString(),
@@ -61,4 +64,20 @@ public class SegmentPushUtilsTest {
assertEquals(result.get(segmentFiles[1]), segmentFiles[1]);
assertEquals(result.get(segmentFiles[3]), segmentFiles[3]);
}
+
+ @Test
+ public void testGenerateSegmentMetadataURI()
+ throws URISyntaxException {
+ assertEquals(
+
SegmentPushUtils.generateSegmentMetadataURI("/a/b/c/my-segment.tar.gz",
"my-segment"),
+ URI.create("/a/b/c/my-segment.metadata.tar.gz"));
+
+ assertEquals(
+
SegmentPushUtils.generateSegmentMetadataURI("s3://a/b/c/my-segment.tar.gz",
"my-segment"),
+ URI.create("s3://a/b/c/my-segment.metadata.tar.gz"));
+
+ assertEquals(
+
SegmentPushUtils.generateSegmentMetadataURI("hdfs://a/b/c/my-segment.tar.gz",
"my-segment"),
+ URI.create("hdfs://a/b/c/my-segment.metadata.tar.gz"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
