This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ee890965f4 LocalInputSource: Serialize File paths without forcing
resolution. (#13534)
ee890965f4 is described below
commit ee890965f493a2752711c78459455d5acb895d9a
Author: Gian Merlino <[email protected]>
AuthorDate: Sun Dec 18 22:17:36 2022 -0800
LocalInputSource: Serialize File paths without forcing resolution. (#13534)
* LocalInputSource: Serialize File paths without forcing resolution.
Fixes #13359.
* Add one more javadoc.
---
.../druid/data/input/impl/LocalInputSource.java | 41 +++++++++++++++++-----
.../data/input/impl/LocalInputSourceTest.java | 30 +++++++++++++++-
docs/multi-stage-query/reference.md | 4 ---
3 files changed, 62 insertions(+), 13 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java
b/core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java
index e811de7cc3..4a3dd9e257 100644
--- a/core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java
+++ b/core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java
@@ -87,13 +87,28 @@ public class LocalInputSource extends AbstractInputSource
implements SplittableI
}
@Nullable
- @JsonProperty
- @JsonInclude(JsonInclude.Include.NON_NULL)
public File getBaseDir()
{
return baseDir;
}
+ /**
+ * Returns the base directory for serialization. This is better than
returning {@link File} directly, because
+ * Jackson serializes {@link File} using {@link File#getAbsolutePath()}, and
we'd prefer to not force relative
+ * path resolution as part of serialization.
+ */
+ @Nullable
+ @JsonProperty("baseDir")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String getBaseDirForSerialization()
+ {
+ if (baseDir == null) {
+ return null;
+ } else {
+ return baseDir.getPath();
+ }
+ }
+
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -102,13 +117,23 @@ public class LocalInputSource extends AbstractInputSource
implements SplittableI
return filter;
}
- @JsonProperty
- @JsonInclude(JsonInclude.Include.NON_EMPTY)
public List<File> getFiles()
{
return files;
}
+ /**
+ * Returns the list of file paths for serialization. This is better than
returning {@link File} directly, because
+ * Jackson serializes {@link File} using {@link File#getAbsolutePath()}, and
we'd prefer to not force relative
+ * path resolution as part of serialization.
+ */
+ @JsonProperty("files")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private List<String> getFilesForSerialization()
+ {
+ return getFiles().stream().map(File::getPath).collect(Collectors.toList());
+ }
+
@Override
public Stream<InputSplit<List<File>>> createSplits(InputFormat inputFormat,
@Nullable SplitHintSpec splitHintSpec)
{
@@ -233,9 +258,9 @@ public class LocalInputSource extends AbstractInputSource
implements SplittableI
public String toString()
{
return "LocalInputSource{" +
- "baseDir=\"" + baseDir +
- "\", filter=" + filter +
- ", files=" + files +
- "}";
+ "baseDir=\"" + baseDir +
+ "\", filter=" + filter +
+ ", files=" + files +
+ "}";
}
}
diff --git
a/core/src/test/java/org/apache/druid/data/input/impl/LocalInputSourceTest.java
b/core/src/test/java/org/apache/druid/data/input/impl/LocalInputSourceTest.java
index 67361b8171..2917a2aa5f 100644
---
a/core/src/test/java/org/apache/druid/data/input/impl/LocalInputSourceTest.java
+++
b/core/src/test/java/org/apache/druid/data/input/impl/LocalInputSourceTest.java
@@ -20,6 +20,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.data.input.InputSource;
@@ -51,7 +52,7 @@ public class LocalInputSourceTest
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test
- public void testSerde() throws IOException
+ public void testSerdeAbsoluteBaseDir() throws IOException
{
final ObjectMapper mapper = new ObjectMapper();
final LocalInputSource source = new LocalInputSource(new
File("myFile").getAbsoluteFile(), "myFilter");
@@ -60,6 +61,33 @@ public class LocalInputSourceTest
Assert.assertEquals(source, fromJson);
}
+ @Test
+ public void testSerdeRelativeBaseDir() throws IOException
+ {
+ final ObjectMapper mapper = new ObjectMapper();
+ final LocalInputSource source = new LocalInputSource(new File("myFile"),
"myFilter");
+ final byte[] json = mapper.writeValueAsBytes(source);
+ final LocalInputSource fromJson = (LocalInputSource)
mapper.readValue(json, InputSource.class);
+ Assert.assertEquals(source, fromJson);
+ }
+
+ @Test
+ public void testSerdeMixedAbsoluteAndRelativeFiles() throws IOException
+ {
+ final ObjectMapper mapper = new ObjectMapper();
+ final LocalInputSource source = new LocalInputSource(
+ null,
+ null,
+ ImmutableList.of(
+ new File("myFile1"),
+ new File("myFile2").getAbsoluteFile()
+ )
+ );
+ final byte[] json = mapper.writeValueAsBytes(source);
+ final LocalInputSource fromJson = (LocalInputSource)
mapper.readValue(json, InputSource.class);
+ Assert.assertEquals(source, fromJson);
+ }
+
@Test
public void testEquals()
{
diff --git a/docs/multi-stage-query/reference.md
b/docs/multi-stage-query/reference.md
index 5016b6ab48..1768572e42 100644
--- a/docs/multi-stage-query/reference.md
+++ b/docs/multi-stage-query/reference.md
@@ -134,10 +134,6 @@ You must either provide the `baseDir` or the list of
`files`. You can provide bo
the files are assumed relative to the `baseDir`. If you provide a `filter`,
you must provide the
`baseDir`.
-Note that, due to [Issue
#13359](https://github.com/apache/druid/issues/13359), the functionality
-described above is broken. Until that issue is resolved, you must provide one
or more absolute
-file paths in the `files` property and the other two properties are
unavailable.
-
#### Table Function Format
Each of the table functions above requires that you specify a format.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
