This is an automated email from the ASF dual-hosted git repository.
junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 9fd81eebce [core] Introduce 'index-file-in-data-file-dir' to put index
files in data dir (#6132)
9fd81eebce is described below
commit 9fd81eebced3cc5b14d8aa0a57f0e5a86ca8de5e
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Aug 25 17:58:57 2025 +0800
[core] Introduce 'index-file-in-data-file-dir' to put index files in data
dir (#6132)
---
.../shortcodes/generated/core_configuration.html | 6 ++
.../main/java/org/apache/paimon/CoreOptions.java | 11 +++
.../java/org/apache/paimon/utils/IntFileUtils.java | 6 +-
.../java/org/apache/paimon/AbstractFileStore.java | 3 +-
.../apache/paimon/compact/CompactDeletionFile.java | 2 +-
.../paimon/deletionvectors/DeletionFileWriter.java | 14 +++-
.../DeletionVectorIndexFileWriter.java | 10 +--
.../deletionvectors/DeletionVectorsIndexFile.java | 10 ++-
.../append/BaseAppendDeleteFileMaintainer.java | 3 +-
.../paimon/iceberg/IcebergCommitCallback.java | 3 +-
.../paimon/index/DynamicBucketIndexMaintainer.java | 4 +-
.../org/apache/paimon/index/HashIndexFile.java | 34 ++++-----
.../java/org/apache/paimon/index/IndexFile.java | 29 +++++---
.../org/apache/paimon/index/IndexFileHandler.java | 8 +--
.../org/apache/paimon/index/IndexFileMeta.java | 35 +++++----
.../paimon/index/IndexFileMetaSerializer.java | 25 ++++---
...lizer.java => IndexFileMetaV1Deserializer.java} | 7 +-
...lizer.java => IndexFileMetaV2Deserializer.java} | 33 ++-------
.../index/IndexInDataFileDirPathFactory.java | 60 ++++++++++++++++
.../IndexPathFactory.java} | 24 ++-----
.../org/apache/paimon/index/PartitionIndex.java | 2 +-
.../org/apache/paimon/io/DataFilePathFactory.java | 6 +-
.../apache/paimon/manifest/IndexManifestEntry.java | 3 +-
.../manifest/IndexManifestEntrySerializer.java | 20 +++---
.../sink/CommitMessageLegacyV2Serializer.java | 1 +
.../paimon/table/sink/CommitMessageSerializer.java | 19 +++--
.../apache/paimon/table/sink/TableCommitImpl.java | 7 +-
.../table/source/snapshot/SnapshotReaderImpl.java | 9 ++-
.../paimon/table/system/TableIndexesTable.java | 2 +-
.../apache/paimon/utils/FileStorePathFactory.java | 46 ++++++++----
.../paimon/utils/IndexFilePathFactories.java | 5 +-
.../DeletionVectorsIndexFileTest.java | 42 ++++++-----
.../append/AppendDeletionFileMaintainerTest.java | 13 ++--
.../index/DynamicBucketIndexMaintainerTest.java | 2 +-
.../org/apache/paimon/index/HashIndexFileTest.java | 20 +++---
.../paimon/index/IndexFileMetaSerializerTest.java | 13 ++--
.../apache/paimon/io/DataFileIndexWriterTest.java | 3 +-
.../paimon/io/KeyValueFileReadWriteTest.java | 6 +-
...festCommittableSerializerCompatibilityTest.java | 79 ++++++++++++---------
.../paimon/manifest/ManifestFileMetaTestBase.java | 3 +-
.../apache/paimon/manifest/ManifestFileTest.java | 3 +-
.../apache/paimon/manifest/ManifestListTest.java | 3 +-
.../paimon/operation/FileStoreCommitTest.java | 27 ++-----
.../paimon/utils/FileStorePathFactoryTest.java | 6 +-
.../compatibility/manifest-committable-v8 | Bin 3762 -> 3786 bytes
.../apache/paimon/flink/DeletionVectorITCase.java | 61 ++++++++++++++++
.../flink/source/TestChangelogDataReadWrite.java | 3 +-
.../apache/paimon/spark/SparkFileIndexITCase.java | 3 +-
.../paimon/spark/sql/DeletionVectorTest.scala | 2 +-
49 files changed, 460 insertions(+), 276 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 29d110d495..80f6d51046 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -542,6 +542,12 @@ under the License.
<td>String</td>
<td>Used to specify the end tag (inclusive), and Paimon will find
an earlier tag and return changes between them. If the tag doesn't exist or the
earlier tag doesn't exist, return empty. This option requires
'tag.creation-period' and 'tag.period-formatter' configured.</td>
</tr>
+ <tr>
+ <td><h5>index-file-in-data-file-dir</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether index file in data file directory.</td>
+ </tr>
<tr>
<td><h5>local-merge-buffer-size</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 8d86219cfb..00c5574d8d 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -1943,6 +1943,13 @@ public class CoreOptions implements Serializable {
.noDefaultValue()
.withDescription("Whether ignore empty commit.");
+ @Immutable
+ public static final ConfigOption<Boolean> INDEX_FILE_IN_DATA_FILE_DIR =
+ key("index-file-in-data-file-dir")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether index file in data file
directory.");
+
private final Options options;
public CoreOptions(Map<String, String> options) {
@@ -2484,6 +2491,10 @@ public class CoreOptions implements Serializable {
return options.get(DISABLE_EXPLICIT_TYPE_CASTING);
}
+ public boolean indexFileInDataFileDir() {
+ return options.get(INDEX_FILE_IN_DATA_FILE_DIR);
+ }
+
public LookupStrategy lookupStrategy() {
return LookupStrategy.from(
mergeEngine().equals(MergeEngine.FIRST_ROW),
diff --git
a/paimon-common/src/main/java/org/apache/paimon/utils/IntFileUtils.java
b/paimon-common/src/main/java/org/apache/paimon/utils/IntFileUtils.java
index e1758dd74e..161dc4f7be 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/IntFileUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/IntFileUtils.java
@@ -46,15 +46,17 @@ public class IntFileUtils {
};
}
- public static void writeInts(FileIO fileIO, Path path, IntIterator input)
throws IOException {
+ public static int writeInts(FileIO fileIO, Path path, IntIterator input)
throws IOException {
try (FastBufferedOutputStream out =
new
FastBufferedOutputStream(fileIO.newOutputStream(path, false));
IntIterator iterator = input) {
+ int count = 0;
while (true) {
try {
writeInt(out, iterator.next());
+ count++;
} catch (EOFException ignored) {
- break;
+ return count;
}
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 1c12877e75..2d71f61580 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -131,7 +131,8 @@ abstract class AbstractFileStore<T> implements FileStore<T>
{
options.fileSuffixIncludeCompression(),
options.fileCompression(),
options.dataFilePathDirectory(),
- createExternalPaths());
+ createExternalPaths(),
+ options.indexFileInDataFileDir());
}
private List<Path> createExternalPaths() {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/compact/CompactDeletionFile.java
b/paimon-core/src/main/java/org/apache/paimon/compact/CompactDeletionFile.java
index 18a3169330..8e9634e86e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/compact/CompactDeletionFile.java
+++
b/paimon-core/src/main/java/org/apache/paimon/compact/CompactDeletionFile.java
@@ -92,7 +92,7 @@ public interface CompactDeletionFile {
@Override
public void clean() {
if (deletionFile != null) {
- dvIndexFile.delete(deletionFile.fileName());
+ dvIndexFile.delete(deletionFile);
}
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionFileWriter.java
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionFileWriter.java
index 48dad0ec84..ca2330664d 100644
---
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionFileWriter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionFileWriter.java
@@ -22,6 +22,7 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.index.DeletionVectorMeta;
import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import java.io.Closeable;
import java.io.DataOutputStream;
@@ -35,11 +36,13 @@ import static
org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.VERSION
public class DeletionFileWriter implements Closeable {
private final Path path;
+ private final boolean isExternalPath;
private final DataOutputStream out;
private final LinkedHashMap<String, DeletionVectorMeta> dvMetas;
- public DeletionFileWriter(Path path, FileIO fileIO) throws IOException {
- this.path = path;
+ public DeletionFileWriter(IndexPathFactory pathFactory, FileIO fileIO)
throws IOException {
+ this.path = pathFactory.newPath();
+ this.isExternalPath = pathFactory.isExternalPath();
this.out = new DataOutputStream(fileIO.newOutputStream(path, true));
out.writeByte(VERSION_ID_V1);
this.dvMetas = new LinkedHashMap<>();
@@ -63,6 +66,11 @@ public class DeletionFileWriter implements Closeable {
public IndexFileMeta result() {
return new IndexFileMeta(
- DELETION_VECTORS_INDEX, path.getName(), getPos(),
dvMetas.size(), dvMetas);
+ DELETION_VECTORS_INDEX,
+ path.getName(),
+ getPos(),
+ dvMetas.size(),
+ dvMetas,
+ isExternalPath ? path.toString() : null);
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
index 4104305e05..67b9738917 100644
---
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java
@@ -20,8 +20,8 @@ package org.apache.paimon.deletionvectors;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.options.MemorySize;
-import org.apache.paimon.utils.PathFactory;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,12 +33,12 @@ import java.util.Map;
/** Writer for deletion vector index file. */
public class DeletionVectorIndexFileWriter {
- private final PathFactory indexPathFactory;
+ private final IndexPathFactory indexPathFactory;
private final FileIO fileIO;
private final long targetSizeInBytes;
public DeletionVectorIndexFileWriter(
- FileIO fileIO, PathFactory pathFactory, MemorySize
targetSizePerIndexFile) {
+ FileIO fileIO, IndexPathFactory pathFactory, MemorySize
targetSizePerIndexFile) {
this.indexPathFactory = pathFactory;
this.fileIO = fileIO;
this.targetSizeInBytes = targetSizePerIndexFile.getBytes();
@@ -53,7 +53,7 @@ public class DeletionVectorIndexFileWriter {
*/
public IndexFileMeta writeSingleFile(Map<String, DeletionVector> input)
throws IOException {
- DeletionFileWriter writer = new
DeletionFileWriter(indexPathFactory.newPath(), fileIO);
+ DeletionFileWriter writer = new DeletionFileWriter(indexPathFactory,
fileIO);
try {
for (Map.Entry<String, DeletionVector> entry : input.entrySet()) {
writer.write(entry.getKey(), entry.getValue());
@@ -79,7 +79,7 @@ public class DeletionVectorIndexFileWriter {
private IndexFileMeta tryWriter(Iterator<Map.Entry<String,
DeletionVector>> iterator)
throws IOException {
- DeletionFileWriter writer = new
DeletionFileWriter(indexPathFactory.newPath(), fileIO);
+ DeletionFileWriter writer = new DeletionFileWriter(indexPathFactory,
fileIO);
try {
while (iterator.hasNext()) {
Map.Entry<String, DeletionVector> entry = iterator.next();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
index cc554b816a..f643b0d6e9 100644
---
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
+++
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java
@@ -24,9 +24,9 @@ import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.index.DeletionVectorMeta;
import org.apache.paimon.index.IndexFile;
import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.table.source.DeletionFile;
-import org.apache.paimon.utils.PathFactory;
import java.io.DataInputStream;
import java.io.IOException;
@@ -51,7 +51,7 @@ public class DeletionVectorsIndexFile extends IndexFile {
public DeletionVectorsIndexFile(
FileIO fileIO,
- PathFactory pathFactory,
+ IndexPathFactory pathFactory,
MemorySize targetSizePerIndexFile,
boolean bitmap64) {
super(fileIO, pathFactory);
@@ -71,13 +71,11 @@ public class DeletionVectorsIndexFile extends IndexFile {
* @throws UncheckedIOException If an I/O error occurs while reading from
the file.
*/
public Map<String, DeletionVector> readAllDeletionVectors(IndexFileMeta
fileMeta) {
- LinkedHashMap<String, DeletionVectorMeta> deletionVectorMetas =
- fileMeta.deletionVectorMetas();
+ LinkedHashMap<String, DeletionVectorMeta> deletionVectorMetas =
fileMeta.dvRanges();
checkNotNull(deletionVectorMetas);
- String indexFileName = fileMeta.fileName();
Map<String, DeletionVector> deletionVectors = new HashMap<>();
- Path filePath = pathFactory.toPath(indexFileName);
+ Path filePath = pathFactory.toPath(fileMeta);
try (SeekableInputStream inputStream =
fileIO.newInputStream(filePath)) {
checkVersion(inputStream);
DataInputStream dataInputStream = new DataInputStream(inputStream);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
index 21d1870b76..b61985a177 100644
---
a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
@@ -82,8 +82,7 @@ public interface BaseAppendDeleteFileMaintainer {
.collect(Collectors.toList());
Map<String, DeletionFile> deletionFiles = new HashMap<>();
for (IndexManifestEntry file : manifestEntries) {
- LinkedHashMap<String, DeletionVectorMeta> dvMetas =
- file.indexFile().deletionVectorMetas();
+ LinkedHashMap<String, DeletionVectorMeta> dvMetas =
file.indexFile().dvRanges();
checkNotNull(dvMetas);
for (DeletionVectorMeta dvMeta : dvMetas.values()) {
deletionFiles.put(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
index 25545c532c..c8a3717025 100644
---
a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
+++
b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
@@ -1178,8 +1178,7 @@ public class IcebergCommitCallback implements
CommitCallback, TagCallback {
return Collections.emptyList();
}
for (IndexManifestEntry entry : newIndexes) {
- LinkedHashMap<String, DeletionVectorMeta> dvMetas =
- entry.indexFile().deletionVectorMetas();
+ LinkedHashMap<String, DeletionVectorMeta> dvMetas =
entry.indexFile().dvRanges();
Path bucketPath =
fileStorePathFactory.bucketPath(entry.partition(), entry.bucket());
if (dvMetas != null) {
for (DeletionVectorMeta dvMeta : dvMetas.values()) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/DynamicBucketIndexMaintainer.java
b/paimon-core/src/main/java/org/apache/paimon/index/DynamicBucketIndexMaintainer.java
index 50397a7a85..e7b3630f1e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/DynamicBucketIndexMaintainer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/DynamicBucketIndexMaintainer.java
@@ -54,7 +54,7 @@ public class DynamicBucketIndexMaintainer {
}
private void restore(HashIndexFile indexFile, IntHashSet hashcode,
IndexFileMeta file) {
- try (IntIterator iterator = indexFile.read(file.fileName())) {
+ try (IntIterator iterator = indexFile.read(file)) {
while (true) {
try {
hashcode.add(iterator.next());
@@ -82,7 +82,7 @@ public class DynamicBucketIndexMaintainer {
if (modified) {
IndexFileMeta entry;
try {
- entry = indexFile.write(hashcode.size(),
hashcode.toIntIterator());
+ entry = indexFile.write(hashcode.toIntIterator());
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/HashIndexFile.java
b/paimon-core/src/main/java/org/apache/paimon/index/HashIndexFile.java
index 33deb96a62..3457ac30d6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/HashIndexFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/HashIndexFile.java
@@ -21,7 +21,6 @@ package org.apache.paimon.index;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.utils.IntIterator;
-import org.apache.paimon.utils.PathFactory;
import java.io.IOException;
import java.util.List;
@@ -34,34 +33,31 @@ public class HashIndexFile extends IndexFile {
public static final String HASH_INDEX = "HASH";
- public HashIndexFile(FileIO fileIO, PathFactory pathFactory) {
+ public HashIndexFile(FileIO fileIO, IndexPathFactory pathFactory) {
super(fileIO, pathFactory);
}
- public Path path(String fileName) {
- return pathFactory.toPath(fileName);
+ public IntIterator read(IndexFileMeta file) throws IOException {
+ return readInts(fileIO, pathFactory.toPath(file));
}
- public IntIterator read(String fileName) throws IOException {
- return readInts(fileIO, pathFactory.toPath(fileName));
+ public List<Integer> readList(IndexFileMeta file) throws IOException {
+ return IntIterator.toIntList(read(file));
}
- public List<Integer> readList(String fileName) throws IOException {
- return IntIterator.toIntList(read(fileName));
- }
-
- public String write(IntIterator input) throws IOException {
+ public IndexFileMeta write(IntIterator input) throws IOException {
Path path = pathFactory.newPath();
- writeInts(fileIO, path, input);
- return path.getName();
- }
-
- public IndexFileMeta write(int size, IntIterator input) throws IOException
{
- String fileName = write(input);
- return new IndexFileMeta(HASH_INDEX, fileName, fileSize(fileName),
size);
+ int count = writeInts(fileIO, path, input);
+ return new IndexFileMeta(
+ HASH_INDEX,
+ path.getName(),
+ fileSize(path),
+ count,
+ null,
+ isExternalPath() ? path.toString() : null);
}
public IndexFileMeta write(int[] ints) throws IOException {
- return write(ints.length, IntIterator.create(ints));
+ return write(IntIterator.create(ints));
}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFile.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFile.java
index 441cdd80ac..4cf30da3c4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFile.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFile.java
@@ -20,7 +20,6 @@ package org.apache.paimon.index;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.utils.PathFactory;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -30,34 +29,42 @@ public abstract class IndexFile {
protected final FileIO fileIO;
- protected final PathFactory pathFactory;
+ protected final IndexPathFactory pathFactory;
- public IndexFile(FileIO fileIO, PathFactory pathFactory) {
+ public IndexFile(FileIO fileIO, IndexPathFactory pathFactory) {
this.fileIO = fileIO;
this.pathFactory = pathFactory;
}
- public Path path(String fileName) {
- return pathFactory.toPath(fileName);
+ public Path path(IndexFileMeta file) {
+ return pathFactory.toPath(file);
}
- public long fileSize(String fileName) {
+ public long fileSize(IndexFileMeta file) {
+ return fileSize(path(file));
+ }
+
+ public long fileSize(Path file) {
try {
- return fileIO.getFileSize(path(fileName));
+ return fileIO.getFileSize(file);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
- public void delete(String fileName) {
- fileIO.deleteQuietly(path(fileName));
+ public void delete(IndexFileMeta file) {
+ fileIO.deleteQuietly(path(file));
}
- public boolean exists(String fileName) {
+ public boolean exists(IndexFileMeta file) {
try {
- return fileIO.exists(path(fileName));
+ return fileIO.exists(path(file));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
+
+ public boolean isExternalPath() {
+ return pathFactory.isExternalPath();
+ }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
index 186f375633..205db897ea 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java
@@ -171,9 +171,7 @@ public class IndexFileHandler {
}
public Path filePath(IndexManifestEntry entry) {
- return pathFactories
- .get(entry.partition(), entry.bucket())
- .toPath(entry.indexFile().fileName());
+ return pathFactories.get(entry.partition(),
entry.bucket()).toPath(entry.indexFile());
}
public boolean existsManifest(String indexManifest) {
@@ -202,11 +200,11 @@ public class IndexFileHandler {
}
public boolean existsIndexFile(IndexManifestEntry file) {
- return indexFile(file).exists(file.indexFile().fileName());
+ return indexFile(file).exists(file.indexFile());
}
public void deleteIndexFile(IndexManifestEntry entry) {
- indexFile(entry).delete(entry.indexFile().fileName());
+ indexFile(entry).delete(entry.indexFile());
}
public void deleteManifest(String indexManifest) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
index aae4f8c473..f90566792d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java
@@ -52,7 +52,8 @@ public class IndexFileMeta {
new DataField(
4,
"_DELETIONS_VECTORS_RANGES",
- new ArrayType(true,
DeletionVectorMeta.SCHEMA))));
+ new ArrayType(true,
DeletionVectorMeta.SCHEMA)),
+ new DataField(5, "_EXTERNAL_PATH",
newStringType(true))));
private final String indexType;
private final String fileName;
@@ -63,23 +64,23 @@ public class IndexFileMeta {
* Metadata only used by {@link DeletionVectorsIndexFile}, use
LinkedHashMap to ensure that the
* order of DeletionVectorMetas and the written DeletionVectors is
consistent.
*/
- private final @Nullable LinkedHashMap<String, DeletionVectorMeta>
deletionVectorMetas;
+ private final @Nullable LinkedHashMap<String, DeletionVectorMeta> dvRanges;
- public IndexFileMeta(String indexType, String fileName, long fileSize,
long rowCount) {
- this(indexType, fileName, fileSize, rowCount, null);
- }
+ private final @Nullable String externalPath;
public IndexFileMeta(
String indexType,
String fileName,
long fileSize,
long rowCount,
- @Nullable LinkedHashMap<String, DeletionVectorMeta>
deletionVectorMetas) {
+ @Nullable LinkedHashMap<String, DeletionVectorMeta> dvRanges,
+ @Nullable String externalPath) {
this.indexType = indexType;
this.fileName = fileName;
this.fileSize = fileSize;
this.rowCount = rowCount;
- this.deletionVectorMetas = deletionVectorMetas;
+ this.dvRanges = dvRanges;
+ this.externalPath = externalPath;
}
public String indexType() {
@@ -98,8 +99,13 @@ public class IndexFileMeta {
return rowCount;
}
- public @Nullable LinkedHashMap<String, DeletionVectorMeta>
deletionVectorMetas() {
- return deletionVectorMetas;
+ public @Nullable LinkedHashMap<String, DeletionVectorMeta> dvRanges() {
+ return dvRanges;
+ }
+
+ @Nullable
+ public String externalPath() {
+ return externalPath;
}
@Override
@@ -115,12 +121,13 @@ public class IndexFileMeta {
&& Objects.equals(fileName, that.fileName)
&& fileSize == that.fileSize
&& rowCount == that.rowCount
- && Objects.equals(deletionVectorMetas,
that.deletionVectorMetas);
+ && Objects.equals(dvRanges, that.dvRanges)
+ && Objects.equals(externalPath, that.externalPath);
}
@Override
public int hashCode() {
- return Objects.hash(indexType, fileName, fileSize, rowCount,
deletionVectorMetas);
+ return Objects.hash(indexType, fileName, fileSize, rowCount, dvRanges,
externalPath);
}
@Override
@@ -135,8 +142,10 @@ public class IndexFileMeta {
+ fileSize
+ ", rowCount="
+ rowCount
- + ", deletionVectorMetas="
- + deletionVectorMetas
+ + ", dvRanges="
+ + dvRanges
+ + ", externalPath='"
+ + externalPath
+ '}';
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
index db4a44838f..4496ec50f4 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
@@ -18,7 +18,6 @@
package org.apache.paimon.index;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalArray;
@@ -29,6 +28,8 @@ import org.apache.paimon.utils.VersionedObjectSerializer;
import java.util.Collection;
import java.util.LinkedHashMap;
+import static org.apache.paimon.data.BinaryString.fromString;
+
/** A {@link VersionedObjectSerializer} for {@link IndexFileMeta}. */
public class IndexFileMetaSerializer extends ObjectSerializer<IndexFileMeta> {
@@ -39,13 +40,12 @@ public class IndexFileMetaSerializer extends
ObjectSerializer<IndexFileMeta> {
@Override
public InternalRow toRow(IndexFileMeta record) {
return GenericRow.of(
- BinaryString.fromString(record.indexType()),
- BinaryString.fromString(record.fileName()),
+ fromString(record.indexType()),
+ fromString(record.fileName()),
record.fileSize(),
record.rowCount(),
- record.deletionVectorMetas() == null
- ? null
- :
dvMetasToRowArrayData(record.deletionVectorMetas().values()));
+ dvMetasToRowArrayData(record.dvRanges()),
+ fromString(record.externalPath()));
}
@Override
@@ -55,7 +55,16 @@ public class IndexFileMetaSerializer extends
ObjectSerializer<IndexFileMeta> {
row.getString(1).toString(),
row.getLong(2),
row.getLong(3),
- row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)));
+ row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)),
+ row.isNullAt(5) ? null : row.getString(5).toString());
+ }
+
+ public static InternalArray dvMetasToRowArrayData(
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges) {
+ if (dvRanges == null) {
+ return null;
+ }
+ return dvMetasToRowArrayData(dvRanges.values());
}
public static InternalArray
dvMetasToRowArrayData(Collection<DeletionVectorMeta> dvMetas) {
@@ -64,7 +73,7 @@ public class IndexFileMetaSerializer extends
ObjectSerializer<IndexFileMeta> {
.map(
dvMeta ->
GenericRow.of(
-
BinaryString.fromString(dvMeta.dataFileName()),
+
fromString(dvMeta.dataFileName()),
dvMeta.offset(),
dvMeta.length(),
dvMeta.cardinality()))
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV1Deserializer.java
similarity index 96%
copy from
paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java
copy to
paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV1Deserializer.java
index 915d904569..36e5190540 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV1Deserializer.java
@@ -39,7 +39,7 @@ import java.util.List;
import static org.apache.paimon.utils.SerializationUtils.newStringType;
/** Serializer for {@link IndexFileMeta} with 0.9 version. */
-public class IndexFileMeta09Serializer implements Serializable {
+public class IndexFileMetaV1Deserializer implements Serializable {
private static final long serialVersionUID = 1L;
@@ -63,7 +63,7 @@ public class IndexFileMeta09Serializer implements
Serializable {
protected final InternalRowSerializer rowSerializer;
- public IndexFileMeta09Serializer() {
+ public IndexFileMetaV1Deserializer() {
this.rowSerializer = InternalSerializers.create(SCHEMA);
}
@@ -73,7 +73,8 @@ public class IndexFileMeta09Serializer implements
Serializable {
row.getString(1).toString(),
row.getLong(2),
row.getLong(3),
- row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)));
+ row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)),
+ null);
}
public final List<IndexFileMeta> deserializeList(DataInputView source)
throws IOException {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV2Deserializer.java
similarity index 70%
rename from
paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java
rename to
paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV2Deserializer.java
index 915d904569..32f192938f 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV2Deserializer.java
@@ -18,7 +18,6 @@
package org.apache.paimon.index;
-import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.data.serializer.InternalSerializers;
@@ -26,20 +25,19 @@ import org.apache.paimon.io.DataInputView;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.LinkedHashMap;
import java.util.List;
+import static
org.apache.paimon.index.IndexFileMetaSerializer.rowArrayDataToDvMetas;
import static org.apache.paimon.utils.SerializationUtils.newStringType;
-/** Serializer for {@link IndexFileMeta} with 0.9 version. */
-public class IndexFileMeta09Serializer implements Serializable {
+/** Serializer for {@link IndexFileMeta} with 1.2 version. */
+public class IndexFileMetaV2Deserializer implements Serializable {
private static final long serialVersionUID = 1L;
@@ -54,16 +52,11 @@ public class IndexFileMeta09Serializer implements
Serializable {
new DataField(
4,
"_DELETIONS_VECTORS_RANGES",
- new ArrayType(
- true,
- RowType.of(
- newStringType(false),
- new IntType(false),
- new IntType(false))))));
+ new ArrayType(true,
DeletionVectorMeta.SCHEMA))));
protected final InternalRowSerializer rowSerializer;
- public IndexFileMeta09Serializer() {
+ public IndexFileMetaV2Deserializer() {
this.rowSerializer = InternalSerializers.create(SCHEMA);
}
@@ -73,7 +66,8 @@ public class IndexFileMeta09Serializer implements
Serializable {
row.getString(1).toString(),
row.getLong(2),
row.getLong(3),
- row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)));
+ row.isNullAt(4) ? null :
rowArrayDataToDvMetas(row.getArray(4)),
+ null);
}
public final List<IndexFileMeta> deserializeList(DataInputView source)
throws IOException {
@@ -88,17 +82,4 @@ public class IndexFileMeta09Serializer implements
Serializable {
public IndexFileMeta deserialize(DataInputView in) throws IOException {
return fromRow(rowSerializer.deserialize(in));
}
-
- public static LinkedHashMap<String, DeletionVectorMeta>
rowArrayDataToDvMetas(
- InternalArray arrayData) {
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>(arrayData.size());
- for (int i = 0; i < arrayData.size(); i++) {
- InternalRow row = arrayData.getRow(i, 3);
- dvMetas.put(
- row.getString(0).toString(),
- new DeletionVectorMeta(
- row.getString(0).toString(), row.getInt(1),
row.getInt(2), null));
- }
- return dvMetas;
- }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java
new file mode 100644
index 0000000000..e6808c14b9
--- /dev/null
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexInDataFileDirPathFactory.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.index;
+
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.io.DataFilePathFactory;
+
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.paimon.utils.FileStorePathFactory.INDEX_PREFIX;
+
+/** Path factory to create an index path. */
+public class IndexInDataFileDirPathFactory implements IndexPathFactory {
+
+ private final String uuid;
+ private final DataFilePathFactory dataFilePathFactory;
+ private final AtomicInteger indexFileCount;
+
+ public IndexInDataFileDirPathFactory(
+ String uuid, AtomicInteger indexFileCount, DataFilePathFactory
dataFilePathFactory) {
+ this.uuid = uuid;
+ this.dataFilePathFactory = dataFilePathFactory;
+ this.indexFileCount = indexFileCount;
+ }
+
+ @Override
+ public Path newPath() {
+ String name = INDEX_PREFIX + uuid + "-" +
indexFileCount.getAndIncrement();
+ return dataFilePathFactory.newPathFromName(name);
+ }
+
+ @Override
+ public Path toPath(IndexFileMeta file) {
+ return Optional.ofNullable(file.externalPath())
+ .map(Path::new)
+ .orElse(new Path(dataFilePathFactory.parent(),
file.fileName()));
+ }
+
+ @Override
+ public boolean isExternalPath() {
+ return dataFilePathFactory.isExternalPath();
+ }
+}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java
similarity index 54%
copy from
paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
copy to paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java
index 934e6b01f2..4e5d0a6d01 100644
---
a/paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexPathFactory.java
@@ -16,26 +16,16 @@
* limitations under the License.
*/
-package org.apache.paimon.utils;
+package org.apache.paimon.index;
-import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.fs.Path;
-import java.util.HashMap;
-import java.util.Map;
+/** Path factory to create an index path. */
+public interface IndexPathFactory {
-/** Cache for index {@link PathFactory}s. */
-public class IndexFilePathFactories {
+ Path newPath();
- private final Map<Pair<BinaryRow, Integer>, PathFactory> cache = new
HashMap<>();
- private final FileStorePathFactory pathFactory;
+ Path toPath(IndexFileMeta file);
- public IndexFilePathFactories(FileStorePathFactory pathFactory) {
- this.pathFactory = pathFactory;
- }
-
- public PathFactory get(BinaryRow partition, int bucket) {
- return cache.computeIfAbsent(
- Pair.of(partition, bucket),
- k -> pathFactory.indexFileFactory(k.getKey(), k.getValue()));
- }
+ boolean isExternalPath();
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/PartitionIndex.java
b/paimon-core/src/main/java/org/apache/paimon/index/PartitionIndex.java
index 1e525ea4e0..b0404b9d13 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/PartitionIndex.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/PartitionIndex.java
@@ -130,7 +130,7 @@ public class PartitionIndex {
try (IntIterator iterator =
indexFileHandler
.hashIndex(file.partition(), file.bucket())
- .read(file.indexFile().fileName())) {
+ .read(file.indexFile())) {
while (true) {
try {
int hash = iterator.next();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
index 8be778fc86..3baa9d226a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
@@ -70,6 +70,10 @@ public class DataFilePathFactory {
this.externalPathProvider = externalPathProvider;
}
+ public Path parent() {
+ return parent;
+ }
+
public String dataFilePrefix() {
return dataFilePrefix;
}
@@ -106,7 +110,7 @@ public class DataFilePathFactory {
return newPathFromName(newFileName(dataFilePrefix, extension));
}
- private Path newPathFromName(String fileName) {
+ public Path newPathFromName(String fileName) {
if (externalPathProvider != null) {
return externalPathProvider.getNextExternalDataPath(fileName);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java
index 2431a1c264..3fe3cf5083 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java
@@ -58,7 +58,8 @@ public class IndexManifestEntry {
new DataField(
7,
"_DELETIONS_VECTORS_RANGES",
- new ArrayType(true,
DeletionVectorMeta.SCHEMA))));
+ new ArrayType(true,
DeletionVectorMeta.SCHEMA)),
+ new DataField(8, "_EXTERNAL_PATH",
newStringType(true))));
private final FileKind kind;
private final BinaryRow partition;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
index 6f2ec17dda..581027c7bb 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
@@ -18,13 +18,14 @@
package org.apache.paimon.manifest;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.index.IndexFileMetaSerializer;
import org.apache.paimon.utils.VersionedObjectSerializer;
+import static org.apache.paimon.data.BinaryString.fromString;
+import static
org.apache.paimon.index.IndexFileMetaSerializer.dvMetasToRowArrayData;
+import static
org.apache.paimon.index.IndexFileMetaSerializer.rowArrayDataToDvMetas;
import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
@@ -47,14 +48,12 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
record.kind().toByteValue(),
serializeBinaryRow(record.partition()),
record.bucket(),
- BinaryString.fromString(indexFile.indexType()),
- BinaryString.fromString(indexFile.fileName()),
+ fromString(indexFile.indexType()),
+ fromString(indexFile.fileName()),
indexFile.fileSize(),
indexFile.rowCount(),
- record.indexFile().deletionVectorMetas() == null
- ? null
- : IndexFileMetaSerializer.dvMetasToRowArrayData(
-
record.indexFile().deletionVectorMetas().values()));
+ dvMetasToRowArrayData(indexFile.dvRanges()),
+ fromString(indexFile.externalPath()));
}
@Override
@@ -72,8 +71,7 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
row.getString(4).toString(),
row.getLong(5),
row.getLong(6),
- row.isNullAt(7)
- ? null
- :
IndexFileMetaSerializer.rowArrayDataToDvMetas(row.getArray(7))));
+ row.isNullAt(7) ? null :
rowArrayDataToDvMetas(row.getArray(7)),
+ row.isNullAt(8) ? null : row.getString(8).toString()));
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java
b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java
index e3ebbf1ad1..88e9e3513d 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java
@@ -186,6 +186,7 @@ public class CommitMessageLegacyV2Serializer {
row.getString(1).toString(),
row.getLong(2),
row.getLong(3),
+ null,
null);
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
index 47c69855cb..5c5fc7c0c0 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java
@@ -20,8 +20,9 @@ package org.apache.paimon.table.sink;
import org.apache.paimon.data.serializer.VersionedSerializer;
import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.index.IndexFileMeta09Serializer;
import org.apache.paimon.index.IndexFileMetaSerializer;
+import org.apache.paimon.index.IndexFileMetaV1Deserializer;
+import org.apache.paimon.index.IndexFileMetaV2Deserializer;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFileMeta08Serializer;
@@ -58,7 +59,8 @@ public class CommitMessageSerializer implements
VersionedSerializer<CommitMessag
private DataFileMeta10LegacySerializer dataFileMeta10LegacySerializer;
private DataFileMeta09Serializer dataFile09Serializer;
private DataFileMeta08Serializer dataFile08Serializer;
- private IndexFileMeta09Serializer indexEntry09Serializer;
+ private IndexFileMetaV1Deserializer indexEntryV1Deserializer;
+ private IndexFileMetaV2Deserializer indexEntryV2Deserializer;
public CommitMessageSerializer() {
this.dataFileSerializer = new DataFileMetaSerializer();
@@ -171,13 +173,18 @@ public class CommitMessageSerializer implements
VersionedSerializer<CommitMessag
private IOExceptionSupplier<List<IndexFileMeta>> indexEntryDeserializer(
int version, DataInputView view) {
- if (version >= 5) {
+ if (version >= 8) {
return () -> indexEntrySerializer.deserializeList(view);
+ } else if (version >= 5) {
+ if (indexEntryV2Deserializer == null) {
+ indexEntryV2Deserializer = new IndexFileMetaV2Deserializer();
+ }
+ return () -> indexEntryV2Deserializer.deserializeList(view);
} else {
- if (indexEntry09Serializer == null) {
- indexEntry09Serializer = new IndexFileMeta09Serializer();
+ if (indexEntryV1Deserializer == null) {
+ indexEntryV1Deserializer = new IndexFileMetaV1Deserializer();
}
- return () -> indexEntry09Serializer.deserializeList(view);
+ return () -> indexEntryV1Deserializer.deserializeList(view);
}
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
index fd2aa13f36..c1a69a2391 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
@@ -21,7 +21,7 @@ package org.apache.paimon.table.sink;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.consumer.ConsumerManager;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.manifest.ManifestCommittable;
@@ -36,7 +36,6 @@ import org.apache.paimon.tag.TagTimeExpire;
import org.apache.paimon.utils.DataFilePathFactories;
import org.apache.paimon.utils.ExecutorThreadFactory;
import org.apache.paimon.utils.IndexFilePathFactories;
-import org.apache.paimon.utils.PathFactory;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
import
org.apache.paimon.shade.guava30.com.google.common.util.concurrent.MoreExecutors;
@@ -285,7 +284,7 @@ public class TableCommitImpl implements InnerTableCommit {
CommitMessageImpl msg = (CommitMessageImpl) message;
DataFilePathFactory pathFactory =
factories.get(message.partition(), message.bucket());
- PathFactory indexFileFactory =
+ IndexPathFactory indexFileFactory =
indexFactories.get(message.partition(),
message.bucket());
Consumer<DataFileMeta> collector = f ->
files.addAll(f.collectFiles(pathFactory));
msg.newFilesIncrement().newFiles().forEach(collector);
@@ -293,11 +292,9 @@ public class TableCommitImpl implements InnerTableCommit {
msg.compactIncrement().compactBefore().forEach(collector);
msg.compactIncrement().compactAfter().forEach(collector);
msg.indexIncrement().newIndexFiles().stream()
- .map(IndexFileMeta::fileName)
.map(indexFileFactory::toPath)
.forEach(files::add);
msg.indexIncrement().deletedIndexFiles().stream()
- .map(IndexFileMeta::fileName)
.map(indexFileFactory::toPath)
.forEach(files::add);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
index e7c8366372..a51c1516cb 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java
@@ -566,8 +566,8 @@ public class SnapshotReaderImpl implements SnapshotReader {
List<DeletionFile> deletionFiles = new ArrayList<>(dataFiles.size());
Map<String, IndexFileMeta> dataFileToIndexFileMeta = new HashMap<>();
for (IndexFileMeta indexFileMeta : indexFileMetas) {
- if (indexFileMeta.deletionVectorMetas() != null) {
- for (DeletionVectorMeta dvMeta :
indexFileMeta.deletionVectorMetas().values()) {
+ if (indexFileMeta.dvRanges() != null) {
+ for (DeletionVectorMeta dvMeta :
indexFileMeta.dvRanges().values()) {
dataFileToIndexFileMeta.put(dvMeta.dataFileName(),
indexFileMeta);
}
}
@@ -575,12 +575,11 @@ public class SnapshotReaderImpl implements SnapshotReader
{
for (DataFileMeta file : dataFiles) {
IndexFileMeta indexFileMeta =
dataFileToIndexFileMeta.get(file.fileName());
if (indexFileMeta != null) {
- LinkedHashMap<String, DeletionVectorMeta> dvMetas =
- indexFileMeta.deletionVectorMetas();
+ LinkedHashMap<String, DeletionVectorMeta> dvMetas =
indexFileMeta.dvRanges();
if (dvMetas != null && dvMetas.containsKey(file.fileName())) {
deletionFiles.add(
new DeletionFile(
-
indexFile.path(indexFileMeta.fileName()).toString(),
+ indexFile.path(indexFileMeta).toString(),
dvMetas.get(file.fileName()).offset(),
dvMetas.get(file.fileName()).length(),
dvMetas.get(file.fileName()).cardinality()));
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/system/TableIndexesTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/system/TableIndexesTable.java
index d49efb41d5..9cc1d95962 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/system/TableIndexesTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/system/TableIndexesTable.java
@@ -214,7 +214,7 @@ public class TableIndexesTable implements ReadonlyTable {
IndexManifestEntry indexManifestEntry,
CastExecutor<InternalRow, BinaryString> partitionCastExecutor)
{
LinkedHashMap<String, DeletionVectorMeta> dvMetas =
- indexManifestEntry.indexFile().deletionVectorMetas();
+ indexManifestEntry.indexFile().dvRanges();
return GenericRow.of(
partitionCastExecutor.cast(indexManifestEntry.partition()),
indexManifestEntry.bucket(),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
index 63d2b06dc0..32135518ba 100644
---
a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
+++
b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java
@@ -23,6 +23,9 @@ import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.fs.ExternalPathProvider;
import org.apache.paimon.fs.Path;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexInDataFileDirPathFactory;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.types.RowType;
@@ -67,6 +70,7 @@ public class FileStorePathFactory {
private final String fileCompression;
@Nullable private final String dataFilePathDirectory;
+ private final boolean indexFileInDataFileDir;
private final AtomicInteger manifestFileCount;
private final AtomicInteger manifestListCount;
@@ -86,9 +90,11 @@ public class FileStorePathFactory {
boolean fileSuffixIncludeCompression,
String fileCompression,
@Nullable String dataFilePathDirectory,
- List<Path> externalPaths) {
+ List<Path> externalPaths,
+ boolean indexFileInDataFileDir) {
this.root = root;
this.dataFilePathDirectory = dataFilePathDirectory;
+ this.indexFileInDataFileDir = indexFileInDataFileDir;
this.uuid = UUID.randomUUID().toString();
this.partitionComputer =
@@ -268,18 +274,32 @@ public class FileStorePathFactory {
};
}
- public PathFactory indexFileFactory(BinaryRow partition, int bucket) {
- return new PathFactory() {
- @Override
- public Path newPath() {
- return toPath(INDEX_PREFIX + uuid + "-" +
indexFileCount.getAndIncrement());
- }
-
- @Override
- public Path toPath(String fileName) {
- return new Path(indexPath(), fileName);
- }
- };
+ public IndexPathFactory indexFileFactory(BinaryRow partition, int bucket) {
+ if (indexFileInDataFileDir) {
+ DataFilePathFactory dataFilePathFactory =
createDataFilePathFactory(partition, bucket);
+ return new IndexInDataFileDirPathFactory(uuid, indexFileCount,
dataFilePathFactory);
+ } else {
+ return new IndexPathFactory() {
+ @Override
+ public Path newPath() {
+ return toPath(INDEX_PREFIX + uuid + "-" +
indexFileCount.getAndIncrement());
+ }
+
+ @Override
+ public Path toPath(IndexFileMeta file) {
+ return toPath(file.fileName());
+ }
+
+ @Override
+ public boolean isExternalPath() {
+ return false;
+ }
+
+ private Path toPath(String fileName) {
+ return new Path(indexPath(), fileName);
+ }
+ };
+ }
}
public PathFactory statsFileFactory() {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
b/paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
index 934e6b01f2..72e8233a1e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
+++
b/paimon-core/src/main/java/org/apache/paimon/utils/IndexFilePathFactories.java
@@ -19,6 +19,7 @@
package org.apache.paimon.utils;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.index.IndexPathFactory;
import java.util.HashMap;
import java.util.Map;
@@ -26,14 +27,14 @@ import java.util.Map;
/** Cache for index {@link PathFactory}s. */
public class IndexFilePathFactories {
- private final Map<Pair<BinaryRow, Integer>, PathFactory> cache = new
HashMap<>();
+ private final Map<Pair<BinaryRow, Integer>, IndexPathFactory> cache = new
HashMap<>();
private final FileStorePathFactory pathFactory;
public IndexFilePathFactories(FileStorePathFactory pathFactory) {
this.pathFactory = pathFactory;
}
- public PathFactory get(BinaryRow partition, int bucket) {
+ public IndexPathFactory get(BinaryRow partition, int bucket) {
return cache.computeIfAbsent(
Pair.of(partition, bucket),
k -> pathFactory.indexFileFactory(k.getKey(), k.getValue()));
diff --git
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
index d30462c25d..516441f7aa 100644
---
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
@@ -22,9 +22,9 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.index.DeletionVectorMeta;
import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.table.source.DeletionFile;
-import org.apache.paimon.utils.PathFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -52,7 +52,7 @@ public class DeletionVectorsIndexFileTest {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadDvIndex(boolean bitmap64) {
- PathFactory pathFactory = getPathFactory();
+ IndexPathFactory pathFactory = getPathFactory();
DeletionVectorsIndexFile deletionVectorsIndexFile =
deletionVectorsIndexFile(pathFactory, bitmap64);
@@ -76,7 +76,7 @@ public class DeletionVectorsIndexFileTest {
assertThat(indexFiles.size()).isEqualTo(1);
// read
- String fileName = indexFiles.get(0).fileName();
+ IndexFileMeta file = indexFiles.get(0);
Map<String, DeletionVector> actualDeleteMap =
deletionVectorsIndexFile.readAllDeletionVectors(indexFiles);
assertThat(actualDeleteMap.get("file1.parquet").isDeleted(1)).isTrue();
@@ -86,14 +86,14 @@ public class DeletionVectorsIndexFileTest {
assertThat(actualDeleteMap.get("file33.parquet").isDeleted(3)).isTrue();
// delete
- deletionVectorsIndexFile.delete(fileName);
- assertThat(deletionVectorsIndexFile.exists(fileName)).isFalse();
+ deletionVectorsIndexFile.delete(file);
+ assertThat(deletionVectorsIndexFile.exists(file)).isFalse();
}
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadDvIndexWithCopiousDv(boolean bitmap64) {
- PathFactory pathFactory = getPathFactory();
+ IndexPathFactory pathFactory = getPathFactory();
DeletionVectorsIndexFile deletionVectorsIndexFile =
deletionVectorsIndexFile(pathFactory, bitmap64);
@@ -125,7 +125,7 @@ public class DeletionVectorsIndexFileTest {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadDvIndexWithEnormousDv(boolean bitmap64) {
- PathFactory pathFactory = getPathFactory();
+ IndexPathFactory pathFactory = getPathFactory();
DeletionVectorsIndexFile deletionVectorsIndexFile =
deletionVectorsIndexFile(pathFactory, bitmap64);
@@ -157,7 +157,7 @@ public class DeletionVectorsIndexFileTest {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testWriteDVIndexWithLimitedTargetSizePerIndexFile(boolean
bitmap64) {
- PathFactory pathFactory = getPathFactory();
+ IndexPathFactory pathFactory = getPathFactory();
DeletionVectorsIndexFile deletionVectorsIndexFile =
deletionVectorsIndexFile(pathFactory, MemorySize.parse("2MB"),
bitmap64);
@@ -208,7 +208,7 @@ public class DeletionVectorsIndexFileTest {
@Test
public void testReadV1AndV2() {
- PathFactory pathFactory = getPathFactory();
+ IndexPathFactory pathFactory = getPathFactory();
DeletionVectorsIndexFile v1DeletionVectorsIndexFile =
deletionVectorsIndexFile(pathFactory, false);
DeletionVectorsIndexFile v2DeletionVectorsIndexFile =
@@ -263,7 +263,7 @@ public class DeletionVectorsIndexFileTest {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadDeletionFile(boolean bitmap64) throws IOException {
- PathFactory pathFactory = getPathFactory();
+ IndexPathFactory pathFactory = getPathFactory();
DeletionVectorsIndexFile deletionVectorsIndexFile =
deletionVectorsIndexFile(pathFactory, bitmap64);
@@ -278,12 +278,11 @@ public class DeletionVectorsIndexFileTest {
assertThat(indexFiles.size()).isEqualTo(1);
IndexFileMeta indexFileMeta = indexFiles.get(0);
- DeletionVectorMeta deletionVectorMeta =
- indexFileMeta.deletionVectorMetas().get("file1.parquet");
+ DeletionVectorMeta deletionVectorMeta =
indexFileMeta.dvRanges().get("file1.parquet");
DeletionFile deletionFile =
new DeletionFile(
-
pathFactory.toPath(indexFileMeta.fileName()).toString(),
+ pathFactory.toPath(indexFileMeta).toString(),
deletionVectorMeta.offset(),
deletionVectorMeta.length(),
deletionVectorMeta.cardinality());
@@ -341,27 +340,32 @@ public class DeletionVectorsIndexFileTest {
}
private DeletionVectorsIndexFile deletionVectorsIndexFile(
- PathFactory pathFactory, boolean bitmap64) {
+ IndexPathFactory pathFactory, boolean bitmap64) {
return deletionVectorsIndexFile(pathFactory,
MemorySize.ofBytes(Long.MAX_VALUE), bitmap64);
}
private DeletionVectorsIndexFile deletionVectorsIndexFile(
- PathFactory pathFactory, MemorySize targetSizePerIndexFile,
boolean bitmap64) {
+ IndexPathFactory pathFactory, MemorySize targetSizePerIndexFile,
boolean bitmap64) {
return new DeletionVectorsIndexFile(
LocalFileIO.create(), pathFactory, targetSizePerIndexFile,
bitmap64);
}
- private PathFactory getPathFactory() {
+ private IndexPathFactory getPathFactory() {
Path dir = new Path(tempPath.toUri());
- return new PathFactory() {
+ return new IndexPathFactory() {
@Override
public Path newPath() {
return new Path(dir, UUID.randomUUID().toString());
}
@Override
- public Path toPath(String fileName) {
- return new Path(dir, fileName);
+ public boolean isExternalPath() {
+ return false;
+ }
+
+ @Override
+ public Path toPath(IndexFileMeta file) {
+ return new Path(dir, file.fileName());
}
};
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java
index ab0376095c..3cd9308d79 100644
---
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java
@@ -26,11 +26,11 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.index.DeletionVectorMeta;
import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.table.sink.CommitMessageImpl;
import org.apache.paimon.table.source.DeletionFile;
-import org.apache.paimon.utils.PathFactory;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
@@ -66,7 +66,8 @@ class AppendDeletionFileMaintainerTest {
Collections.singletonMap("f3", Arrays.asList(1, 2,
3)));
store.commit(commitMessage1, commitMessage2);
- PathFactory indexPathFactory =
store.pathFactory().indexFileFactory(BinaryRow.EMPTY_ROW, 0);
+ IndexPathFactory indexPathFactory =
+ store.pathFactory().indexFileFactory(BinaryRow.EMPTY_ROW, 0);
Map<String, DeletionFile> dataFileToDeletionFiles = new HashMap<>();
dataFileToDeletionFiles.putAll(
createDeletionFileMapFromIndexFileMetas(
@@ -99,7 +100,7 @@ class AppendDeletionFileMaintainerTest {
assertThat(res.size()).isEqualTo(3);
IndexManifestEntry entry =
res.stream().filter(file -> file.kind() ==
FileKind.ADD).findAny().get();
-
assertThat(entry.indexFile().deletionVectorMetas().containsKey("f2")).isTrue();
+ assertThat(entry.indexFile().dvRanges().containsKey("f2")).isTrue();
entry =
res.stream()
.filter(file -> file.kind() == FileKind.DELETE)
@@ -119,15 +120,15 @@ class AppendDeletionFileMaintainerTest {
}
private Map<String, DeletionFile> createDeletionFileMapFromIndexFileMetas(
- PathFactory indexPathFactory, List<IndexFileMeta> fileMetas) {
+ IndexPathFactory indexPathFactory, List<IndexFileMeta> fileMetas) {
Map<String, DeletionFile> dataFileToDeletionFiles = new HashMap<>();
for (IndexFileMeta indexFileMeta : fileMetas) {
for (Map.Entry<String, DeletionVectorMeta> dvMeta :
- indexFileMeta.deletionVectorMetas().entrySet()) {
+ indexFileMeta.dvRanges().entrySet()) {
dataFileToDeletionFiles.put(
dvMeta.getKey(),
new DeletionFile(
-
indexPathFactory.toPath(indexFileMeta.fileName()).toString(),
+
indexPathFactory.toPath(indexFileMeta).toString(),
dvMeta.getValue().offset(),
dvMeta.getValue().length(),
dvMeta.getValue().cardinality()));
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/DynamicBucketIndexMaintainerTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/DynamicBucketIndexMaintainerTest.java
index 769817d63b..6f5e0d2ecd 100644
---
a/paimon-core/src/test/java/org/apache/paimon/index/DynamicBucketIndexMaintainerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/index/DynamicBucketIndexMaintainerTest.java
@@ -81,7 +81,7 @@ public class DynamicBucketIndexMaintainerTest extends
PrimaryKeyTableTestBase {
}
int[] ints =
fileHandler.hashIndex(message.partition(),
message.bucket())
- .readList(files.get(0).fileName()).stream()
+ .readList(files.get(0)).stream()
.mapToInt(Integer::intValue)
.toArray();
index.computeIfAbsent(message.partition(), k -> new HashMap<>())
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
index 9fcc459c39..7aa598260c 100644
--- a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexFileTest.java
@@ -21,7 +21,6 @@ package org.apache.paimon.index;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.utils.IntIterator;
-import org.apache.paimon.utils.PathFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -42,16 +41,21 @@ public class HashIndexFileTest {
@Test
public void test() throws IOException {
Path dir = new Path(tempPath.toUri());
- PathFactory pathFactory =
- new PathFactory() {
+ IndexPathFactory pathFactory =
+ new IndexPathFactory() {
@Override
public Path newPath() {
return new Path(dir, UUID.randomUUID().toString());
}
@Override
- public Path toPath(String fileName) {
- return new Path(dir, fileName);
+ public Path toPath(IndexFileMeta file) {
+ return new Path(dir, file.fileName());
+ }
+
+ @Override
+ public boolean isExternalPath() {
+ return false;
}
};
@@ -63,13 +67,13 @@ public class HashIndexFileTest {
random.add(rnd.nextInt());
}
- String name =
+ IndexFileMeta meta =
file.write(
IntIterator.create(random.stream().mapToInt(Integer::intValue).toArray()));
- List<Integer> result = IntIterator.toIntList(file.read(name));
+ List<Integer> result = IntIterator.toIntList(file.read(meta));
assertThat(result).containsExactlyInAnyOrderElementsOf(random);
- assertThat(file.fileSize(name)).isEqualTo(random.size() * 4L);
+ assertThat(file.fileSize(meta)).isEqualTo(random.size() * 4L);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
index a7e692d2e5..c5ad3ae8fc 100644
---
a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java
@@ -53,17 +53,19 @@ public class IndexFileMetaSerializerTest extends
ObjectSerializerTestBase<IndexF
HashIndexFile.HASH_INDEX,
"my_file_name" + rnd.nextLong(),
rnd.nextInt(),
- rnd.nextInt());
+ rnd.nextInt(),
+ null,
+ null);
}
public static IndexFileMeta randomDeletionVectorIndexFile() {
Random rnd = new Random();
- LinkedHashMap<String, DeletionVectorMeta> deletionVectorMetas = new
LinkedHashMap<>();
- deletionVectorMetas.put(
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put(
"my_file_name1",
new DeletionVectorMeta(
"my_file_name1", rnd.nextInt(), rnd.nextInt(),
rnd.nextLong()));
- deletionVectorMetas.put(
+ dvRanges.put(
"my_file_name2",
new DeletionVectorMeta(
"my_file_name2", rnd.nextInt(), rnd.nextInt(),
rnd.nextLong()));
@@ -72,6 +74,7 @@ public class IndexFileMetaSerializerTest extends
ObjectSerializerTestBase<IndexF
"deletion_vectors_index_file_name" + rnd.nextLong(),
rnd.nextInt(),
rnd.nextInt(),
- deletionVectorMetas);
+ dvRanges,
+ null);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java
b/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java
index 074010db8c..9c2106bf62 100644
---
a/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java
@@ -171,7 +171,8 @@ public class DataFileIndexWriterTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
Table table = fileSystemCatalog.getTable(Identifier.create(tableName,
tableName));
ReadBuilder readBuilder = table.newReadBuilder();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
index 1d7bf1a634..02f096ff24 100644
---
a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java
@@ -237,7 +237,8 @@ public class KeyValueFileReadWriteTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) +
1024;
FileIO fileIO = FileIOFinder.find(path);
Options options = new Options();
@@ -258,7 +259,8 @@ public class KeyValueFileReadWriteTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
}
};
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
index 82ff4d1be1..b458c29fe8 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java
@@ -81,11 +81,17 @@ public class ManifestCommittableSerializerCompatibilityTest
{
Arrays.asList("asdf", "qwer", "zxcv"));
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type",
+ "my_index_file",
+ 1024 * 100,
+ 1002,
+ dvRanges,
+ "external_path");
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -158,11 +164,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -234,11 +241,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -308,11 +316,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -382,11 +391,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -455,11 +465,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -529,11 +540,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -603,11 +615,12 @@ public class
ManifestCommittableSerializerCompatibilityTest {
null);
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
- LinkedHashMap<String, DeletionVectorMeta> dvMetas = new
LinkedHashMap<>();
- dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null));
- dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null));
+ LinkedHashMap<String, DeletionVectorMeta> dvRanges = new
LinkedHashMap<>();
+ dvRanges.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null));
+ dvRanges.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null));
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, dvMetas);
+ new IndexFileMeta(
+ "my_index_type", "my_index_file", 1024 * 100, 1002,
dvRanges, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
@@ -678,7 +691,7 @@ public class ManifestCommittableSerializerCompatibilityTest
{
List<DataFileMeta> dataFiles = Collections.singletonList(dataFile);
IndexFileMeta indexFile =
- new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, null);
+ new IndexFileMeta("my_index_type", "my_index_file", 1024 *
100, 1002, null, null);
List<IndexFileMeta> indexFiles = Collections.singletonList(indexFile);
CommitMessageImpl commitMessage =
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
index 5cac5e65e2..592c27917d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java
@@ -154,7 +154,8 @@ public abstract class ManifestFileMetaTestBase {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null),
+ null,
+ false),
Long.MAX_VALUE,
null)
.create();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
index 3e6b72b8b8..78045d9440 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
@@ -107,7 +107,8 @@ public class ManifestFileTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) +
1024;
FileIO fileIO = FileIOFinder.find(path);
return new ManifestFile.Factory(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
index 5008bd2021..80962ff64e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
@@ -172,7 +172,8 @@ public class ManifestListTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
}
private ManifestList createManifestList(String pathStr) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
index f8a9309a09..47a876f345 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
@@ -752,18 +752,12 @@ public class FileStoreCommitTest {
IndexManifestEntry indexManifestEntry =
part1Index.stream().filter(entry -> entry.bucket() ==
0).findAny().get();
- assertThat(
- indexFileHandler
- .hashIndex(part1, 0)
-
.readList(indexManifestEntry.indexFile().fileName()))
+ assertThat(indexFileHandler.hashIndex(part1,
0).readList(indexManifestEntry.indexFile()))
.containsExactlyInAnyOrder(1, 2, 5);
indexManifestEntry =
part1Index.stream().filter(entry -> entry.bucket() ==
1).findAny().get();
- assertThat(
- indexFileHandler
- .hashIndex(part1, 1)
-
.readList(indexManifestEntry.indexFile().fileName()))
+ assertThat(indexFileHandler.hashIndex(part1,
1).readList(indexManifestEntry.indexFile()))
.containsExactlyInAnyOrder(6, 8);
// assert part2
@@ -771,10 +765,7 @@ public class FileStoreCommitTest {
indexFileHandler.scanEntries(snapshot, HASH_INDEX, part2);
assertThat(part2Index.size()).isEqualTo(1);
assertThat(part2Index.get(0).bucket()).isEqualTo(2);
- assertThat(
- indexFileHandler
- .hashIndex(part2, 2)
-
.readList(part2Index.get(0).indexFile().fileName()))
+ assertThat(indexFileHandler.hashIndex(part2,
2).readList(part2Index.get(0).indexFile()))
.containsExactlyInAnyOrder(3, 5);
// update part1
@@ -791,24 +782,18 @@ public class FileStoreCommitTest {
indexManifestEntry =
part1Index.stream().filter(entry -> entry.bucket() ==
0).findAny().get();
- assertThat(
- indexFileHandler
- .hashIndex(part1, 0)
-
.readList(indexManifestEntry.indexFile().fileName()))
+ assertThat(indexFileHandler.hashIndex(part1,
0).readList(indexManifestEntry.indexFile()))
.containsExactlyInAnyOrder(1, 4);
indexManifestEntry =
part1Index.stream().filter(entry -> entry.bucket() ==
1).findAny().get();
- assertThat(
- indexFileHandler
- .hashIndex(part1, 1)
-
.readList(indexManifestEntry.indexFile().fileName()))
+ assertThat(indexFileHandler.hashIndex(part1,
1).readList(indexManifestEntry.indexFile()))
.containsExactlyInAnyOrder(6, 8);
// assert scan one bucket
Optional<IndexFileMeta> file =
indexFileHandler.scanHashIndex(snapshot, part1, 0);
assertThat(file).isPresent();
- assertThat(indexFileHandler.hashIndex(part1,
0).readList(file.get().fileName()))
+ assertThat(indexFileHandler.hashIndex(part1, 0).readList(file.get()))
.containsExactlyInAnyOrder(1, 4);
// overwrite one partition
diff --git
a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
index d0bd266ecf..7e7b36d7c5 100644
---
a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java
@@ -93,7 +93,8 @@ public class FileStorePathFactoryTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
assertPartition("20211224", 16, pathFactory, "/dt=20211224/hr=16");
assertPartition("20211224", null, pathFactory,
"/dt=20211224/hr=default");
@@ -136,6 +137,7 @@ public class FileStorePathFactoryTest {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
}
}
diff --git
a/paimon-core/src/test/resources/compatibility/manifest-committable-v8
b/paimon-core/src/test/resources/compatibility/manifest-committable-v8
index 865bbd2dcc..922e55c5d7 100644
Binary files
a/paimon-core/src/test/resources/compatibility/manifest-committable-v8 and
b/paimon-core/src/test/resources/compatibility/manifest-committable-v8 differ
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
index 34200cbc41..a524c3633d 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
@@ -18,16 +18,22 @@
package org.apache.paimon.flink;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.utils.BlockingIterator;
+import org.apache.paimon.utils.TraceableFileIO;
import org.apache.flink.types.Row;
import org.apache.flink.types.RowKind;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
+import java.io.IOException;
+import java.util.Arrays;
import java.util.stream.Stream;
import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
@@ -35,6 +41,8 @@ import static
org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
/** ITCase for deletion vector table. */
public class DeletionVectorITCase extends CatalogITCaseBase {
+ @TempDir java.nio.file.Path tempExternalPath;
+
private static Stream<Arguments> parameters1() {
// parameters: changelogProducer, dvBitmap64
return Stream.of(
@@ -419,4 +427,57 @@ public class DeletionVectorITCase extends
CatalogITCaseBase {
.containsExactlyInAnyOrder(Row.of(1, 1), Row.of(2, 2),
Row.of(3, 3));
}
}
+
+ @Test
+ public void testIndexFileInDataFileDir() throws IOException {
+ sql(
+ "CREATE TABLE IT (a INT PRIMARY KEY NOT ENFORCED, b INT) WITH
("
+ + "'deletion-vectors.enabled' = 'true', "
+ + "'index-file-in-data-file-dir' = 'true')");
+ sql("INSERT INTO IT VALUES (1, 1)");
+ assertThat(sql("SELECT * FROM IT")).containsExactly(Row.of(1, 1));
+ Path path = getTableDirectory("IT");
+ LocalFileIO fileIO = LocalFileIO.create();
+ String result = Arrays.asList(fileIO.listFiles(path, true)).toString();
+ assertThat(result).contains("default.db/IT/bucket-0/index-");
+ assertThat(result).doesNotContain("default.db/IT/index/index-");
+ }
+
+ @Test
+ public void testIndexFileInIndexDir() throws IOException {
+ sql(
+ "CREATE TABLE IT (a INT PRIMARY KEY NOT ENFORCED, b INT) WITH
("
+ + "'deletion-vectors.enabled' = 'true')");
+ sql("INSERT INTO IT (a, b) VALUES (1, 1)");
+ assertThat(sql("SELECT * FROM IT")).containsExactly(Row.of(1, 1));
+ Path path = getTableDirectory("IT");
+ LocalFileIO fileIO = LocalFileIO.create();
+ String result = Arrays.asList(fileIO.listFiles(path, true)).toString();
+ assertThat(result).doesNotContain("default.db/IT/bucket-0/index-");
+ assertThat(result).contains("default.db/IT/index/index-");
+ }
+
+ @Test
+ public void testIndexFileInDataFileDirWithExternalPath() throws
IOException {
+ String externalPaths = TraceableFileIO.SCHEME + "://" +
tempExternalPath.toString();
+ sql(
+ "CREATE TABLE IT (a INT PRIMARY KEY NOT ENFORCED, b INT) WITH
("
+ + "'deletion-vectors.enabled' = 'true', "
+ + "'index-file-in-data-file-dir' = 'true', "
+ + "'data-file.external-paths.strategy' =
'round-robin', "
+ + String.format("'data-file.external-paths' = '%s')",
externalPaths));
+ sql("INSERT INTO IT (a, b) VALUES (1, 1)");
+ assertThat(sql("SELECT * FROM IT")).containsExactly(Row.of(1, 1));
+ LocalFileIO fileIO = LocalFileIO.create();
+
+ Path path = getTableDirectory("IT");
+ String inTablePath = Arrays.asList(fileIO.listFiles(path,
true)).toString();
+ assertThat(inTablePath).doesNotContain("bucket-0/index-");
+ assertThat(inTablePath).doesNotContain("index/index-");
+
+ Path externalPath = new Path(externalPaths);
+ String inExternalPath = Arrays.asList(fileIO.listFiles(externalPath,
true)).toString();
+ assertThat(inExternalPath).contains("bucket-0/index-");
+ assertThat(inExternalPath).doesNotContain("index/index-");
+ }
}
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
index 9847276fe8..1aca14fa31 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java
@@ -111,7 +111,8 @@ public class TestChangelogDataReadWrite {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
this.snapshotManager = newSnapshotManager(LocalFileIO.create(), new
Path(root));
this.commitUser = UUID.randomUUID().toString();
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
index fe99b98645..b4ccd81bf7 100644
---
a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
+++
b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java
@@ -209,7 +209,8 @@ public class SparkFileIndexITCase extends SparkWriteITCase {
CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
CoreOptions.FILE_COMPRESSION.defaultValue(),
null,
- null);
+ null,
+ false);
Table table = fileSystemCatalog.getTable(Identifier.create("db",
tableName));
ReadBuilder readBuilder = table.newReadBuilder();
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala
index 445d1507d9..68e741fb13 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala
@@ -655,7 +655,7 @@ class DeletionVectorTest extends PaimonSparkTestBase with
AdaptiveSparkPlanHelpe
val fileStore = loadTable("T").store()
val indexManifest =
fileStore.snapshotManager().latestSnapshot().indexManifest()
val entry =
fileStore.newIndexFileHandler().readManifest(indexManifest).get(0)
- val dvMeta =
entry.indexFile().deletionVectorMetas().values().iterator().next()
+ val dvMeta = entry.indexFile().dvRanges().values().iterator().next()
assert(dvMeta.cardinality() == 334)
}
