This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/table_disk_usage_statistics_with_cache by this push:
new 415e426e99f fix bug
415e426e99f is described below
commit 415e426e99fc168fd70306723aa8b15108d99cb2
Author: shuwenwei <[email protected]>
AuthorDate: Tue Jan 27 14:55:04 2026 +0800
fix bug
---
.../AbstractTableSizeCacheWriter.java | 2 +-
.../tsfile/TsFileTableDiskUsageCacheWriter.java | 38 +++--
.../utils/TsFileTableSizeCacheWriterTest.java | 190 ++++++++++++++++++++-
3 files changed, 214 insertions(+), 16 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
index 84a017f474d..4f080ca5073 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/AbstractTableSizeCacheWriter.java
@@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit;
public abstract class AbstractTableSizeCacheWriter {
protected static final Logger logger =
LoggerFactory.getLogger(AbstractTableSizeCacheWriter.class);
- protected static final String TEMP_CACHE_FILE_SUBFIX = ".tmp";
+ public static final String TEMP_CACHE_FILE_SUBFIX = ".tmp";
protected final int regionId;
protected long previousCompactionTimestamp = System.currentTimeMillis();
protected long lastWriteTimestamp = System.currentTimeMillis();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
index 772b93732ef..2ecd3d39662 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/tsfile/TsFileTableDiskUsageCacheWriter.java
@@ -20,6 +20,7 @@
package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile;
import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
@@ -43,8 +44,8 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
public class TsFileTableDiskUsageCacheWriter extends
AbstractTableSizeCacheWriter {
- private static final String TSFILE_CACHE_KEY_FILENAME_PREFIX =
"TableSizeKeyFile_";
- private static final String TSFILE_CACHE_VALUE_FILENAME_PREFIX =
"TableSizeValueFile_";
+ public static final String TSFILE_CACHE_KEY_FILENAME_PREFIX =
"TableSizeKeyFile_";
+ public static final String TSFILE_CACHE_VALUE_FILENAME_PREFIX =
"TableSizeValueFile_";
public static final int KEY_FILE_OFFSET_RECORD_LENGTH = 5 * Long.BYTES + 1;
public static final int KEY_FILE_REDIRECT_RECORD_LENGTH = 7 * Long.BYTES + 1;
public static final byte KEY_FILE_RECORD_TYPE_OFFSET = 1;
@@ -76,32 +77,33 @@ public class TsFileTableDiskUsageCacheWriter extends
AbstractTableSizeCacheWrite
}
continue;
}
- if (isTempFile) {
- try {
- Files.delete(file.toPath());
- } catch (IOException ignored) {
- }
- }
int version;
try {
- version =
Integer.parseInt(fileName.substring(TSFILE_CACHE_KEY_FILENAME_PREFIX.length()));
+ version = getVersion(fileName);
} catch (NumberFormatException ignored) {
continue;
}
+ if (isTempFile) {
+ FileUtils.deleteFileIfExist(file);
+ FileUtils.deleteFileIfExist(generateValueFile(version, true));
+ continue;
+ }
File valueFile =
new File(dir + File.separator + TSFILE_CACHE_VALUE_FILENAME_PREFIX
+ version);
// may have a valid value index file
if (!valueFile.exists()) {
File tempValueFile = new File(valueFile.getPath() +
TEMP_CACHE_FILE_SUBFIX);
if (tempValueFile.exists()) {
- tempValueFile.renameTo(valueFile);
+ try {
+ Files.move(tempValueFile.toPath(), valueFile.toPath());
+ } catch (IOException e) {
+ logger.warn("Failed to move {} to {}", tempValueFile, valueFile,
e);
+ continue;
+ }
valueFiles.add(valueFile);
} else {
// lost value file
- try {
- Files.delete(file.toPath());
- } catch (IOException ignored) {
- }
+ FileUtils.deleteFileIfExist(file);
continue;
}
}
@@ -127,6 +129,14 @@ public class TsFileTableDiskUsageCacheWriter extends
AbstractTableSizeCacheWrite
}
}
+ private int getVersion(String fileName) throws NumberFormatException {
+ int version = 0;
+ String removePrefixStr =
fileName.substring(TSFILE_CACHE_KEY_FILENAME_PREFIX.length());
+ int suffixIdx = removePrefixStr.indexOf('.');
+ return Integer.parseInt(
+ suffixIdx > 0 ? removePrefixStr.substring(0, suffixIdx) :
removePrefixStr);
+ }
+
public void write(TsFileID tsFileID, Map<String, Long> tableSizeMap) throws
IOException {
tsFileTableSizeIndexFileWriter.write(tsFileID, tableSizeMap);
markWritten();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
index 7b860c570e0..2a7a7eefd33 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
@@ -28,6 +28,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactio
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.AbstractTableSizeCacheWriter;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.DataRegionTableSizeQueryContext;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.TimePartitionTableSizeQueryContext;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.tsfile.TsFileTableDiskUsageCacheWriter;
@@ -35,6 +36,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache.ts
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -43,9 +46,11 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -240,7 +245,7 @@ public class TsFileTableSizeCacheWriterTest extends
AbstractCompactionTest {
}
@Test
- public void testRecoverWriter() throws IOException {
+ public void testRecoverWriter1() throws IOException {
TsFileTableDiskUsageCacheWriter writer =
new TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(),
0);
for (int i = 1; i <= 10; i++) {
@@ -264,6 +269,189 @@ public class TsFileTableSizeCacheWriterTest extends
AbstractCompactionTest {
writer.close();
Assert.assertEquals(keyFileValidLength, keyFile.length());
Assert.assertEquals(valueFileValidLength, valueFile.length());
+
+ Files.write(keyFile.toPath(), new byte[] {0, 0, 0, 0},
StandardOpenOption.APPEND);
+ writer = new
TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(), 0);
+ writer.close();
+ Assert.assertEquals(keyFileValidLength, keyFile.length());
+ Assert.assertEquals(valueFileValidLength, valueFile.length());
+
+ ByteBuffer buffer =
+
ByteBuffer.allocate(TsFileTableDiskUsageCacheWriter.KEY_FILE_OFFSET_RECORD_LENGTH);
+ TsFileID tsFileID = new TsFileID(0, 1, 2, 3, 4);
+
ReadWriteIOUtils.write(TsFileTableDiskUsageCacheWriter.KEY_FILE_RECORD_TYPE_OFFSET,
buffer);
+ ReadWriteIOUtils.write(tsFileID.timePartitionId, buffer);
+ ReadWriteIOUtils.write(tsFileID.timestamp, buffer);
+ ReadWriteIOUtils.write(tsFileID.fileVersion, buffer);
+ ReadWriteIOUtils.write(tsFileID.compactionVersion, buffer);
+ ReadWriteIOUtils.write(valueFileValidLength, buffer);
+ Files.write(
+ keyFile.toPath(),
+ Arrays.copyOf(buffer.array(), buffer.position()),
+ StandardOpenOption.APPEND);
+ writer = new
TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(), 0);
+ writer.close();
+ Assert.assertEquals(keyFileValidLength, keyFile.length());
+ Assert.assertEquals(valueFileValidLength, valueFile.length());
+
+ buffer.clear();
+ ReadWriteForEncodingUtils.writeVarInt(1, buffer);
+ ReadWriteIOUtils.writeVar("table1", buffer);
+ ReadWriteIOUtils.write(10L, buffer);
+ Files.write(
+ valueFile.toPath(),
+ Arrays.copyOf(buffer.array(), buffer.position()),
+ StandardOpenOption.APPEND);
+ writer = new
TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(), 0);
+ writer.close();
+ Assert.assertEquals(keyFileValidLength, keyFile.length());
+ Assert.assertEquals(valueFileValidLength, valueFile.length());
+ }
+
+ @Test
+ public void testRecoverWriter2() throws IOException {
+ File dir =
+ StorageEngine.getDataRegionSystemDir(
+ mockDataRegion.getDatabaseName(),
mockDataRegion.getDataRegionIdString());
+ dir.mkdirs();
+ File keyFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "0");
+ File valueFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "0");
+ File tempKeyFile2 =
+ new File(
+ dir,
+ TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX
+ + "1"
+ + AbstractTableSizeCacheWriter.TEMP_CACHE_FILE_SUBFIX);
+ File tempValueFile2 =
+ new File(
+ dir,
+ TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX
+ + "1"
+ + AbstractTableSizeCacheWriter.TEMP_CACHE_FILE_SUBFIX);
+
+ keyFile1.createNewFile();
+ valueFile1.createNewFile();
+ tempKeyFile2.createNewFile();
+ tempValueFile2.createNewFile();
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(),
0);
+ writer.close();
+ Assert.assertTrue(keyFile1.exists());
+ Assert.assertTrue(valueFile1.exists());
+ Assert.assertFalse(tempKeyFile2.exists());
+ Assert.assertFalse(tempValueFile2.exists());
+ Assert.assertEquals(keyFile1, writer.getKeyFile());
+ Assert.assertEquals(valueFile1, writer.getValueFile());
+ }
+
+ @Test
+ public void testRecoverWriter3() throws IOException {
+ File dir =
+ StorageEngine.getDataRegionSystemDir(
+ mockDataRegion.getDatabaseName(),
mockDataRegion.getDataRegionIdString());
+ dir.mkdirs();
+ File keyFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "0");
+ File valueFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "0");
+ File keyFile2 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "1");
+ File valueFile2 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "1");
+
+ keyFile1.createNewFile();
+ valueFile1.createNewFile();
+ keyFile2.createNewFile();
+ valueFile2.createNewFile();
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(),
0);
+ writer.close();
+ Assert.assertFalse(keyFile1.exists());
+ Assert.assertFalse(valueFile1.exists());
+ Assert.assertTrue(keyFile2.exists());
+ Assert.assertTrue(valueFile2.exists());
+ Assert.assertEquals(keyFile2, writer.getKeyFile());
+ Assert.assertEquals(valueFile2, writer.getValueFile());
+ }
+
+ @Test
+ public void testRecoverWriter4() throws IOException {
+ File dir =
+ StorageEngine.getDataRegionSystemDir(
+ mockDataRegion.getDatabaseName(),
mockDataRegion.getDataRegionIdString());
+ dir.mkdirs();
+ File keyFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "0");
+ File valueFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "0");
+ File keyFile2 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "1");
+ File tempValueFile2 =
+ new File(
+ dir,
+ TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX
+ + "1"
+ + AbstractTableSizeCacheWriter.TEMP_CACHE_FILE_SUBFIX);
+ File valueFile2 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "1");
+
+ keyFile1.createNewFile();
+ valueFile1.createNewFile();
+ keyFile2.createNewFile();
+ tempValueFile2.createNewFile();
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(),
0);
+ writer.close();
+ Assert.assertFalse(keyFile1.exists());
+ Assert.assertFalse(valueFile1.exists());
+ Assert.assertFalse(tempValueFile2.exists());
+ Assert.assertTrue(keyFile2.exists());
+ Assert.assertTrue(valueFile2.exists());
+ Assert.assertEquals(keyFile2, writer.getKeyFile());
+ Assert.assertEquals(valueFile2, writer.getValueFile());
+ }
+
+ @Test
+ public void testRecoverWriter5() throws IOException {
+ File dir =
+ StorageEngine.getDataRegionSystemDir(
+ mockDataRegion.getDatabaseName(),
mockDataRegion.getDataRegionIdString());
+ dir.mkdirs();
+ File keyFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "1");
+ File valueFile1 =
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "1");
+ File tempKeyFile2 =
+ new File(
+ dir,
+ TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX
+ + "2"
+ + TsFileTableDiskUsageCacheWriter.TEMP_CACHE_FILE_SUBFIX);
+ File tempValueFile2 =
+ new File(
+ dir,
+ TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX
+ + "2"
+ + AbstractTableSizeCacheWriter.TEMP_CACHE_FILE_SUBFIX);
+
+ keyFile1.createNewFile();
+ tempKeyFile2.createNewFile();
+ tempValueFile2.createNewFile();
+ TsFileTableDiskUsageCacheWriter writer =
+ new TsFileTableDiskUsageCacheWriter(mockDataRegion.getDatabaseName(),
0);
+ writer.close();
+ Assert.assertFalse(keyFile1.exists());
+ Assert.assertFalse(valueFile1.exists());
+ Assert.assertFalse(tempValueFile2.exists());
+ Assert.assertFalse(tempKeyFile2.exists());
+ Assert.assertEquals(
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_KEY_FILENAME_PREFIX + "0"),
+ writer.getKeyFile());
+ Assert.assertEquals(
+ new File(dir,
TsFileTableDiskUsageCacheWriter.TSFILE_CACHE_VALUE_FILENAME_PREFIX + "0"),
+ writer.getValueFile());
}
private Map<String, Long> generateTableSizeMap(int tableNum) {
