This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch table_disk_usage_statistics_with_cache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to
refs/heads/table_disk_usage_statistics_with_cache by this push:
new dfd2078a73c fix bug
dfd2078a73c is described below
commit dfd2078a73c64edcfacab0104e1cff7729b9a65a
Author: shuwenwei <[email protected]>
AuthorDate: Fri Jan 30 18:42:56 2026 +0800
fix bug
---
.../utils/TableDiskUsageStatisticUtil.java | 12 ++-
.../DataRegionTableSizeQueryContext.java | 11 ++
.../TableDiskUsageCacheReader.java | 4 +
.../dataregion/utils/TableDiskUsageTest.java | 118 +++++++++++----------
.../utils/TsFileTableSizeCacheWriterTest.java | 3 +
5 files changed, 88 insertions(+), 60 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
index a15d0d8ea27..d19bcf5752f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
@@ -49,11 +49,13 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
private final List<Pair<TsFileID, Long>> tsFilesToQueryInCache;
private final TimePartitionTableSizeQueryContext tableSizeQueryContext;
private final boolean databaseHasOnlyOneTable;
+ private final boolean needAllData;
public TableDiskUsageStatisticUtil(
DataRegion dataRegion,
long timePartition,
TimePartitionTableSizeQueryContext tableSizeQueryContext,
+ boolean needAllData,
boolean databaseHasOnlyOneTable,
List<Pair<TsFileID, Long>> tsFilesToQueryInCache,
Optional<FragmentInstanceContext> context) {
@@ -62,6 +64,7 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
this.tableSizeQueryContext = tableSizeQueryContext;
this.tsFilesToQueryInCache = tsFilesToQueryInCache;
this.databaseHasOnlyOneTable = databaseHasOnlyOneTable;
+ this.needAllData = needAllData;
}
@Override
@@ -77,7 +80,7 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
return false;
}
String table =
tableSizeQueryContext.getTableSizeResultMap().keySet().iterator().next();
- tableSizeQueryContext.updateResult(table, tsFileResource.getTsFileSize(),
false);
+ tableSizeQueryContext.updateResult(table, tsFileResource.getTsFileSize(),
needAllData);
TableDiskUsageCache.getInstance()
.write(
database,
@@ -90,14 +93,15 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
protected void calculateNextFile(TsFileResource tsFileResource,
TsFileSequenceReader reader)
throws IOException {
TsFileMetadata tsFileMetadata = reader.readFileMetadata();
- if (!hasSatisfiedData(tsFileMetadata)) {
+ if (!needAllData && !hasSatisfiedData(tsFileMetadata)) {
return;
}
if (tsFileMetadata.getTableMetadataIndexNodeMap().size() == 1) {
String satisfiedTable =
tsFileMetadata.getTableMetadataIndexNodeMap().keySet().iterator().next();
- tableSizeQueryContext.updateResult(satisfiedTable,
tsFileResource.getTsFileSize(), false);
+ tableSizeQueryContext.updateResult(
+ satisfiedTable, tsFileResource.getTsFileSize(), needAllData);
TableDiskUsageCache.getInstance()
.write(
database,
@@ -132,7 +136,7 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
nextTable = iterator.hasNext() ? iterator.next() : null;
long tableSize =
calculateTableSize(tableOffsetMap, tsFileMetadata, reader,
currentTable, nextTable);
- tableSizeQueryContext.updateResult(currentTable, tableSize, false);
+ tableSizeQueryContext.updateResult(currentTable, tableSize, needAllData);
tsFileTableSizeMap.put(currentTable, tableSize);
currentTable = nextTable;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
index 4a53faa797d..960d6cec419 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/DataRegionTableSizeQueryContext.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageCache;
import
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import
org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.tsfile.utils.RamUsageEstimator;
@@ -58,6 +59,10 @@ public class DataRegionTableSizeQueryContext {
return timePartitionTableSizeQueryContextMap;
}
+ public boolean isNeedAllData() {
+ return needAllData;
+ }
+
public boolean isEmpty() {
return timePartitionTableSizeQueryContextMap.isEmpty();
}
@@ -107,6 +112,12 @@ public class DataRegionTableSizeQueryContext {
return true;
}
+ public void addAllTimePartitionsInTsFileManager(TsFileManager tsFileManager)
{
+ for (Long timePartition : tsFileManager.getTimePartitions()) {
+ addTimePartition(timePartition, new
TimePartitionTableSizeQueryContext(new HashMap<>()));
+ }
+ }
+
public void addTimePartition(
long timePartition, TimePartitionTableSizeQueryContext
timePartitionTableSizeQueryContext) {
timePartitionTableSizeQueryContextMap.put(timePartition,
timePartitionTableSizeQueryContext);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
index 6d26856824f..d4f9a88bba8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageCache/TableDiskUsageCacheReader.java
@@ -65,6 +65,9 @@ public class TableDiskUsageCacheReader implements Closeable {
this.dataRegion = dataRegion;
this.dataRegionContext = dataRegionContext;
this.currentDatabaseOnlyHasOneTable = databaseHasOnlyOneTable;
+ if (dataRegionContext.isNeedAllData()) {
+
dataRegionContext.addAllTimePartitionsInTsFileManager(dataRegion.getTsFileManager());
+ }
this.timePartitionIterator =
dataRegionContext.getTimePartitionTableSizeQueryContextMap().entrySet().iterator();
dataRegionContext.reserveMemoryForResultMap();
@@ -146,6 +149,7 @@ public class TableDiskUsageCacheReader implements Closeable
{
dataRegion,
timePartition,
currentTimePartitionEntry.getValue(),
+ dataRegionContext.isNeedAllData(),
currentDatabaseOnlyHasOneTable,
tsFilesToQueryInCache,
dataRegionContext.getFragmentInstanceContext());
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageTest.java
index 2b1981cf2c3..ce390308551 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageTest.java
@@ -37,6 +37,7 @@ import
org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.utils.Pair;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -86,21 +87,7 @@ public class TableDiskUsageTest extends
AbstractCompactionTest {
timePartitionTableSizeMap.put("table1", 0L);
timePartitionTableSizeMap.put("table2", 0L);
context.addTimePartition(0, new
TimePartitionTableSizeQueryContext(timePartitionTableSizeMap));
- TableDiskUsageCacheReader reader =
- new TableDiskUsageCacheReader(mockDataRegion, context, false);
- try {
- Assert.assertTrue(reader.prepareCacheReader(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.loadObjectFileTableSizeCache(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.prepareCachedTsFileIDKeys(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.checkAllFilesInTsFileManager(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
-
reader.readCacheValueFilesAndUpdateResultMap(System.currentTimeMillis(),
Long.MAX_VALUE));
- } finally {
- reader.close();
- }
+ queryTableSize(context);
int entryNum = 0;
for (Map.Entry<Long, TimePartitionTableSizeQueryContext>
timePartitionEntry :
context.getTimePartitionTableSizeQueryContextMap().entrySet()) {
@@ -140,19 +127,7 @@ public class TableDiskUsageTest extends
AbstractCompactionTest {
context.addTimePartition(0, new
TimePartitionTableSizeQueryContext(timePartitionTableSizeMap));
TableDiskUsageCacheReader reader =
new TableDiskUsageCacheReader(mockDataRegion, context, false);
- try {
- Assert.assertTrue(reader.prepareCacheReader(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.loadObjectFileTableSizeCache(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.prepareCachedTsFileIDKeys(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.checkAllFilesInTsFileManager(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
-
reader.readCacheValueFilesAndUpdateResultMap(System.currentTimeMillis(),
Long.MAX_VALUE));
- } finally {
- reader.close();
- }
+ queryTableSize(context);
int entryNum = 0;
for (Map.Entry<Long, TimePartitionTableSizeQueryContext>
timePartitionEntry :
context.getTimePartitionTableSizeQueryContextMap().entrySet()) {
@@ -191,19 +166,7 @@ public class TableDiskUsageTest extends
AbstractCompactionTest {
context.addTimePartition(0, new
TimePartitionTableSizeQueryContext(timePartitionTableSizeMap));
TableDiskUsageCacheReader reader =
new TableDiskUsageCacheReader(mockDataRegion, context, false);
- try {
- Assert.assertTrue(reader.prepareCacheReader(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.loadObjectFileTableSizeCache(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.prepareCachedTsFileIDKeys(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.checkAllFilesInTsFileManager(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
-
reader.readCacheValueFilesAndUpdateResultMap(System.currentTimeMillis(),
Long.MAX_VALUE));
- } finally {
- reader.close();
- }
+ queryTableSize(context);
int entryNum = 0;
for (Map.Entry<Long, TimePartitionTableSizeQueryContext>
timePartitionEntry :
context.getTimePartitionTableSizeQueryContextMap().entrySet()) {
@@ -244,19 +207,7 @@ public class TableDiskUsageTest extends
AbstractCompactionTest {
context.addTimePartition(0, new
TimePartitionTableSizeQueryContext(timePartitionTableSizeMap));
TableDiskUsageCacheReader reader =
new TableDiskUsageCacheReader(mockDataRegion, context, false);
- try {
- Assert.assertTrue(reader.prepareCacheReader(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.loadObjectFileTableSizeCache(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.prepareCachedTsFileIDKeys(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
- reader.checkAllFilesInTsFileManager(System.currentTimeMillis(),
Long.MAX_VALUE));
- Assert.assertTrue(
-
reader.readCacheValueFilesAndUpdateResultMap(System.currentTimeMillis(),
Long.MAX_VALUE));
- } finally {
- reader.close();
- }
+ queryTableSize(context);
int entryNum = 0;
for (Map.Entry<Long, TimePartitionTableSizeQueryContext>
timePartitionEntry :
context.getTimePartitionTableSizeQueryContextMap().entrySet()) {
@@ -274,8 +225,60 @@ public class TableDiskUsageTest extends
AbstractCompactionTest {
Assert.assertEquals(2, entryNum);
}
+ @Test
+ public void testCalculateTableSizeFromFile() throws Exception {
+ Pair<TsFileResource, Map<String, Long>> resourceTableSizeMapPair =
+ prepareFileAndTableSizeMap(10, 100000);
+ TsFileResource resource1 = resourceTableSizeMapPair.getLeft();
+ Map<String, Long> tableSizeMapGeneratedByWrite =
resourceTableSizeMapPair.getRight();
+ Assert.assertEquals(10, tableSizeMapGeneratedByWrite.size());
+ for (Long value : tableSizeMapGeneratedByWrite.values()) {
+ Assert.assertTrue(value > 0);
+ }
+ mockTsFileManager.add(resource1, true);
+
+ DataRegionTableSizeQueryContext context = new
DataRegionTableSizeQueryContext(true);
+ queryTableSize(context);
+ Assert.assertEquals(1,
context.getTimePartitionTableSizeQueryContextMap().size());
+ TimePartitionTableSizeQueryContext timePartitionContext =
+
context.getTimePartitionTableSizeQueryContextMap().values().iterator().next();
+ Map<String, Long> tableSizeMapGeneratedByQuery =
timePartitionContext.getTableSizeResultMap();
+ Assert.assertEquals(10, tableSizeMapGeneratedByQuery.size());
+
+ for (Map.Entry<String, Long> entry :
tableSizeMapGeneratedByQuery.entrySet()) {
+ String tableName = entry.getKey();
+ Long size = tableSizeMapGeneratedByWrite.get(tableName);
+ Assert.assertNotNull(size);
+ Assert.assertTrue(Math.abs(size - entry.getValue()) < 1000);
+ }
+ }
+
+ private void queryTableSize(DataRegionTableSizeQueryContext queryContext)
throws Exception {
+ TableDiskUsageCacheReader reader =
+ new TableDiskUsageCacheReader(mockDataRegion, queryContext, false);
+ try {
+ Assert.assertTrue(reader.prepareCacheReader(System.currentTimeMillis(),
Long.MAX_VALUE));
+ Assert.assertTrue(
+ reader.loadObjectFileTableSizeCache(System.currentTimeMillis(),
Long.MAX_VALUE));
+ Assert.assertTrue(
+ reader.prepareCachedTsFileIDKeys(System.currentTimeMillis(),
Long.MAX_VALUE));
+ Assert.assertTrue(
+ reader.checkAllFilesInTsFileManager(System.currentTimeMillis(),
Long.MAX_VALUE));
+ Assert.assertTrue(
+
reader.readCacheValueFilesAndUpdateResultMap(System.currentTimeMillis(),
Long.MAX_VALUE));
+ } finally {
+ reader.close();
+ }
+ }
+
private TsFileResource prepareFile(int tableNum) throws IOException {
+ return prepareFileAndTableSizeMap(tableNum, 100).getLeft();
+ }
+
+ private Pair<TsFileResource, Map<String, Long>> prepareFileAndTableSizeMap(
+ int tableNum, int pointNum) throws IOException {
TsFileResource resource1 = createEmptyFileAndResource(true);
+ Map<String, Long> tableSizeMap = null;
try (CompactionTableModelTestFileWriter writer =
new CompactionTableModelTestFileWriter(resource1)) {
for (int i = 0; i < tableNum; i++) {
@@ -283,14 +286,17 @@ public class TableDiskUsageTest extends
AbstractCompactionTest {
writer.startChunkGroup("table" + i, Collections.singletonList("d1"));
writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue(
Arrays.asList("s0", "s1"),
- new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new
TimeRange(10, 12)}}},
+ new TimeRange[][][] {
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 10 +
pointNum - 1)}}
+ },
TSEncoding.PLAIN,
CompressionType.LZ4,
Arrays.asList(false, false));
writer.endChunkGroup();
}
writer.endFile();
+ tableSizeMap = writer.getFileWriter().getTableSizeMap();
}
- return resource1;
+ return new Pair<>(resource1, tableSizeMap);
}
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
index 2a7a7eefd33..4b8dba43e44 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileTableSizeCacheWriterTest.java
@@ -98,8 +98,11 @@ public class TsFileTableSizeCacheWriterTest extends
AbstractCompactionTest {
writer.compact();
+ Assert.assertFalse(oldKeyFile.exists());
Assert.assertEquals("TableSizeKeyFile_1", writer.getKeyFile().getName());
Assert.assertEquals("TableSizeValueFile_1",
writer.getValueFile().getName());
+ Assert.assertEquals(0, writer.keyFileLength());
+ Assert.assertEquals(0, writer.valueFileLength());
writer.close();
}
