This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a2c22ac5690 Fix that the initial WAL file is not counted (#17662)
a2c22ac5690 is described below
commit a2c22ac5690ca872616b1bd0f63c7773023f1ee4
Author: Jiang Tian <[email protected]>
AuthorDate: Wed May 13 20:42:18 2026 +0800
Fix that the initial WAL file is not counted (#17662)
---
.../dataregion/wal/buffer/AbstractWALBuffer.java | 2 +
.../wal/buffer/WALBufferFileNumTest.java | 202 +++++++++++++++++++++
2 files changed, 204 insertions(+)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java
index c8f68523032..c9694c46a2e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java
@@ -75,6 +75,8 @@ public abstract class AbstractWALBuffer implements IWALBuffer
{
logDirectory,
WALFileUtils.getLogFileName(
startFileVersion, currentSearchIndex,
WALFileStatus.CONTAINS_SEARCH_INDEX)));
+ // count the newly created WAL file into file number statistics
+ addFileNum(1);
currentWALFileVersion = startFileVersion;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferFileNumTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferFileNumTest.java
new file mode 100644
index 00000000000..9069473f121
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBufferFileNumTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.storageengine.dataregion.wal.buffer;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
+import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode;
+import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils;
+import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.db.utils.constant.TestConstant;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+
+/** Tests that WAL file number statistics correctly account for all WAL files.
*/
+public class WALBufferFileNumTest {
+ private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+ private static final String identifier = String.valueOf(Integer.MAX_VALUE);
+ private static final String logDirectory =
+ TestConstant.BASE_OUTPUT_PATH.concat("wal-file-num-test");
+ private static final String devicePath = "root.test_sg.test_d";
+
+ private WALMode prevMode;
+ private String prevConsensus;
+ private long prevWalFileSizeThreshold;
+ private WALNode walNode;
+
+ @Before
+ public void setUp() throws Exception {
+ EnvironmentUtils.cleanDir(logDirectory);
+ prevMode = config.getWalMode();
+ prevConsensus = config.getDataRegionConsensusProtocolClass();
+ prevWalFileSizeThreshold = config.getWalFileSizeThresholdInByte();
+ config.setWalMode(WALMode.SYNC);
+
config.setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ // use a small threshold so that writes can trigger auto-roll
+ config.setWalFileSizeThresholdInByte(2 * 1024 * 1024);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (walNode != null) {
+ walNode.close();
+ }
+ config.setWalMode(prevMode);
+ config.setDataRegionConsensusProtocolClass(prevConsensus);
+ config.setWalFileSizeThresholdInByte(prevWalFileSizeThreshold);
+ EnvironmentUtils.cleanDir(logDirectory);
+ }
+
+ /** Verify that the initial WAL file writer created in the constructor is
counted in fileNum. */
+ @Test
+ public void testInitialFileNumAfterConstruction() throws Exception {
+ walNode = new WALNode(identifier, logDirectory);
+ // after construction on a fresh directory, there should be exactly 1 WAL
file
+ // (the currentWALFileWriter created in the constructor)
+ assertEquals(1, walNode.getFileNum());
+ // verify disk agrees
+ File[] walFilesOnDisk = WALFileUtils.listAllWALFiles(new
File(logDirectory));
+ assertEquals(1, walFilesOnDisk.length);
+ }
+
+ /**
+ * Verify that fileNum stays correct after rolling the WAL file. After one
roll, there should be 2
+ * files: the original (now closed) and the new writer.
+ */
+ @Test
+ public void testFileNumAfterRoll() throws Exception {
+ walNode = new WALNode(identifier, logDirectory);
+ assertEquals(1, walNode.getFileNum());
+
+ // write some data then roll
+ walNode.log(
+ 0,
+ getInsertTabletNode(devicePath, new long[] {1}),
+ Collections.singletonList(new int[] {0, 1}));
+ walNode.rollWALFile();
+ Awaitility.await().until(() -> walNode.isAllWALEntriesConsumed());
+
+ // after one roll: 1 closed file + 1 new open file = 2
+ assertEquals(2, walNode.getFileNum());
+ File[] walFilesOnDisk = WALFileUtils.listAllWALFiles(new
File(logDirectory));
+ assertEquals(2, walFilesOnDisk.length);
+ }
+
+ /**
+ * Verify that fileNum stays correct after multiple rolls. After N rolls,
there should be N+1
+ * files on disk and fileNum should match.
+ */
+ @Test
+ public void testFileNumAfterMultipleRolls() throws Exception {
+ walNode = new WALNode(identifier, logDirectory);
+ assertEquals(1, walNode.getFileNum());
+
+ int rollCount = 3;
+ for (int i = 0; i < rollCount; i++) {
+ walNode.log(
+ 0,
+ getInsertTabletNode(devicePath, new long[] {i + 1}),
+ Collections.singletonList(new int[] {0, 1}));
+ walNode.rollWALFile();
+ Awaitility.await().until(() -> walNode.isAllWALEntriesConsumed());
+ }
+
+ // rollCount closed files + 1 current open file
+ long expectedFileNum = rollCount + 1;
+ assertEquals(expectedFileNum, walNode.getFileNum());
+ File[] walFilesOnDisk = WALFileUtils.listAllWALFiles(new
File(logDirectory));
+ assertEquals(expectedFileNum, walFilesOnDisk.length);
+ }
+
+ /**
+ * Verify that WALManager's totalFileNum is consistent with the per-node
fileNum for a single WAL
+ * node.
+ */
+ @Test
+ public void testTotalFileNumInWALManager() throws Exception {
+ long totalFileNumBefore = WALManager.getInstance().getTotalFileNum();
+ walNode = new WALNode(identifier, logDirectory);
+
+ // after construction, totalFileNum should increase by 1
+ assertEquals(totalFileNumBefore + 1,
WALManager.getInstance().getTotalFileNum());
+
+ // roll once
+ walNode.log(
+ 0,
+ getInsertTabletNode(devicePath, new long[] {1}),
+ Collections.singletonList(new int[] {0, 1}));
+ walNode.rollWALFile();
+ Awaitility.await().until(() -> walNode.isAllWALEntriesConsumed());
+
+ // after one roll, totalFileNum should increase by 1 more
+ assertEquals(totalFileNumBefore + 2,
WALManager.getInstance().getTotalFileNum());
+ }
+
+ private InsertTabletNode getInsertTabletNode(String devicePath, long[] times)
+ throws IllegalPathException {
+ TSDataType[] dataTypes = new TSDataType[] {TSDataType.TEXT};
+ String[] measurements = new String[] {"s1"};
+ MeasurementSchema[] schemas =
+ new MeasurementSchema[] {new MeasurementSchema("s1", TSDataType.TEXT)};
+
+ Object[] columns = new Object[1];
+ Binary[] binaryValues = new Binary[times.length];
+ for (int i = 0; i < times.length; i++) {
+ binaryValues[i] = new Binary("test" + times[i],
TSFileConfig.STRING_CHARSET);
+ }
+ columns[0] = binaryValues;
+
+ BitMap[] bitMaps = new BitMap[1];
+ bitMaps[0] = new BitMap(times.length);
+
+ InsertTabletNode node =
+ new InsertTabletNode(
+ new PlanNodeId(""),
+ new PartialPath(devicePath),
+ false,
+ measurements,
+ dataTypes,
+ times,
+ bitMaps,
+ columns,
+ times.length);
+ node.setMeasurementSchemas(schemas);
+ return node;
+ }
+}
