This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new de3adcd0363 Fix uncaught IllegalArgumentException when reading broken
WAL files (#12912)
de3adcd0363 is described below
commit de3adcd03633f9e049e4a3e3ed4710fcda4ec83e
Author: Jiang Tian <[email protected]>
AuthorDate: Tue Jul 16 09:35:36 2024 +0800
Fix uncaught IllegalArgumentException when reading broken WAL files (#12912)
* Fix uncaught IllegalArgumentException when reading broken WAL files
* spotless
* also catch other exception
---
.../dataregion/wal/io/WALMetaData.java | 65 +++++++++++++---------
.../dataregion/wal/io/WALFileTest.java | 15 +++++
2 files changed, 53 insertions(+), 27 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
index ecd44268f59..b5c71ba0e5c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java
@@ -131,38 +131,49 @@ public class WALMetaData implements SerializedSize {
public static WALMetaData readFromWALFile(File logFile, FileChannel channel)
throws IOException {
if (channel.size() < WALWriter.MAGIC_STRING_V2_BYTES ||
!isValidMagicString(channel)) {
- throw new IOException(String.format("Broken wal file %s", logFile));
+ throw new IOException(
+ String.format("Broken wal file %s, size %d", logFile,
logFile.length()));
}
// load metadata size
- ByteBuffer metadataSizeBuf = ByteBuffer.allocate(Integer.BYTES);
+ WALMetaData metaData = null;
long position;
- WALFileVersion version = WALFileVersion.getVersion(channel);
- position =
- channel.size()
- - Integer.BYTES
- - (version == WALFileVersion.V2
- ? WALWriter.MAGIC_STRING_V2_BYTES
- : WALWriter.MAGIC_STRING_V1_BYTES);
- channel.read(metadataSizeBuf, position);
- metadataSizeBuf.flip();
- // load metadata
- int metadataSize = metadataSizeBuf.getInt();
- ByteBuffer metadataBuf = ByteBuffer.allocate(metadataSize);
- channel.read(metadataBuf, position - metadataSize);
- metadataBuf.flip();
- WALMetaData metaData = WALMetaData.deserialize(metadataBuf);
- // versions before V1.3, should recover memTable ids from entries
- if (metaData.memTablesId.isEmpty()) {
- int offset = Byte.BYTES;
- for (int size : metaData.buffersSize) {
- channel.position(offset);
- ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
- channel.read(buffer);
- buffer.clear();
- metaData.memTablesId.add(buffer.getLong());
- offset += size;
+ try {
+ ByteBuffer metadataSizeBuf = ByteBuffer.allocate(Integer.BYTES);
+ WALFileVersion version = WALFileVersion.getVersion(channel);
+ position =
+ channel.size()
+ - Integer.BYTES
+ - (version == WALFileVersion.V2
+ ? WALWriter.MAGIC_STRING_V2_BYTES
+ : WALWriter.MAGIC_STRING_V1_BYTES);
+ channel.read(metadataSizeBuf, position);
+ metadataSizeBuf.flip();
+ // load metadata
+ int metadataSize = metadataSizeBuf.getInt();
+ ByteBuffer metadataBuf = ByteBuffer.allocate(metadataSize);
+ channel.read(metadataBuf, position - metadataSize);
+ metadataBuf.flip();
+ metaData = WALMetaData.deserialize(metadataBuf);
+ // versions before V1.3, should recover memTable ids from entries
+ if (metaData.memTablesId.isEmpty()) {
+ int offset = Byte.BYTES;
+ for (int size : metaData.buffersSize) {
+ channel.position(offset);
+ ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
+ channel.read(buffer);
+ buffer.clear();
+ metaData.memTablesId.add(buffer.getLong());
+ offset += size;
+ }
}
+ } catch (IllegalArgumentException e) {
+ throw new IOException(
+ String.format("Broken wal file %s, size %d", logFile,
logFile.length()));
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException("Unexpected exception", e);
}
return metaData;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
index 603231ccbdd..d1c0a86ce39 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java
@@ -45,12 +45,15 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
public class WALFileTest {
@@ -154,6 +157,18 @@ public class WALFileTest {
assertEquals(expectedWALEntries, actualWALEntries);
}
+ @Test
+ public void testReadMetadataFromBrokenFile() throws IOException {
+ ILogWriter walWriter = new WALWriter(walFile);
+ assertThrows(
+ IOException.class,
+ () -> WALMetaData.readFromWALFile(walFile,
FileChannel.open(walFile.toPath())));
+ walWriter.close();
+ WALMetaData walMetaData =
+ WALMetaData.readFromWALFile(walFile,
FileChannel.open(walFile.toPath()));
+ assertTrue(walMetaData.getMemTablesId().isEmpty());
+ }
+
public static InsertRowNode getInsertRowNode(String devicePath) throws
IllegalPathException {
long time = 110L;
TSDataType[] dataTypes =
