This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/2.0.0 in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 769c1e6ea24ddc9f19d0ada1a98c81db5201bbc4 Author: Caideyipi <[email protected]> AuthorDate: Thu Jan 9 10:24:26 2025 +0800 Fixed the bug that ReadWriteIOUtils does not guarantee to use UTF-8 in deserialization --- NOTICE | 2 +- .../org/apache/tsfile/utils/ReadWriteIOUtils.java | 10 +++---- .../apache/tsfile/utils/ReadWriteIOUtilsTest.java | 35 ++++++++++++++++++++-- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/NOTICE b/NOTICE index 6e2085f6..c50186ee 100644 --- a/NOTICE +++ b/NOTICE @@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/). ============================================================================ -TsFile project uses 4 Chinese Patents: +TsFile project uses 2 Chinese Patents: * 201711384490X * 201711319331.1 diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java index 50aed117..88f1388d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java @@ -617,7 +617,7 @@ public class ReadWriteIOUtils { if (readLen != strLength) { throw new IOException(String.format(RETURN_ERROR, strLength, readLen)); } - return new String(bytes, 0, strLength); + return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET); } /** String length's type is varInt */ @@ -633,7 +633,7 @@ public class ReadWriteIOUtils { if (readLen != strLength) { throw new IOException(String.format(RETURN_ERROR, strLength, readLen)); } - return new String(bytes, 0, strLength); + return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET); } /** Read string from byteBuffer. */ @@ -646,7 +646,7 @@ public class ReadWriteIOUtils { } byte[] bytes = new byte[strLength]; buffer.get(bytes, 0, strLength); - return new String(bytes, 0, strLength); + return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET); } /** String length's type is varInt */ @@ -659,7 +659,7 @@ public class ReadWriteIOUtils { } byte[] bytes = new byte[strLength]; buffer.get(bytes, 0, strLength); - return new String(bytes, 0, strLength); + return new String(bytes, 0, strLength, TSFileConfig.STRING_CHARSET); } /** Read string from byteBuffer with user define length. */ @@ -671,7 +671,7 @@ public class ReadWriteIOUtils { } byte[] bytes = new byte[length]; buffer.get(bytes, 0, length); - return new String(bytes, 0, length); + return new String(bytes, 0, length, TSFileConfig.STRING_CHARSET); } public static ByteBuffer getByteBuffer(String s) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java index 17400fe1..a0cb9a0a 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/ReadWriteIOUtilsTest.java @@ -54,7 +54,21 @@ public class ReadWriteIOUtilsTest { Assert.assertNotNull(result); Assert.assertEquals(str, result); - // 2. null value + // 2. Chinese value + str = "中文"; + byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE); + stream = new DataOutputStream(byteArrayOutputStream); + try { + ReadWriteIOUtils.write(str, stream); + } catch (IOException e) { + fail(e.toString()); + } + + result = ReadWriteIOUtils.readString(ByteBuffer.wrap(byteArrayOutputStream.toByteArray())); + Assert.assertNotNull(result); + Assert.assertEquals(str, result); + + // 3. null value str = null; byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE); stream = new DataOutputStream(byteArrayOutputStream); @@ -153,7 +167,24 @@ public class ReadWriteIOUtilsTest { Assert.assertNotNull(result); Assert.assertTrue(result.isEmpty()); - // 6. null + // 6. key: chinese; value: chinese + key = "中文"; + value = "中文"; + map = new HashMap<>(); + map.put(key, value); + byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE); + stream = new DataOutputStream(byteArrayOutputStream); + try { + ReadWriteIOUtils.write(map, stream); + } catch (IOException e) { + fail(e.toString()); + } + + result = ReadWriteIOUtils.readMap(ByteBuffer.wrap(byteArrayOutputStream.toByteArray())); + Assert.assertNotNull(result); + Assert.assertEquals(map, result); + + // 7. null map = null; byteArrayOutputStream = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE); stream = new DataOutputStream(byteArrayOutputStream);
