This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new c1d81120 delete AES and related tests (#326)
c1d81120 is described below
commit c1d81120683bc27910af3d61bca9cab0c60d764f
Author: jintao zhu <[email protected]>
AuthorDate: Mon Dec 9 12:11:50 2024 +0800
delete AES and related tests (#326)
* delete AES
* delete AES
* delete AES
---------
Co-authored-by: zhujt <[email protected]>
---
.../apache/tsfile/common/conf/TSFileConfig.java | 2 +-
.../java/org/apache/tsfile/encrypt/AES128.java | 38 ---
.../org/apache/tsfile/encrypt/AES128Decryptor.java | 84 ------
.../org/apache/tsfile/encrypt/AES128Encryptor.java | 84 ------
.../org/apache/tsfile/encrypt/EncryptUtils.java | 4 -
.../java/org/apache/tsfile/encrypt/IDecryptor.java | 2 +-
.../java/org/apache/tsfile/encrypt/IEncryptor.java | 2 +-
.../tsfile/file/metadata/enums/EncryptionType.java | 7 +-
.../java/org/apache/tsfile/encrypt/AES128Test.java | 89 -------
.../tsfile/encrypt/AES128TsFileReadWriteTest.java | 265 -------------------
.../tsfile/encrypt/AES128TsFileWriteTest.java | 292 ---------------------
.../org/apache/tsfile/encrypt/EncryptTest.java | 49 +---
12 files changed, 12 insertions(+), 906 deletions(-)
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
index 99a71589..4db62146 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
@@ -161,7 +161,7 @@ public class TSFileConfig implements Serializable {
/** encryptKey, this should be 16 bytes String. */
private String encryptKey = "abcdefghijklmnop";
- /** default encryptType is "UNENCRYPTED", TsFile supports UNENCRYPTED or
AES128. */
+ /** default encryptType is "UNENCRYPTED". */
private String encryptType = "UNENCRYPTED";
/** Line count threshold for checking page memory occupied size. */
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128.java
b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128.java
deleted file mode 100644
index b69eea8e..00000000
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tsfile.encrypt;
-
-public class AES128 implements IEncrypt {
-
- private final byte[] key;
-
- public AES128(byte[] key) {
- this.key = key;
- }
-
- @Override
- public IEncryptor getEncryptor() {
- return new AES128Encryptor(key);
- }
-
- @Override
- public IDecryptor getDecryptor() {
- return new AES128Decryptor(key);
- }
-}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128Decryptor.java
b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128Decryptor.java
deleted file mode 100644
index 768f9afc..00000000
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128Decryptor.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tsfile.encrypt;
-
-import org.apache.tsfile.exception.encrypt.EncryptException;
-import org.apache.tsfile.exception.encrypt.EncryptKeyLengthNotMatchException;
-import org.apache.tsfile.file.metadata.enums.EncryptionType;
-
-import javax.crypto.BadPaddingException;
-import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-
-public class AES128Decryptor implements IDecryptor {
- private final Cipher AES;
-
- private final SecretKeySpec secretKeySpec;
-
- private final IvParameterSpec ivParameterSpec;
-
- AES128Decryptor(byte[] key) {
- if (key.length != 16) {
- throw new EncryptKeyLengthNotMatchException(16, key.length);
- }
- secretKeySpec = new SecretKeySpec(key, "AES");
- // Create IV parameter
- ivParameterSpec = new IvParameterSpec(key);
- try {
- // Create Cipher instance and initialize it for encryption in CTR mode
without padding
- this.AES = Cipher.getInstance("AES/CTR/NoPadding");
- AES.init(Cipher.DECRYPT_MODE, secretKeySpec, ivParameterSpec);
- } catch (InvalidAlgorithmParameterException
- | NoSuchPaddingException
- | NoSuchAlgorithmException
- | InvalidKeyException e) {
- throw new EncryptException("AES128Decryptor init failed ", e);
- }
- }
-
- @Override
- public byte[] decrypt(byte[] data) {
- try {
- return AES.doFinal(data);
- } catch (IllegalBlockSizeException | BadPaddingException e) {
- throw new EncryptException("AES128Decryptor decrypt failed ", e);
- }
- }
-
- @Override
- public byte[] decrypt(byte[] data, int offset, int size) {
- try {
- return AES.doFinal(data, offset, size);
- } catch (IllegalBlockSizeException | BadPaddingException e) {
- throw new EncryptException("AES128Decryptor decrypt failed ", e);
- }
- }
-
- @Override
- public EncryptionType getEncryptionType() {
- return EncryptionType.AES128;
- }
-}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128Encryptor.java
b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128Encryptor.java
deleted file mode 100644
index b4d8c2a7..00000000
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/AES128Encryptor.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tsfile.encrypt;
-
-import org.apache.tsfile.exception.encrypt.EncryptException;
-import org.apache.tsfile.exception.encrypt.EncryptKeyLengthNotMatchException;
-import org.apache.tsfile.file.metadata.enums.EncryptionType;
-
-import javax.crypto.BadPaddingException;
-import javax.crypto.Cipher;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-
-public class AES128Encryptor implements IEncryptor {
- private final Cipher AES;
-
- private final SecretKeySpec secretKeySpec;
-
- private final IvParameterSpec ivParameterSpec;
-
- AES128Encryptor(byte[] key) {
- if (key.length != 16) {
- throw new EncryptKeyLengthNotMatchException(16, key.length);
- }
- secretKeySpec = new SecretKeySpec(key, "AES");
- // Create IV parameter
- ivParameterSpec = new IvParameterSpec(key);
- try {
- // Create Cipher instance and initialize it for encryption in CTR mode
without padding
- this.AES = Cipher.getInstance("AES/CTR/NoPadding");
- AES.init(Cipher.ENCRYPT_MODE, secretKeySpec, ivParameterSpec);
- } catch (InvalidAlgorithmParameterException
- | NoSuchPaddingException
- | NoSuchAlgorithmException
- | InvalidKeyException e) {
- throw new EncryptException("AES128Encryptor init failed ", e);
- }
- }
-
- @Override
- public byte[] encrypt(byte[] data) {
- try {
- return AES.doFinal(data);
- } catch (IllegalBlockSizeException | BadPaddingException e) {
- throw new EncryptException("AES128Encryptor encrypt failed ", e);
- }
- }
-
- @Override
- public byte[] encrypt(byte[] data, int offset, int size) {
- try {
- return AES.doFinal(data, offset, size);
- } catch (IllegalBlockSizeException | BadPaddingException e) {
- throw new EncryptException("AES128Encryptor encrypt failed ", e);
- }
- }
-
- @Override
- public EncryptionType getEncryptionType() {
- return EncryptionType.AES128;
- }
-}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
index 4d70646b..86e86341 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java
@@ -21,7 +21,6 @@ package org.apache.tsfile.encrypt;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.exception.encrypt.EncryptException;
-import org.apache.tsfile.exception.encrypt.EncryptKeyLengthNotMatchException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,9 +78,6 @@ public class EncryptUtils {
sb.append("\n").append(line);
}
}
- if (sb.toString().length() != 16) {
- throw new EncryptKeyLengthNotMatchException(16,
sb.toString().length());
- }
return sb.toString();
} catch (IOException e) {
throw new EncryptException("Read main encrypt key error", e);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java
b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java
index bdf90b92..70724363 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IDecryptor.java
@@ -44,7 +44,7 @@ public interface IDecryptor {
IEncrypt.encryptMap.put(className, constructor);
return ((IEncrypt) constructor.newInstance(key)).getDecryptor();
} catch (ClassNotFoundException e) {
- throw new EncryptException("Get decryptor class failed: " + type, e);
+ throw new EncryptException("Get decryptor class failed, class not found:
" + type, e);
} catch (NoSuchMethodException e) {
throw new EncryptException("Get constructor for decryptor failed: " +
type, e);
} catch (InvocationTargetException | InstantiationException |
IllegalAccessException e) {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java
b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java
index 4fa63fd1..8e7878c2 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/IEncryptor.java
@@ -44,7 +44,7 @@ public interface IEncryptor {
IEncrypt.encryptMap.put(className, constructor);
return ((IEncrypt) constructor.newInstance(key)).getEncryptor();
} catch (ClassNotFoundException e) {
- throw new EncryptException("Get encryptor class failed: " + type, e);
+ throw new EncryptException("Get encryptor class failed, class not found:
" + type, e);
} catch (NoSuchMethodException e) {
throw new EncryptException("Get constructor for encryptor failed: " +
type, e);
} catch (InvocationTargetException | InstantiationException |
IllegalAccessException e) {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java
index f28d4812..6494f4c3 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/EncryptionType.java
@@ -27,7 +27,10 @@ public enum EncryptionType {
SM4128("SM4128", (byte) 1),
/** AES128. */
- AES128("AES128", (byte) 2);
+ AES128("AES128", (byte) 2),
+
+ /** NewWay. */
+ NewWay("NewWay", (byte) 3);
private final String extensionName;
private final byte index;
@@ -53,7 +56,7 @@ public enum EncryptionType {
case 2:
return EncryptionType.AES128;
default:
- throw new IllegalArgumentException("Invalid input: " + encryptor);
+ return EncryptionType.NewWay;
}
}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128Test.java
b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128Test.java
deleted file mode 100644
index 21d8d804..00000000
--- a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128Test.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tsfile.encrypt;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.ThreadLocalRandom;
-
-public class AES128Test {
- private final String key = "mkmkmkmkmkmkmkmk";
-
- private String randomString(int length) {
- StringBuilder builder = new StringBuilder(length);
- for (int i = 0; i < length; i++) {
- builder.append((char) (ThreadLocalRandom.current().nextInt(33, 128)));
- }
- return builder.toString();
- }
-
- @Before
- public void setUp() {}
-
- @After
- public void tearDown() {}
-
- @Test
- public void testBytes1() throws IOException {
- String input = randomString(2000000);
- byte[] unencrypted = input.getBytes(StandardCharsets.UTF_8);
- long time = System.currentTimeMillis();
- IEncryptor encryptor = new
AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] encrypted = encryptor.encrypt(unencrypted);
- System.out.println("encryption time cost:" + (System.currentTimeMillis() -
time));
- time = System.currentTimeMillis();
- IDecryptor decryptor = new
AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] decrypted = decryptor.decrypt(encrypted);
- System.out.println("decryption time cost:" + (System.currentTimeMillis() -
time));
- Assert.assertArrayEquals(unencrypted, decrypted);
- }
-
- @Test
- public void testBytes2() throws IOException {
- String input = randomString(500000);
- byte[] unencrypted = input.getBytes(StandardCharsets.UTF_8);
- long time = System.currentTimeMillis();
- IEncryptor encryptor = new
AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] encrypted = encryptor.encrypt(unencrypted, 0, unencrypted.length);
- System.out.println("encryption time cost:" + (System.currentTimeMillis() -
time));
- time = System.currentTimeMillis();
- IDecryptor decryptor = new
AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] decrypted = decryptor.decrypt(encrypted, 0, encrypted.length);
- System.out.println("decryption time cost:" + (System.currentTimeMillis() -
time));
- Assert.assertArrayEquals(unencrypted, decrypted);
- }
-
- @Test
- public void testBytes3() throws IOException {
- byte[] unencrypted = new byte[71];
- long time = System.currentTimeMillis();
- IEncryptor encryptor = new
AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] encrypted = encryptor.encrypt(unencrypted, 0, 35);
- System.out.println("encryption time cost:" + (System.currentTimeMillis() -
time));
- time = System.currentTimeMillis();
- IDecryptor decryptor = new
AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] decrypted = decryptor.decrypt(encrypted);
- System.out.println("decryption time cost:" + (System.currentTimeMillis() -
time));
- }
-}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileReadWriteTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileReadWriteTest.java
deleted file mode 100644
index f11c376d..00000000
---
a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileReadWriteTest.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tsfile.encrypt;
-
-import org.apache.tsfile.common.conf.TSFileConfig;
-import org.apache.tsfile.common.conf.TSFileDescriptor;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TsFileReader;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.common.Field;
-import org.apache.tsfile.read.common.Path;
-import org.apache.tsfile.read.common.RowRecord;
-import org.apache.tsfile.read.expression.QueryExpression;
-import org.apache.tsfile.read.query.dataset.QueryDataSet;
-import org.apache.tsfile.utils.TsFileGeneratorForTest;
-import org.apache.tsfile.write.TsFileWriter;
-import org.apache.tsfile.write.record.TSRecord;
-import org.apache.tsfile.write.record.datapoint.DataPoint;
-import org.apache.tsfile.write.record.datapoint.DoubleDataPoint;
-import org.apache.tsfile.write.record.datapoint.FloatDataPoint;
-import org.apache.tsfile.write.record.datapoint.IntDataPoint;
-import org.apache.tsfile.write.record.datapoint.LongDataPoint;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-import org.apache.tsfile.write.schema.Schema;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class AES128TsFileReadWriteTest {
- private final double delta = 0.0000001;
- private final String path =
TsFileGeneratorForTest.getTestTsFilePath("root.sg1", 0, 0, 1);
- private File f;
- private final IDeviceID deviceID =
IDeviceID.Factory.DEFAULT_FACTORY.create("device_1");
-
- private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
-
- @Before
- public void setUp() {
- conf.setEncryptFlag("true");
- conf.setEncryptType("AES128");
- conf.setEncryptKey("thisisourtestkey");
- f = new File(path);
- if (f.exists()) {
- assertTrue(f.delete());
- }
- if (!f.getParentFile().exists()) {
- assertTrue(f.getParentFile().mkdirs());
- }
- }
-
- @After
- public void tearDown() {
- conf.setEncryptKey("abcdefghijklmnop");
- conf.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED");
- conf.setEncryptFlag("false");
- f = new File(path);
- if (f.exists()) {
- assertTrue(f.delete());
- }
- }
-
- @Test
- public void intTest() throws IOException, WriteProcessException {
- List<TSEncoding> encodings =
- Arrays.asList(
- TSEncoding.PLAIN,
- TSEncoding.RLE,
- TSEncoding.TS_2DIFF,
- TSEncoding.REGULAR,
- TSEncoding.GORILLA,
- TSEncoding.ZIGZAG);
- for (TSEncoding encoding : encodings) {
- intTest(encoding);
- }
- }
-
- private void intTest(TSEncoding encoding) throws IOException,
WriteProcessException {
- writeDataByTSRecord(TSDataType.INT32, (i) -> new IntDataPoint("sensor_1",
(int) i), encoding);
- readData((i, field, delta) -> assertEquals(i, field.getIntV()));
- }
-
- @Test
- public void longTest() throws IOException, WriteProcessException {
- List<TSEncoding> encodings =
- Arrays.asList(
- TSEncoding.PLAIN,
- TSEncoding.RLE,
- TSEncoding.TS_2DIFF,
- TSEncoding.REGULAR,
- TSEncoding.GORILLA);
- for (TSEncoding encoding : encodings) {
- longTest(encoding);
- }
- }
-
- public void longTest(TSEncoding encoding) throws IOException,
WriteProcessException {
- writeDataByTSRecord(TSDataType.INT64, (i) -> new LongDataPoint("sensor_1",
i), encoding);
- readData((i, field, delta) -> assertEquals(i, field.getLongV()));
- }
-
- @Test
- public void floatTest() throws IOException, WriteProcessException {
- List<TSEncoding> encodings =
- Arrays.asList(
- TSEncoding.PLAIN,
- TSEncoding.RLE,
- TSEncoding.TS_2DIFF,
- TSEncoding.GORILLA_V1,
- TSEncoding.GORILLA);
- for (TSEncoding encoding : encodings) {
- floatTest(encoding);
- }
- }
-
- public void floatTest(TSEncoding encoding) throws IOException,
WriteProcessException {
- writeDataByTSRecord(
- TSDataType.FLOAT, (i) -> new FloatDataPoint("sensor_1", (float) i),
encoding);
- readData((i, field, delta) -> assertEquals(i, field.getFloatV(), delta));
- }
-
- @Test
- public void doubleTest() throws IOException, WriteProcessException {
- List<TSEncoding> encodings =
- Arrays.asList(
- TSEncoding.PLAIN,
- TSEncoding.RLE,
- TSEncoding.TS_2DIFF,
- TSEncoding.GORILLA_V1,
- TSEncoding.GORILLA);
- for (TSEncoding encoding : encodings) {
- doubleTest(encoding);
- }
- }
-
- public void doubleTest(TSEncoding encoding) throws IOException,
WriteProcessException {
- writeDataByTSRecord(
- TSDataType.DOUBLE, (i) -> new DoubleDataPoint("sensor_1", (double) i),
encoding);
- readData((i, field, delta) -> assertEquals(i, field.getDoubleV(), delta));
- }
-
- // If no dataPoint in "device_1.sensor_2", it will throws a nomeasurement
- // exception,
- // cause no schema in tsfilemetadata anymore.
- @Test
- public void readEmptyMeasurementTest() throws IOException,
WriteProcessException {
- try (TsFileWriter tsFileWriter = new TsFileWriter(f, new Schema(), conf)) {
- // add measurements into file schema
- tsFileWriter.registerTimeseries(
- new Path(deviceID), new MeasurementSchema("sensor_1",
TSDataType.FLOAT, TSEncoding.RLE));
- tsFileWriter.registerTimeseries(
- new Path(deviceID),
- new MeasurementSchema("sensor_2", TSDataType.INT32,
TSEncoding.TS_2DIFF));
- // construct TSRecord
- TSRecord tsRecord = new TSRecord(deviceID, 1);
- DataPoint dPoint1 = new FloatDataPoint("sensor_1", 1.2f);
- tsRecord.addTuple(dPoint1);
- // write a TSRecord to TsFile
- tsFileWriter.writeRecord(tsRecord);
- }
-
- // read example : no filter
- TsFileSequenceReader reader = new TsFileSequenceReader(path);
- TsFileReader readTsFile = new TsFileReader(reader);
- ArrayList<Path> paths = new ArrayList<>();
- paths.add(new Path(deviceID, "sensor_2", true));
- QueryExpression queryExpression = QueryExpression.create(paths, null);
- try {
- QueryDataSet queryDataSet = readTsFile.query(queryExpression);
- } catch (IOException e) {
- // Assert.fail();
- } finally {
- reader.close();
- }
-
- assertTrue(f.delete());
- }
-
- @Test
- public void readMeasurementWithRegularEncodingTest() throws IOException,
WriteProcessException {
- TSFileDescriptor.getInstance().getConfig().setTimeEncoder("REGULAR");
- writeDataByTSRecord(
- TSDataType.INT64, (i) -> new LongDataPoint("sensor_1", i),
TSEncoding.REGULAR);
- readData((i, field, delta) -> assertEquals(i, field.getLongV()));
- TSFileDescriptor.getInstance().getConfig().setTimeEncoder("TS_2DIFF");
- }
-
- private void writeDataByTSRecord(
- TSDataType dataType, AES128TsFileReadWriteTest.DataPointProxy proxy,
TSEncoding encodingType)
- throws IOException, WriteProcessException {
- int floatCount = 1024 * 1024 * 13 + 1023;
- // add measurements into file schema
- try (TsFileWriter tsFileWriter = new TsFileWriter(f, new Schema(), conf)) {
- tsFileWriter.registerTimeseries(
- new Path(deviceID), new MeasurementSchema("sensor_1", dataType,
encodingType));
- for (long i = 1; i < floatCount; i++) {
- // construct TSRecord
- TSRecord tsRecord = new TSRecord(deviceID, i);
- DataPoint dPoint1 = proxy.generateOne(i);
- tsRecord.addTuple(dPoint1);
- // write a TSRecord to TsFile
- tsFileWriter.writeRecord(tsRecord);
- }
- }
- }
-
- private void readData(AES128TsFileReadWriteTest.ReadDataPointProxy proxy)
throws IOException {
- TsFileSequenceReader reader = new TsFileSequenceReader(path);
- TsFileReader readTsFile = new TsFileReader(reader);
- ArrayList<Path> paths = new ArrayList<>();
- paths.add(new Path(deviceID, "sensor_1", true));
- QueryExpression queryExpression = QueryExpression.create(paths, null);
-
- QueryDataSet queryDataSet = readTsFile.query(queryExpression);
- for (int j = 0; j < paths.size(); j++) {
- assertEquals(paths.get(j), queryDataSet.getPaths().get(j));
- }
- int i = 1;
- while (queryDataSet.hasNext()) {
- RowRecord r = queryDataSet.next();
- assertEquals(i, r.getTimestamp());
- proxy.assertEqualProxy(i, r.getFields().get(0), delta);
- i++;
- }
- reader.close();
- }
-
- private interface DataPointProxy {
-
- DataPoint generateOne(long value);
- }
-
- private interface ReadDataPointProxy {
-
- void assertEqualProxy(long i, Field field, double delta);
- }
-}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileWriteTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileWriteTest.java
deleted file mode 100644
index 1eb49bc7..00000000
---
a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/AES128TsFileWriteTest.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tsfile.encrypt;
-
-import org.apache.tsfile.common.conf.TSFileConfig;
-import org.apache.tsfile.common.conf.TSFileDescriptor;
-import org.apache.tsfile.common.constant.JsonFormatConstant;
-import org.apache.tsfile.constant.TestConstant;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.exception.write.WriteProcessException;
-import org.apache.tsfile.file.metadata.TsFileMetadata;
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.common.Path;
-import org.apache.tsfile.utils.RecordUtils;
-import org.apache.tsfile.utils.StringContainer;
-import org.apache.tsfile.write.TsFileWriter;
-import org.apache.tsfile.write.WriteTest;
-import org.apache.tsfile.write.record.TSRecord;
-import org.apache.tsfile.write.schema.IMeasurementSchema;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-import org.apache.tsfile.write.schema.Schema;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Random;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class AES128TsFileWriteTest {
- private static final Logger LOG = LoggerFactory.getLogger(WriteTest.class);
- private final int ROW_COUNT = 2000000;
- private TsFileWriter tsFileWriter;
- private String inputDataFile;
- private String outputDataFile;
- private String errorOutputDataFile;
- private Random rm = new Random();
- private ArrayList<IMeasurementSchema> measurementArray;
- private ArrayList<Path> pathArray;
- private Schema schema;
- private int stageSize = 4;
- private int stageState = -1;
- private int prePageSize;
- private int prePageCheckThres;
- private TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
-
- private String[][] stageDeviceIds = {{"d1", "d2", "d3"}, {"d1"}, {"d2",
"d3"}};
- private String[] measurementIds = {"s0", "s1", "s2", "s3", "s4", "s5"};
- private long longBase = System.currentTimeMillis() * 1000;
- private String[] enums = {"MAN", "WOMAN"};
-
- @Before
- public void prepare() throws IOException {
- inputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("writeTestInputData");
- outputDataFile =
TestConstant.BASE_OUTPUT_PATH.concat("writeTestOutputData.tsfile");
- errorOutputDataFile =
TestConstant.BASE_OUTPUT_PATH.concat("writeTestErrorOutputData.tsfile");
- // for each row, flush page forcely
- prePageSize = conf.getPageSizeInByte();
- conf.setPageSizeInByte(0);
- prePageCheckThres = conf.getPageCheckSizeThreshold();
- conf.setPageCheckSizeThreshold(0);
- conf.setEncryptFlag("true");
- conf.setEncryptType("org.apache.tsfile.encrypt.AES128");
- conf.setEncryptKey("thisisourtestkey");
-
- try {
- generateSampleInputDataFile();
- } catch (IOException e) {
- LOG.error("meet error while creating test data " + e.getMessage());
- fail();
- }
- File file = new File(outputDataFile);
- File errorFile = new File(errorOutputDataFile);
- if (file.exists()) {
- file.delete();
- }
- if (errorFile.exists()) {
- errorFile.delete();
- }
- measurementArray = new ArrayList<>();
- measurementArray.add(new MeasurementSchema("s0", TSDataType.INT32,
TSEncoding.RLE));
- measurementArray.add(new MeasurementSchema("s1", TSDataType.INT64,
TSEncoding.TS_2DIFF));
- HashMap<String, String> props = new HashMap<>();
- props.put("max_point_number", "2");
- measurementArray.add(
- new MeasurementSchema(
- "s2",
- TSDataType.FLOAT,
- TSEncoding.RLE,
- TSFileDescriptor.getInstance().getConfig().getCompressor(),
- props));
- props = new HashMap<>();
- props.put("max_point_number", "3");
- measurementArray.add(
- new MeasurementSchema(
- "s3",
- TSDataType.DOUBLE,
- TSEncoding.TS_2DIFF,
- TSFileDescriptor.getInstance().getConfig().getCompressor(),
- props));
- measurementArray.add(new MeasurementSchema("s4", TSDataType.BOOLEAN,
TSEncoding.PLAIN));
- pathArray = new ArrayList<>();
- for (int i = 0; i < 5; i++) {
- pathArray.add(new Path("d1", "s" + i, true));
- }
- schema = new Schema();
- LOG.info(schema.toString());
- tsFileWriter = new TsFileWriter(file, schema, conf);
- }
-
- @After
- public void after() {
- conf.setPageSizeInByte(prePageSize);
- conf.setPageCheckSizeThreshold(prePageCheckThres);
- conf.setEncryptKey("abcdefghijklmnop");
- conf.setEncryptType("org.apache.tsfile.encrypt.UNENCRYPTED");
- conf.setEncryptFlag("false");
- File file = new File(inputDataFile);
- if (file.exists()) {
- file.delete();
- }
- file = new File(outputDataFile);
- if (file.exists()) {
- file.delete();
- }
- file = new File(errorOutputDataFile);
- if (file.exists()) {
- file.delete();
- }
- }
-
- private void generateSampleInputDataFile() throws IOException {
- File file = new File(inputDataFile);
- if (file.exists()) {
- file.delete();
- }
- FileWriter fw = new FileWriter(file);
-
- long startTime = System.currentTimeMillis();
- startTime = startTime - startTime % 1000;
-
- // first stage:int, long, float, double, boolean, enums
- try {
- for (int i = 0; i < ROW_COUNT; i++) {
- // write d1
- String d1 = "d1," + (startTime + i) + ",s1," + (i * 10 + 1) + ",s2," +
(i * 10 + 2);
- if (rm.nextInt(1000) < 100) {
- d1 = "d1," + (startTime + i) + ",s1,,s2," + (i * 10 + 2) +
",s4,HIGH";
- }
- if (i % 5 == 0) {
- d1 += ",s3," + (i * 10 + 3);
- }
- fw.write(d1 + "\r\n");
-
- // write d2
- String d2 = "d2," + (startTime + i) + ",s2," + (i * 10 + 2) + ",s3," +
(i * 10 + 3);
- if (rm.nextInt(1000) < 100) {
- d2 = "d2," + (startTime + i) + ",s2,,s3," + (i * 10 + 3) + ",s5,MAN";
- }
- if (i % 5 == 0) {
- d2 += ",s1," + (i * 10 + 1);
- }
- fw.write(d2 + "\r\n");
- }
- // write error
- String d =
- "d2,3,"
- + (startTime + ROW_COUNT)
- + ",s2,"
- + (ROW_COUNT * 10 + 2)
- + ",s3,"
- + (ROW_COUNT * 10 + 3);
- fw.write(d + "\r\n");
- d = "d2," + (startTime + ROW_COUNT + 1) + ",2,s-1," + (ROW_COUNT * 10 +
2);
- fw.write(d + "\r\n");
- } finally {
- fw.close();
- }
- }
-
- @Test
- public void writeTest() throws IOException {
- try {
- write();
- } catch (WriteProcessException e) {
- e.printStackTrace();
- }
- LOG.info("write processing has finished");
- TsFileSequenceReader reader = new TsFileSequenceReader(outputDataFile);
- TsFileMetadata metaData = reader.readFileMetadata();
- }
-
- public void write() throws IOException, WriteProcessException {
- long lineCount = 0;
- long startTime = System.currentTimeMillis();
- String[] strings;
- // add all measurement except the last one at before writing
- for (int i = 0; i < measurementArray.size() - 1; i++) {
- tsFileWriter.registerTimeseries(
- new Path(pathArray.get(i).getIDeviceID()), measurementArray.get(i));
- }
- while (true) {
- if (lineCount % stageSize == 0) {
- LOG.info(
- "write line:{},use time:{}s",
- lineCount,
- (System.currentTimeMillis() - startTime) / 1000);
- stageState++;
- LOG.info("stage:" + stageState);
- if (stageState == stageDeviceIds.length) {
- break;
- }
- }
- if (lineCount == ROW_COUNT / 2) {
- tsFileWriter.registerTimeseries(
- new Path(pathArray.get(measurementArray.size() -
1).getIDeviceID()),
- measurementArray.get(measurementArray.size() - 1));
- }
- strings = getNextRecord(lineCount, stageState);
- for (String str : strings) {
- TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
- if (record.dataPointList.isEmpty()) {
- continue;
- }
- tsFileWriter.writeRecord(record);
- }
- lineCount++;
- }
- // test duplicate measurement adding
- Path path = pathArray.get(measurementArray.size() - 1);
- IMeasurementSchema dupTimeseries =
measurementArray.get(measurementArray.size() - 1);
- try {
- tsFileWriter.registerTimeseries(new Path(path.getIDeviceID()),
dupTimeseries);
- } catch (WriteProcessException e) {
- assertEquals("given timeseries has exists! " + path, e.getMessage());
- }
- try {
- tsFileWriter.close();
- } catch (IOException e) {
- fail("close writer failed");
- }
- LOG.info("stage size: {}, write {} group data", stageSize, lineCount);
- }
-
- private String[] getNextRecord(long lineCount, int stage) {
-
- String[] ret = new String[stageDeviceIds[stage].length];
- for (int i = 0; i < ret.length; i++) {
- StringContainer sc = new
StringContainer(JsonFormatConstant.TSRECORD_SEPARATOR);
- sc.addTail(stageDeviceIds[stage][i], lineCount);
- sc.addTail(
- measurementIds[0],
- lineCount * 10 + i,
- measurementIds[1],
- longBase + lineCount * 20 + i,
- measurementIds[2],
- (lineCount * 30 + i) / 3.0,
- measurementIds[3],
- (longBase + lineCount * 40 + i) / 7.0);
- sc.addTail(measurementIds[4], ((lineCount + i) & 1) == 0);
- sc.addTail(measurementIds[5], enums[(int) (lineCount + i) %
enums.length]);
- ret[i] = sc.toString();
- }
- return ret;
- }
-}
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java
index dd2a6845..51b8f8a0 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/encrypt/EncryptTest.java
@@ -19,7 +19,6 @@
package org.apache.tsfile.encrypt;
import org.apache.tsfile.file.metadata.enums.EncryptionType;
-import org.apache.tsfile.utils.PublicBAOS;
import org.junit.After;
import org.junit.Before;
@@ -52,51 +51,11 @@ public class EncryptTest {
assertEquals(inputString, result);
}
- @Test
- public void AES128Test() throws IOException {
- IEncryptor encryptor = new
AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
- IDecryptor decryptor = new
AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] encrypted =
encryptor.encrypt(inputString.getBytes(StandardCharsets.UTF_8));
- byte[] decrypted = decryptor.decrypt(encrypted);
-
- String result = new String(decrypted, StandardCharsets.UTF_8);
- assertEquals(inputString, result);
- }
-
- @Test
- public void AES128Test1() throws IOException {
- PublicBAOS out = new PublicBAOS();
- out.write(inputString.getBytes(StandardCharsets.UTF_8));
- IEncryptor encryptor = new
AES128Encryptor(key.getBytes(StandardCharsets.UTF_8));
- IDecryptor decryptor = new
AES128Decryptor(key.getBytes(StandardCharsets.UTF_8));
- byte[] encrypted = encryptor.encrypt(out.getBuf(), 0, out.size());
- byte[] decrypted = decryptor.decrypt(encrypted);
-
- String result = new String(decrypted, StandardCharsets.UTF_8);
- assertEquals(inputString, result);
- }
-
- @Test
- public void AES128Test3() throws IOException {
- IEncryptor encryptor =
- IEncryptor.getEncryptor(
- "org.apache.tsfile.encrypt.AES128",
key.getBytes(StandardCharsets.UTF_8));
- IDecryptor decryptor =
- IDecryptor.getDecryptor(
- "org.apache.tsfile.encrypt.AES128",
key.getBytes(StandardCharsets.UTF_8));
- byte[] encrypted =
encryptor.encrypt(inputString.getBytes(StandardCharsets.UTF_8));
- byte[] decrypted = decryptor.decrypt(encrypted);
-
- String result = new String(decrypted, StandardCharsets.UTF_8);
- assertEquals(inputString, result);
- }
-
@Test
public void GetEncryptorTest() {
IEncryptor encryptor =
- IEncryptor.getEncryptor(
- "org.apache.tsfile.encrypt.AES128",
key.getBytes(StandardCharsets.UTF_8));
- assertEquals(encryptor.getEncryptionType(), EncryptionType.AES128);
+ IEncryptor.getEncryptor("UNENCRYPTED",
key.getBytes(StandardCharsets.UTF_8));
+ assertEquals(encryptor.getEncryptionType(), EncryptionType.UNENCRYPTED);
IEncryptor encryptor2 =
IEncryptor.getEncryptor(
"org.apache.tsfile.encrypt.UNENCRYPTED",
key.getBytes(StandardCharsets.UTF_8));
@@ -107,8 +66,8 @@ public class EncryptTest {
public void GetDecryptorTest() {
IEncryptor encryptor =
IEncryptor.getEncryptor(
- "org.apache.tsfile.encrypt.AES128",
key.getBytes(StandardCharsets.UTF_8));
- assertEquals(encryptor.getEncryptionType(), EncryptionType.AES128);
+ "org.apache.tsfile.encrypt.UNENCRYPTED",
key.getBytes(StandardCharsets.UTF_8));
+ assertEquals(encryptor.getEncryptionType(), EncryptionType.UNENCRYPTED);
IEncryptor encryptor2 =
IEncryptor.getEncryptor(
"org.apache.tsfile.encrypt.UNENCRYPTED",
key.getBytes(StandardCharsets.UTF_8));
