This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/2.2.1 in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit ec219619e8bf88dda0f80b4fa2ec04ebb5d2a65d Author: jintao zhu <[email protected]> AuthorDate: Fri Oct 10 19:02:11 2025 +0800 add encrypt param for all the write/read struct initialization (#601) * add encrypt param for all the write/read struct initialization * code review --- .../org/apache/tsfile/encrypt/EncryptUtils.java | 53 ++++++++-- .../tsfile/file/metadata/TsFileMetadata.java | 44 ++++---- .../apache/tsfile/read/TsFileRestorableReader.java | 30 +++++- .../apache/tsfile/read/TsFileSequenceReader.java | 99 +++++++++++++++++- .../apache/tsfile/read/UnClosedTsFileReader.java | 18 ++-- .../java/org/apache/tsfile/write/TsFileWriter.java | 111 ++++++++++++--------- .../write/chunk/AlignedChunkGroupWriterImpl.java | 6 +- .../write/v4/AbstractTableModelTsFileWriter.java | 71 +++++-------- .../write/writer/ForceAppendTsFileWriter.java | 14 ++- .../write/writer/RestorableTsFileIOWriter.java | 34 ++++++- .../apache/tsfile/write/writer/TsFileIOWriter.java | 85 ++++++++++------ 11 files changed, 386 insertions(+), 179 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java index 97b7ad57..ae75d88b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encrypt/EncryptUtils.java @@ -35,6 +35,7 @@ import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.Arrays; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; public class EncryptUtils { @@ -46,6 +47,9 @@ public class EncryptUtils { private static volatile EncryptParameter encryptParam; + private static ConcurrentHashMap<EncryptParameter, EncryptParameter> encryptParamCache = + new ConcurrentHashMap<>(); + private static final String HMAC_ALGORITHM = "HmacSHA256"; private static final int ITERATION_COUNT = 1024; private static final int SALT_LENGTH = 16; @@ -199,13 +203,23 @@ public class EncryptUtils { } md.update("IoTDB is the best".getBytes()); md.update(conf.getEncryptKey()); - byte[] data_key = Arrays.copyOfRange(md.digest(), 0, 16); - data_key = - IEncryptor.getEncryptor(conf.getEncryptType(), conf.getEncryptKey()).encrypt(data_key); + byte[] dataKey = Arrays.copyOfRange(md.digest(), 0, 16); + dataKey = IEncryptor.getEncryptor(conf.getEncryptType(), conf.getEncryptKey()).encrypt(dataKey); + + StringBuilder valueStr = new StringBuilder(); + + for (byte b : dataKey) { + valueStr.append(b).append(","); + } + + valueStr.deleteCharAt(valueStr.length() - 1); + return valueStr.toString(); + } + public static String getKeyStr(byte[] key) { StringBuilder valueStr = new StringBuilder(); - for (byte b : data_key) { + for (byte b : key) { valueStr.append(b).append(","); } @@ -213,23 +227,44 @@ public class EncryptUtils { return valueStr.toString(); } + /** Get the second EncryptParameter object according to the config file. */ public static EncryptParameter getEncryptParameter() { if (encryptParam == null) { synchronized (EncryptUtils.class) { if (encryptParam == null) { encryptParam = getEncryptParameter(TSFileDescriptor.getInstance().getConfig()); + if (!encryptParamCache.containsKey(encryptParam)) { + encryptParamCache.put( + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey()), + encryptParam); + } } } } return encryptParam; } + /** Get the second EncryptParameter object according to the given type and first key. */ + public static EncryptParameter getEncryptParameter(EncryptParameter param) { + return encryptParamCache.computeIfAbsent(param, EncryptUtils::generateEncryptParameter); + } + public static EncryptParameter getEncryptParameter(TSFileConfig conf) { - String encryptType; + return generateEncryptParameter( + new EncryptParameter(conf.getEncryptType(), conf.getEncryptKey())); + } + + /** + * Given a main EncryptParameter object, return a second EncryptParameter object with the same + * type but the data key generated from the given key. + */ + private static EncryptParameter generateEncryptParameter(EncryptParameter param) { + String encryptType = param.getType(); byte[] dataEncryptKey; - if (!Objects.equals(conf.getEncryptType(), "UNENCRYPTED") - && !Objects.equals(conf.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { - encryptType = conf.getEncryptType(); + if (!Objects.equals(encryptType, "UNENCRYPTED") + && !Objects.equals(encryptType, "org.apache.tsfile.encrypt.UNENCRYPTED")) { final MessageDigest md; try { md = MessageDigest.getInstance("SHA-256"); @@ -238,7 +273,7 @@ public class EncryptUtils { "SHA-256 algorithm not found while using SHA-256 to generate data key", e); } md.update("IoTDB is the best".getBytes()); - md.update(conf.getEncryptKey()); + md.update(param.getKey()); dataEncryptKey = Arrays.copyOfRange(md.digest(), 0, 16); } else { encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java index 5c77055c..e6a72879 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java @@ -19,11 +19,8 @@ package org.apache.tsfile.file.metadata; -import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.compatibility.DeserializeConfig; -import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; -import org.apache.tsfile.encrypt.IDecryptor; import org.apache.tsfile.exception.encrypt.EncryptException; import org.apache.tsfile.utils.BloomFilter; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; @@ -35,7 +32,6 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.TreeMap; /** TSFileMetaData collects all metadata info and saves in its data structure. */ @@ -55,7 +51,9 @@ public class TsFileMetadata { // offset from MetaMarker.SEPARATOR (exclusive) to tsFileProperties private int propertiesOffset; - private byte[] dataEncryptKey; + private int encryptLevel; + + private byte[] secondKey; private String encryptType; @@ -143,7 +141,8 @@ public class TsFileMetadata { throw new EncryptException("TsfileMetadata null encryptKey while encryptLevel is 1"); } String str = propertiesMap.get("encryptKey"); - fileMetaData.dataEncryptKey = EncryptUtils.getSecondKeyFromStr(str); + fileMetaData.encryptLevel = 1; + fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str); fileMetaData.encryptType = propertiesMap.get("encryptType"); } else if (propertiesMap.get("encryptLevel").equals("2")) { if (!propertiesMap.containsKey("encryptType")) { @@ -155,19 +154,9 @@ public class TsFileMetadata { if (propertiesMap.get("encryptKey") == null || propertiesMap.get("encryptKey").isEmpty()) { throw new EncryptException("TsfileMetadata null encryptKey while encryptLevel is 2"); } - if (Objects.equals( - TSFileDescriptor.getInstance().getConfig().getEncryptType(), - "org.apache.tsfile.encrypt.UNENCRYPTED") - || Objects.equals( - TSFileDescriptor.getInstance().getConfig().getEncryptType(), "UNENCRYPTED")) { - throw new EncryptException("fail to decrypt encrypted tsfile in unencrypted system"); - } - IDecryptor decryptor = - IDecryptor.getDecryptor( - propertiesMap.get("encryptType"), - TSFileDescriptor.getInstance().getConfig().getEncryptKey()); + fileMetaData.encryptLevel = 2; String str = propertiesMap.get("encryptKey"); - fileMetaData.dataEncryptKey = decryptor.decrypt(EncryptUtils.getSecondKeyFromStr(str)); + fileMetaData.secondKey = EncryptUtils.getSecondKeyFromStr(str); fileMetaData.encryptType = propertiesMap.get("encryptType"); } else { throw new EncryptException( @@ -179,13 +168,6 @@ public class TsFileMetadata { return fileMetaData; } - public EncryptParameter getEncryptParam() { - if (dataEncryptKey == null) { - return new EncryptParameter("org.apache.tsfile.encrypt.UNENCRYPTED", null); - } - return new EncryptParameter(encryptType, dataEncryptKey); - } - public void addProperty(String key, String value) { if (tsFileProperties == null) { tsFileProperties = new HashMap<>(); @@ -193,6 +175,18 @@ public class TsFileMetadata { tsFileProperties.put(key, value); } + public String getEncryptType() { + return encryptType; + } + + public byte[] getSecondKey() { + return secondKey; + } + + public int getEncryptLevel() { + return encryptLevel; + } + public BloomFilter getBloomFilter() { return bloomFilter; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java index 147d06e9..b593e262 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileRestorableReader.java @@ -19,6 +19,8 @@ package org.apache.tsfile.read; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.fileSystem.FSFactoryProducer; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.writer.RestorableTsFileIOWriter; @@ -33,17 +35,36 @@ public class TsFileRestorableReader extends TsFileSequenceReader { private static final Logger logger = LoggerFactory.getLogger(TsFileRestorableReader.class); public TsFileRestorableReader(String file) throws IOException { - this(file, true); + this( + file, + true, + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey())); + } + + public TsFileRestorableReader(String file, EncryptParameter param) throws IOException { + this(file, true, param); } public TsFileRestorableReader(String file, boolean autoRepair) throws IOException { + this( + file, + autoRepair, + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey())); + } + + public TsFileRestorableReader(String file, boolean autoRepair, EncryptParameter param) + throws IOException { // if autoRepair == true, then it means the file is likely broken, so we can not // read metadata // otherwise, the user may consider that either the file is complete, or the // user can accept an // Exception when reading broken data. Therefore, we set loadMetadata as true in // this case. - super(file, !autoRepair); + super(file, param, !autoRepair); if (autoRepair) { try { checkAndRepair(); @@ -62,8 +83,9 @@ public class TsFileRestorableReader extends TsFileSequenceReader { // Try to close it logger.info("File {} has no correct tail magic, try to repair...", file); try (RestorableTsFileIOWriter rWriter = - new RestorableTsFileIOWriter(FSFactoryProducer.getFSFactory().getFile(file)); - TsFileWriter writer = new TsFileWriter(rWriter)) { + new RestorableTsFileIOWriter( + FSFactoryProducer.getFSFactory().getFile(file), getFirstEncryptParam()); + TsFileWriter writer = new TsFileWriter(rWriter, getFirstEncryptParam())) { // This writes the right magic string } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 4330a600..bde4387c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -140,6 +140,11 @@ public class TsFileSequenceReader implements AutoCloseable { private DeserializeConfig deserializeConfig = new DeserializeConfig(); private volatile boolean cacheTableSchemaMap = false; + private EncryptParameter firstEncryptParam = + new EncryptParameter(config.getEncryptType(), config.getEncryptKey()); + + private EncryptParameter dataEncryptParam = null; + /** * Create a file reader of the given file. The reader will read the tail of the file to get the * file metadata size.Then the reader will skip the first @@ -150,7 +155,12 @@ public class TsFileSequenceReader implements AutoCloseable { * @throws IOException If some I/O error occurs */ public TsFileSequenceReader(String file) throws IOException { - this(file, null); + this(file, true, null); + } + + public TsFileSequenceReader(String file, EncryptParameter firstEncryptParam) throws IOException { + this(file, true, null); + this.firstEncryptParam = firstEncryptParam; } /** @@ -167,6 +177,13 @@ public class TsFileSequenceReader implements AutoCloseable { this(file, true, ioSizeRecorder); } + public TsFileSequenceReader( + String file, LongConsumer ioSizeRecorder, EncryptParameter firstEncryptParam) + throws IOException { + this(file, true, ioSizeRecorder); + this.firstEncryptParam = firstEncryptParam; + } + /** * construct function for TsFileSequenceReader. * @@ -177,6 +194,13 @@ public class TsFileSequenceReader implements AutoCloseable { this(file, loadMetadataSize, null); } + public TsFileSequenceReader( + String file, EncryptParameter firstEncryptParam, boolean loadMetadataSize) + throws IOException { + this(file, loadMetadataSize, null); + this.firstEncryptParam = firstEncryptParam; + } + /** * construct function for TsFileSequenceReader. * @@ -203,6 +227,16 @@ public class TsFileSequenceReader implements AutoCloseable { } } + public TsFileSequenceReader( + String file, + boolean loadMetadataSize, + LongConsumer ioSizeRecorder, + EncryptParameter firstEncryptParam) + throws IOException { + this(file, loadMetadataSize, ioSizeRecorder); + this.firstEncryptParam = firstEncryptParam; + } + // used in merge resource public TsFileSequenceReader(String file, boolean loadMetadata, boolean cacheDeviceMetadata) throws IOException { @@ -210,6 +244,16 @@ public class TsFileSequenceReader implements AutoCloseable { this.cacheDeviceMetadata = cacheDeviceMetadata; } + public TsFileSequenceReader( + String file, + boolean loadMetadata, + boolean cacheDeviceMetadata, + EncryptParameter firstEncryptParam) + throws IOException { + this(file, loadMetadata, cacheDeviceMetadata); + this.firstEncryptParam = firstEncryptParam; + } + /** * Create a file reader of the given file. The reader will read the tail of the file to get the * file metadata size.Then the reader will skip the first @@ -222,6 +266,12 @@ public class TsFileSequenceReader implements AutoCloseable { this(input, true); } + public TsFileSequenceReader(TsFileInput input, EncryptParameter firstEncryptParam) + throws IOException { + this(input, true); + this.firstEncryptParam = firstEncryptParam; + } + /** * construct function for TsFileSequenceReader. * @@ -241,6 +291,13 @@ public class TsFileSequenceReader implements AutoCloseable { } } + public TsFileSequenceReader( + TsFileInput input, boolean loadMetadataSize, EncryptParameter firstEncryptParam) + throws IOException { + this(input, loadMetadataSize); + this.firstEncryptParam = firstEncryptParam; + } + /** * construct function for TsFileSequenceReader. * @@ -256,6 +313,15 @@ public class TsFileSequenceReader implements AutoCloseable { this.fileMetadataSize = fileMetadataSize; } + public TsFileSequenceReader( + TsFileInput input, + long fileMetadataPos, + int fileMetadataSize, + EncryptParameter firstEncryptParam) { + this(input, fileMetadataPos, fileMetadataSize); + this.firstEncryptParam = firstEncryptParam; + } + // ioSizeRecorder can be null private void loadFileVersion(LongConsumer ioSizeRecorder) throws IOException { try { @@ -478,11 +544,34 @@ public class TsFileSequenceReader implements AutoCloseable { * @param ioSizeRecorder can be null */ public EncryptParameter getEncryptParam(LongConsumer ioSizeRecorder) throws IOException { - if (fileMetadataSize != 0) { - readFileMetadata(ioSizeRecorder); - return tsFileMetaData.getEncryptParam(); + if (dataEncryptParam != null) { + return dataEncryptParam; + } else { + if (fileMetadataSize != 0) { + readFileMetadata(ioSizeRecorder); + int encryptLevel = tsFileMetaData.getEncryptLevel(); + byte[] secondKey = tsFileMetaData.getSecondKey(); + String encryptType = tsFileMetaData.getEncryptType(); + if (secondKey == null) { + dataEncryptParam = new EncryptParameter("org.apache.tsfile.encrypt.UNENCRYPTED", null); + return dataEncryptParam; + } + if (encryptLevel == 1) { + dataEncryptParam = new EncryptParameter(encryptType, secondKey); + return dataEncryptParam; + } else if (encryptLevel == 2) { + IDecryptor decryptor = IDecryptor.getDecryptor(firstEncryptParam); + byte[] dataEncryptKey = decryptor.decrypt(secondKey); + dataEncryptParam = new EncryptParameter(encryptType, dataEncryptKey); + return dataEncryptParam; + } + } + return EncryptUtils.getEncryptParameter(firstEncryptParam); } - return EncryptUtils.getEncryptParameter(); + } + + public EncryptParameter getFirstEncryptParam() { + return firstEncryptParam; } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java index ce76fbc8..230a2e16 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/UnClosedTsFileReader.java @@ -19,6 +19,7 @@ package org.apache.tsfile.read; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.exception.NotImplementedException; @@ -29,20 +30,23 @@ import java.util.function.LongConsumer; /** A class for reading unclosed tsfile. */ public class UnClosedTsFileReader extends TsFileSequenceReader { - - private EncryptParameter encryptParam; + private final EncryptParameter dataEncryptParam; // ioSizeRecorder can be null public UnClosedTsFileReader(String file, LongConsumer ioSizeRecorder) throws IOException { - super(file, false, ioSizeRecorder); - encryptParam = EncryptUtils.getEncryptParameter(); + this( + file, + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey()), + ioSizeRecorder); } // ioSizeRecorder can be null public UnClosedTsFileReader( String file, EncryptParameter encryptParam, LongConsumer ioSizeRecorder) throws IOException { - super(file, false, ioSizeRecorder); - this.encryptParam = encryptParam; + super(file, false, ioSizeRecorder, encryptParam); + this.dataEncryptParam = EncryptUtils.getEncryptParameter(encryptParam); } /** unclosed file has no tail magic data. */ @@ -59,6 +63,6 @@ public class UnClosedTsFileReader extends TsFileSequenceReader { @Override public EncryptParameter getEncryptParam() { - return encryptParam; + return dataEncryptParam; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java index ff37612e..08ca25ce 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/TsFileWriter.java @@ -22,9 +22,9 @@ import org.apache.tsfile.annotations.TsFileApi; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.encrypt.IEncryptor; import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.exception.encrypt.EncryptException; import org.apache.tsfile.exception.write.ConflictDataTypeException; import org.apache.tsfile.exception.write.NoDeviceException; import org.apache.tsfile.exception.write.NoMeasurementException; @@ -54,10 +54,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,7 +76,7 @@ public class TsFileWriter implements AutoCloseable { /** IO writer of this TsFile. */ private final TsFileIOWriter fileWriter; - private EncryptParameter encryptParam; + private EncryptParameter secondEncryptParam; private final int pageSize; private long recordCount = 0; @@ -118,6 +115,14 @@ public class TsFileWriter implements AutoCloseable { this(new TsFileIOWriter(file), new Schema(), TSFileDescriptor.getInstance().getConfig()); } + public TsFileWriter(File file, EncryptParameter firstEncryptParam) throws IOException { + this( + new TsFileIOWriter(file), + new Schema(), + TSFileDescriptor.getInstance().getConfig(), + firstEncryptParam); + } + /** * init this TsFileWriter. * @@ -127,6 +132,11 @@ public class TsFileWriter implements AutoCloseable { this(fileWriter, new Schema(), TSFileDescriptor.getInstance().getConfig()); } + public TsFileWriter(TsFileIOWriter fileWriter, EncryptParameter firstEncryptParam) + throws IOException { + this(fileWriter, new Schema(), TSFileDescriptor.getInstance().getConfig(), firstEncryptParam); + } + /** * init this TsFileWriter. * @@ -137,6 +147,15 @@ public class TsFileWriter implements AutoCloseable { this(new TsFileIOWriter(file), schema, TSFileDescriptor.getInstance().getConfig()); } + public TsFileWriter(File file, Schema schema, EncryptParameter firstEncryptParam) + throws IOException { + this( + new TsFileIOWriter(file), + schema, + TSFileDescriptor.getInstance().getConfig(), + firstEncryptParam); + } + /** * init this TsFileWriter. * @@ -147,6 +166,15 @@ public class TsFileWriter implements AutoCloseable { this(new TsFileIOWriter(output), schema, TSFileDescriptor.getInstance().getConfig()); } + public TsFileWriter(TsFileOutput output, Schema schema, EncryptParameter firstEncryptParam) + throws IOException { + this( + new TsFileIOWriter(output), + schema, + TSFileDescriptor.getInstance().getConfig(), + firstEncryptParam); + } + /** * init this TsFileWriter. * @@ -158,6 +186,12 @@ public class TsFileWriter implements AutoCloseable { this(new TsFileIOWriter(file), schema, conf); } + public TsFileWriter( + File file, Schema schema, TSFileConfig conf, EncryptParameter firstEncryptParam) + throws IOException { + this(new TsFileIOWriter(file), schema, conf, firstEncryptParam); + } + /** * init this TsFileWriter. * @@ -167,6 +201,19 @@ public class TsFileWriter implements AutoCloseable { */ protected TsFileWriter(TsFileIOWriter fileWriter, Schema schema, TSFileConfig conf) throws IOException { + this( + fileWriter, + schema, + conf, + new EncryptParameter(conf.getEncryptType(), conf.getEncryptKey())); + } + + protected TsFileWriter( + TsFileIOWriter fileWriter, + Schema schema, + TSFileConfig conf, + EncryptParameter firstEncryptParam) + throws IOException { if (!fileWriter.canWrite()) { throw new IOException( "the given file Writer does not support writing any more. Maybe it is an complete TsFile"); @@ -188,48 +235,20 @@ public class TsFileWriter implements AutoCloseable { pageSize, chunkGroupSizeThreshold); } - + this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam); String encryptLevel; - byte[] encryptKey; - byte[] dataEncryptKey; - String encryptType; - if (!Objects.equals(config.getEncryptType(), "UNENCRYPTED") - && !Objects.equals(config.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { + if (firstEncryptParam != null + && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED") + && !Objects.equals(firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { encryptLevel = "2"; - encryptType = config.getEncryptType(); - final MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new EncryptException( - "SHA-256 algorithm not found while using SHA-256 to generate data key", e); - } - md.update("IoTDB is the best".getBytes()); - md.update(config.getEncryptKey()); - dataEncryptKey = Arrays.copyOfRange(md.digest(), 0, 16); - encryptKey = - IEncryptor.getEncryptor(config.getEncryptType(), config.getEncryptKey()) - .encrypt(dataEncryptKey); + String str = + EncryptUtils.getKeyStr( + IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey()) + .encrypt(secondEncryptParam.getKey())); + fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str); } else { encryptLevel = "0"; - encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; - encryptKey = null; - dataEncryptKey = null; - } - this.encryptParam = new EncryptParameter(encryptType, dataEncryptKey); - if (encryptKey != null) { - StringBuilder valueStr = new StringBuilder(); - - for (byte b : encryptKey) { - valueStr.append(b).append(","); - } - - valueStr.deleteCharAt(valueStr.length() - 1); - String str = valueStr.toString(); - - fileWriter.setEncryptParam(encryptLevel, encryptType, str); - } else { - fileWriter.setEncryptParam(encryptLevel, encryptType, ""); + fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", ""); } } @@ -511,8 +530,8 @@ public class TsFileWriter implements AutoCloseable { if (isAligned) { groupWriter = isTableModel - ? new TableChunkGroupWriterImpl(deviceId, encryptParam) - : new AlignedChunkGroupWriterImpl(deviceId, encryptParam); + ? new TableChunkGroupWriterImpl(deviceId, secondEncryptParam) + : new AlignedChunkGroupWriterImpl(deviceId, secondEncryptParam); initAllSeriesWriterForAlignedSeries( (AlignedChunkGroupWriterImpl) groupWriter, deviceId, isTableModel); if (!isUnseq) { // Sequence File @@ -520,7 +539,7 @@ public class TsFileWriter implements AutoCloseable { .setLastTime(alignedDeviceLastTimeMap.get(deviceId)); } } else { - groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, encryptParam); + groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, secondEncryptParam); if (!isUnseq) { // Sequence File ((NonAlignedChunkGroupWriterImpl) groupWriter) .setLastTimeMap( diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java index a47df975..3691f737 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java @@ -113,7 +113,8 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { measurementSchema.getCompressor(), measurementSchema.getType(), measurementSchema.getEncodingType(), - measurementSchema.getValueEncoder()); + measurementSchema.getValueEncoder(), + this.encryprParam); valueChunkWriterMap.put(measurementName, valueChunkWriter); tryToAddEmptyPageAndData(valueChunkWriter); } @@ -134,7 +135,8 @@ public class AlignedChunkGroupWriterImpl implements IChunkGroupWriter { schema.getCompressor(), schema.getType(), schema.getEncodingType(), - schema.getValueEncoder()); + schema.getValueEncoder(), + this.encryprParam); valueChunkWriterMap.put(measurementName, valueChunkWriter); tryToAddEmptyPageAndData(valueChunkWriter); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java index 3120bb40..260d4264 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java @@ -23,8 +23,8 @@ import org.apache.tsfile.annotations.TsFileApi; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.encrypt.EncryptParameter; +import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.encrypt.IEncryptor; -import org.apache.tsfile.exception.encrypt.EncryptException; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.write.chunk.AlignedChunkGroupWriterImpl; import org.apache.tsfile.write.chunk.IChunkGroupWriter; @@ -38,10 +38,7 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,7 +53,7 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter { /** IO writer of this TsFile. */ protected final TsFileIOWriter fileWriter; - protected EncryptParameter encryptParam; + protected EncryptParameter secondEncryptParam; protected final int pageSize; protected long recordCount = 0; @@ -85,6 +82,16 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter { @TsFileApi protected AbstractTableModelTsFileWriter(File file, long chunkGroupSizeThreshold) throws IOException { + this( + file, + chunkGroupSizeThreshold, + new EncryptParameter(config.getEncryptType(), config.getEncryptKey())); + } + + @TsFileApi + protected AbstractTableModelTsFileWriter( + File file, long chunkGroupSizeThreshold, EncryptParameter firstEncryptParam) + throws IOException { Schema schema = new Schema(); TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); this.fileWriter = new TsFileIOWriter(file); @@ -100,48 +107,20 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter { chunkGroupSizeThreshold); } + this.secondEncryptParam = EncryptUtils.getEncryptParameter(firstEncryptParam); String encryptLevel; - byte[] encryptKey; - byte[] dataEncryptKey; - String encryptType; - if (!Objects.equals(config.getEncryptType(), "UNENCRYPTED") - && !Objects.equals(config.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { + if (firstEncryptParam != null + && !Objects.equals(firstEncryptParam.getType(), "UNENCRYPTED") + && !Objects.equals(firstEncryptParam.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { encryptLevel = "2"; - encryptType = config.getEncryptType(); - - final MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new EncryptException( - "SHA-256 algorithm not found while using SHA-256 to generate data key", e); - } - md.update("IoTDB is the best".getBytes()); - md.update(config.getEncryptKey()); - dataEncryptKey = Arrays.copyOfRange(md.digest(), 0, 16); - encryptKey = - IEncryptor.getEncryptor(config.getEncryptType(), config.getEncryptKey()) - .encrypt(dataEncryptKey); + String str = + EncryptUtils.getKeyStr( + IEncryptor.getEncryptor(firstEncryptParam.getType(), firstEncryptParam.getKey()) + .encrypt(secondEncryptParam.getKey())); + fileWriter.setEncryptParam(encryptLevel, secondEncryptParam.getType(), str); } else { encryptLevel = "0"; - encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; - encryptKey = null; - dataEncryptKey = null; - } - this.encryptParam = new EncryptParameter(encryptType, dataEncryptKey); - if (encryptKey != null) { - StringBuilder valueStr = new StringBuilder(); - - for (byte b : encryptKey) { - valueStr.append(b).append(","); - } - - valueStr.deleteCharAt(valueStr.length() - 1); - String str = valueStr.toString(); - - fileWriter.setEncryptParam(encryptLevel, encryptType, str); - } else { - fileWriter.setEncryptParam(encryptLevel, encryptType, ""); + fileWriter.setEncryptParam(encryptLevel, "org.apache.tsfile.encrypt.UNENCRYPTED", ""); } } @@ -152,13 +131,13 @@ abstract class AbstractTableModelTsFileWriter implements ITsFileWriter { if (isAligned) { groupWriter = isTableModel - ? new TableChunkGroupWriterImpl(deviceId, encryptParam) - : new AlignedChunkGroupWriterImpl(deviceId, encryptParam); + ? new TableChunkGroupWriterImpl(deviceId, secondEncryptParam) + : new AlignedChunkGroupWriterImpl(deviceId, secondEncryptParam); ((AlignedChunkGroupWriterImpl) groupWriter) .setLastTime(alignedDeviceLastTimeMap.get(deviceId)); initAllSeriesWriterForAlignedSeries((AlignedChunkGroupWriterImpl) groupWriter); } else { - groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, encryptParam); + groupWriter = new NonAlignedChunkGroupWriterImpl(deviceId, secondEncryptParam); ((NonAlignedChunkGroupWriterImpl) groupWriter) .setLastTimeMap( nonAlignedTimeseriesLastTimeMap.getOrDefault(deviceId, new HashMap<>())); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java index 9f2de4e8..241514bd 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/ForceAppendTsFileWriter.java @@ -18,6 +18,8 @@ */ package org.apache.tsfile.write.writer; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.exception.write.TsFileNotCompleteException; import org.apache.tsfile.file.metadata.ChunkGroupMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; @@ -44,18 +46,28 @@ public class ForceAppendTsFileWriter extends TsFileIOWriter { private static Logger logger = LoggerFactory.getLogger(ForceAppendTsFileWriter.class); public ForceAppendTsFileWriter(File file) throws IOException { + this( + file, + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey())); + } + + public ForceAppendTsFileWriter(File file, EncryptParameter param) throws IOException { if (logger.isDebugEnabled()) { logger.debug("{} writer is opened.", file.getName()); } this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true); this.file = file; + setEncryptParam(param); // file doesn't exist if (file.length() == 0 || !file.exists()) { throw new TsFileNotCompleteException("File " + file.getPath() + " is not a complete TsFile"); } - try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) { + try (TsFileSequenceReader reader = + new TsFileSequenceReader(file.getAbsolutePath(), param, true)) { // this tsfile is not complete if (!reader.isComplete()) { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java index e01b741c..351fe58a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/RestorableTsFileIOWriter.java @@ -19,6 +19,8 @@ package org.apache.tsfile.write.writer; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotCompatibleTsFileException; import org.apache.tsfile.file.metadata.ChunkGroupMetadata; @@ -72,6 +74,8 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { private final Map<IDeviceID, Map<String, List<ChunkMetadata>>> metadatasForQuery = new HashMap<>(); + private EncryptParameter param; + /** * @param file a given tsfile path you want to (continue to) write * @throws IOException if write failed, or the file is broken but autoRepair==false. @@ -80,23 +84,49 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { this(file, true); } + public RestorableTsFileIOWriter(File file, EncryptParameter param) throws IOException { + this(file, true, param); + } + /** * @param file a given tsfile path you want to (continue to) write * @throws IOException if write failed, or the file is broken but autoRepair==false. */ public RestorableTsFileIOWriter(File file, long maxMetadataSize) throws IOException { - this(file, true); + this( + file, + true, + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey())); + } + + public RestorableTsFileIOWriter(File file, long maxMetadataSize, EncryptParameter param) + throws IOException { + this(file, true, param); this.maxMetadataSize = maxMetadataSize; this.chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX); this.checkMetadataSizeAndMayFlush(); } public RestorableTsFileIOWriter(File file, boolean truncate) throws IOException { + this( + file, + truncate, + new EncryptParameter( + TSFileDescriptor.getInstance().getConfig().getEncryptType(), + TSFileDescriptor.getInstance().getConfig().getEncryptKey())); + } + + public RestorableTsFileIOWriter(File file, boolean truncate, EncryptParameter param) + throws IOException { if (logger.isDebugEnabled()) { logger.debug("{} is opened.", file.getName()); } this.file = file; this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), true); + this.param = param; + setEncryptParam(param); // file doesn't exist if (file.length() == 0) { @@ -109,7 +139,7 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { try { if (file.exists()) { try (TsFileSequenceReader reader = - new TsFileSequenceReader(file.getAbsolutePath(), false)) { + new TsFileSequenceReader(file.getAbsolutePath(), param, false)) { schema.setEnabledUpdateSchema(false); truncatedSize = reader.selfCheck(schema, chunkGroupMetadataList, true); minPlanIndex = reader.getMinPlanIndex(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java index 31a6af52..8ac5eb2b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/writer/TsFileIOWriter.java @@ -21,7 +21,9 @@ package org.apache.tsfile.write.writer; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.encrypt.EncryptParameter; import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.encrypt.IEncryptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkGroupHeader; @@ -136,17 +138,12 @@ public class TsFileIOWriter implements AutoCloseable { /** empty construct function. */ protected TsFileIOWriter() { - if (!Objects.equals(TS_FILE_CONFIG.getEncryptType(), "UNENCRYPTED") - && !Objects.equals( - TS_FILE_CONFIG.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { - this.encryptLevel = "2"; - this.encryptType = TS_FILE_CONFIG.getEncryptType(); - this.encryptKey = EncryptUtils.getNormalKeyStr(); - } else { - this.encryptLevel = "0"; - this.encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; - this.encryptKey = null; - } + setEncryptParam( + new EncryptParameter(TS_FILE_CONFIG.getEncryptType(), TS_FILE_CONFIG.getEncryptKey())); + } + + protected TsFileIOWriter(EncryptParameter param) { + setEncryptParam(param); } /** @@ -159,23 +156,22 @@ public class TsFileIOWriter implements AutoCloseable { this(file, TS_FILE_CONFIG); } + public TsFileIOWriter(File file, EncryptParameter param) throws IOException { + this(file, TS_FILE_CONFIG, param); + } + /** for test only */ public TsFileIOWriter(File file, TSFileConfig conf) throws IOException { + this(file, conf, new EncryptParameter(conf.getEncryptType(), conf.getEncryptKey())); + } + + public TsFileIOWriter(File file, TSFileConfig conf, EncryptParameter param) throws IOException { this.out = FSFactoryProducer.getFileOutputFactory().getTsFileOutput(file.getPath(), false); this.file = file; if (resourceLogger.isDebugEnabled()) { resourceLogger.debug("{} writer is opened.", file.getName()); } - if (!Objects.equals(conf.getEncryptType(), "UNENCRYPTED") - && !Objects.equals(conf.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { - this.encryptLevel = "2"; - this.encryptType = conf.getEncryptType(); - this.encryptKey = EncryptUtils.getNormalKeyStr(); - } else { - this.encryptLevel = "0"; - this.encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; - this.encryptKey = null; - } + setEncryptParam(param); startFile(); } @@ -186,23 +182,22 @@ public class TsFileIOWriter implements AutoCloseable { */ public TsFileIOWriter(TsFileOutput output) throws IOException { this.out = output; - if (!Objects.equals(TS_FILE_CONFIG.getEncryptType(), "UNENCRYPTED") - && !Objects.equals( - TS_FILE_CONFIG.getEncryptType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { - this.encryptLevel = "2"; - this.encryptType = TS_FILE_CONFIG.getEncryptType(); - this.encryptKey = EncryptUtils.getNormalKeyStr(); - } else { - this.encryptLevel = "0"; - this.encryptType = "org.apache.tsfile.encrypt.UNENCRYPTED"; - this.encryptKey = null; - } + setEncryptParam( + new EncryptParameter(TS_FILE_CONFIG.getEncryptType(), TS_FILE_CONFIG.getEncryptKey())); + startFile(); + } + + public TsFileIOWriter(TsFileOutput output, EncryptParameter param) throws IOException { + this.out = output; + setEncryptParam(param); startFile(); } /** for test only */ public TsFileIOWriter(TsFileOutput output, boolean test) { this.out = output; + setEncryptParam( + new EncryptParameter(TS_FILE_CONFIG.getEncryptType(), TS_FILE_CONFIG.getEncryptKey())); } /** for write with memory control */ @@ -212,12 +207,38 @@ public class TsFileIOWriter implements AutoCloseable { chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX); } + public TsFileIOWriter(File file, long maxMetadataSize, EncryptParameter param) + throws IOException { + this(file, param); + this.maxMetadataSize = maxMetadataSize; + chunkMetadataTempFile = new File(file.getAbsolutePath() + CHUNK_METADATA_TEMP_FILE_SUFFIX); + } + public void setEncryptParam(String encryptLevel, String encryptType, String encryptKey) { this.encryptLevel = encryptLevel; this.encryptType = encryptType; this.encryptKey = encryptKey; } + public void setEncryptParam(EncryptParameter param) { + if (param == null) { + setEncryptParam("0", "org.apache.tsfile.encrypt.UNENCRYPTED", null); + } else { + if (!Objects.equals(param.getType(), "UNENCRYPTED") + && !Objects.equals(param.getType(), "org.apache.tsfile.encrypt.UNENCRYPTED")) { + String encryptLevel = "2"; + String encryptType = param.getType(); + String encryptKey = + EncryptUtils.getKeyStr( + IEncryptor.getEncryptor(param.getType(), param.getKey()) + .encrypt(EncryptUtils.getEncryptParameter(param).getKey())); + setEncryptParam(encryptLevel, encryptType, encryptKey); + } else { + setEncryptParam("0", "org.apache.tsfile.encrypt.UNENCRYPTED", null); + } + } + } + public void addFlushListener(FlushChunkMetadataListener listener) { flushListeners.add(listener); }
