This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch NewTsFile in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5470d7c3c1c8bb9b846fd9d755dd81fb0e1e3766 Author: JackieTien97 <[email protected]> AuthorDate: Thu Nov 26 11:46:46 2020 +0800 change encode and decode way --- .../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +- .../iotdb/db/engine/upgrade/UpgradeTask.java | 308 ++--- .../iotdb/db/query/control/FileReaderManager.java | 5 - .../java/org/apache/iotdb/db/service/IoTDB.java | 2 +- .../org/apache/iotdb/db/service/UpgradeSevice.java | 244 ++-- .../apache/iotdb/db/tools/TsFileSketchTool.java | 2 +- .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 1233 ++++++++++---------- .../iotdb/tsfile/encoding/common/EndianType.java | 26 - .../iotdb/tsfile/encoding/decoder/Decoder.java | 8 +- .../tsfile/encoding/decoder/FloatDecoder.java | 11 +- .../tsfile/encoding/decoder/IntRleDecoder.java | 11 +- .../tsfile/encoding/decoder/LongRleDecoder.java | 11 +- .../tsfile/encoding/decoder/PlainDecoder.java | 21 +- .../iotdb/tsfile/encoding/decoder/RleDecoder.java | 15 +- .../tsfile/encoding/encoder/FloatEncoder.java | 6 +- .../tsfile/encoding/encoder/IntRleEncoder.java | 8 +- .../tsfile/encoding/encoder/LongRleEncoder.java | 6 +- .../tsfile/encoding/encoder/PlainEncoder.java | 52 +- .../iotdb/tsfile/encoding/encoder/RleEncoder.java | 20 +- .../tsfile/encoding/encoder/TSEncodingBuilder.java | 13 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 6 +- .../tsfile/read/reader/chunk/ChunkReader.java | 4 +- .../write/writer/RestorableTsFileIOWriter.java | 1 - .../tsfile/encoding/decoder/IntRleDecoderTest.java | 83 +- .../encoding/decoder/LongRleDecoderTest.java | 17 +- .../metadata/statistics/DoubleStatisticsTest.java | 10 +- .../metadata/statistics/FloatStatisticsTest.java | 11 +- .../metadata/statistics/IntegerStatisticsTest.java | 10 +- .../metadata/statistics/LongStatisticsTest.java | 15 +- .../iotdb/tsfile/file/metadata/utils/Utils.java | 36 +- .../tsfile/read/TsFileSequenceReaderTest.java | 2 +- .../iotdb/tsfile/read/reader/PageReaderTest.java | 54 +- .../iotdb/tsfile/write/TsFileIOWriterTest.java | 19 +- .../iotdb/tsfile/write/writer/PageWriterTest.java | 50 +- 34 files changed, 1067 insertions(+), 1255 deletions(-) diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java index f837b4b..e9314fa 100644 --- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java +++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java @@ -94,7 +94,7 @@ public class TsFileSequenceRead { break; case MetaMarker.CHUNK_GROUP_HEADER: System.out.println("Chunk Group Footer position: " + reader.position()); - ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupFooter(); + ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader(); System.out.println("device: " + chunkGroupHeader.getDeviceID()); break; case MetaMarker.VERSION: diff --git a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java index 015cc01..b170797 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java @@ -1,154 +1,154 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.upgrade; - -import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import org.apache.iotdb.db.concurrent.WrappedRunnable; -import org.apache.iotdb.db.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.modification.ModificationFile; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.service.UpgradeSevice; -import org.apache.iotdb.db.tools.upgrade.TsFileOnlineUpgradeTool; -import org.apache.iotdb.tsfile.exception.write.WriteProcessException; -import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; -import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class UpgradeTask extends WrappedRunnable { - - private TsFileResource upgradeResource; - private static final Logger logger = LoggerFactory.getLogger(UpgradeTask.class); - private static final String COMMA_SEPERATOR = ","; - private static final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum(); - - private FSFactory fsFactory = FSFactoryProducer.getFSFactory(); - - public UpgradeTask(TsFileResource upgradeResource) { - this.upgradeResource = upgradeResource; - } - - @Override - public void runMayThrow() { - try { - List<TsFileResource> upgradedResources = generateUpgradedFiles(); - upgradeResource.writeLock(); - String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath(); - String oldModificationFilePath = oldTsfilePath + ModificationFile.FILE_SUFFIX; - try { - // delete old TsFile and resource - upgradeResource.delete(); - File modificationFile = FSFactoryProducer.getFSFactory().getFile(oldModificationFilePath); - // move upgraded TsFiles and modificationFile to their own partition directories - for (TsFileResource upgradedResource : upgradedResources) { - File upgradedFile = upgradedResource.getTsFile(); - long partition = upgradedResource.getTimePartition(); - String storageGroupPath = upgradedFile.getParentFile().getParentFile().getParent(); - File partitionDir = FSFactoryProducer.getFSFactory() - .getFile(storageGroupPath, partition + ""); - if (!partitionDir.exists()) { - partitionDir.mkdir(); - } - FSFactoryProducer.getFSFactory().moveFile(upgradedFile, - FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName())); - upgradedResource.setFile( - FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName())); - // copy mods file to partition directories - if (modificationFile.exists()) { - Files.copy(modificationFile.toPath(), - FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName() - + ModificationFile.FILE_SUFFIX).toPath()); - } - upgradedResource.serialize(); - // delete tmp partition folder when it is empty - if (upgradedFile.getParentFile().isDirectory() - && upgradedFile.getParentFile().listFiles().length == 0) { - Files.delete(upgradedFile.getParentFile().toPath()); - } - // rename all files to 0 level - upgradedFile = upgradedResource.getTsFile(); - File zeroMergeVersionFile = getMaxMergeVersionFile(upgradedFile); - fsFactory.moveFile(upgradedFile, zeroMergeVersionFile); - fsFactory.moveFile( - fsFactory.getFile(upgradedFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX), - fsFactory - .getFile( - zeroMergeVersionFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX)); - upgradedResource.setFile(upgradedFile); - } - // delete old modificationFile - if (modificationFile.exists()) { - Files.delete(modificationFile.toPath()); - } - // delete upgrade folder when it is empty - if (upgradeResource.getTsFile().getParentFile().isDirectory() - && upgradeResource.getTsFile().getParentFile().listFiles().length == 0) { - Files.delete(upgradeResource.getTsFile().getParentFile().toPath()); - } - upgradeResource.setUpgradedResources(upgradedResources); - UpgradeLog.writeUpgradeLogFile( - oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.UPGRADE_SUCCESS); - upgradeResource.getUpgradeTsFileResourceCallBack().call(upgradeResource); - } finally { - upgradeResource.writeUnlock(); - } - UpgradeSevice.setCntUpgradeFileNum(UpgradeSevice.getCntUpgradeFileNum() - 1); - logger.info("Upgrade completes, file path:{} , the remaining upgraded file num: {}", - oldTsfilePath, UpgradeSevice.getCntUpgradeFileNum()); - } catch (Exception e) { - logger.error("meet error when upgrade file:{}", upgradeResource.getTsFile().getAbsolutePath(), - e); - } - } - - private List<TsFileResource> generateUpgradedFiles() throws WriteProcessException { - upgradeResource.readLock(); - String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath(); - List<TsFileResource> upgradedResources = new ArrayList<>(); - UpgradeLog.writeUpgradeLogFile( - oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.BEGIN_UPGRADE_FILE); - try { - TsFileOnlineUpgradeTool.upgradeOneTsfile(oldTsfilePath, upgradedResources); - UpgradeLog.writeUpgradeLogFile( - oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE); - } catch (IOException e) { - logger - .error("generate upgrade file failed, the file to be upgraded:{}", oldTsfilePath, e); - } finally { - upgradeResource.readUnlock(); - } - return upgradedResources; - } - - private File getMaxMergeVersionFile(File seqFile) { - String[] splits = seqFile.getName().replace(TSFILE_SUFFIX, "") - .split(IoTDBConstant.FILE_NAME_SEPARATOR); - return fsFactory.getFile(seqFile.getParentFile(), - splits[0] + IoTDBConstant.FILE_NAME_SEPARATOR + splits[1] - + IoTDBConstant.FILE_NAME_SEPARATOR + (maxLevelNum - 1) + TSFILE_SUFFIX); - } - -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, +// * software distributed under the License is distributed on an +// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// * KIND, either express or implied. See the License for the +// * specific language governing permissions and limitations +// * under the License. +// */ +//package org.apache.iotdb.db.engine.upgrade; +// +//import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.file.Files; +//import java.util.ArrayList; +//import java.util.List; +//import org.apache.iotdb.db.concurrent.WrappedRunnable; +//import org.apache.iotdb.db.conf.IoTDBConstant; +//import org.apache.iotdb.db.conf.IoTDBDescriptor; +//import org.apache.iotdb.db.engine.modification.ModificationFile; +//import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +//import org.apache.iotdb.db.service.UpgradeSevice; +//import org.apache.iotdb.db.tools.upgrade.TsFileOnlineUpgradeTool; +//import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +//import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; +//import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//public class UpgradeTask extends WrappedRunnable { +// +// private TsFileResource upgradeResource; +// private static final Logger logger = LoggerFactory.getLogger(UpgradeTask.class); +// private static final String COMMA_SEPERATOR = ","; +// private static final int maxLevelNum = IoTDBDescriptor.getInstance().getConfig().getSeqLevelNum(); +// +// private FSFactory fsFactory = FSFactoryProducer.getFSFactory(); +// +// public UpgradeTask(TsFileResource upgradeResource) { +// this.upgradeResource = upgradeResource; +// } +// +// @Override +// public void runMayThrow() { +// try { +// List<TsFileResource> upgradedResources = generateUpgradedFiles(); +// upgradeResource.writeLock(); +// String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath(); +// String oldModificationFilePath = oldTsfilePath + ModificationFile.FILE_SUFFIX; +// try { +// // delete old TsFile and resource +// upgradeResource.delete(); +// File modificationFile = FSFactoryProducer.getFSFactory().getFile(oldModificationFilePath); +// // move upgraded TsFiles and modificationFile to their own partition directories +// for (TsFileResource upgradedResource : upgradedResources) { +// File upgradedFile = upgradedResource.getTsFile(); +// long partition = upgradedResource.getTimePartition(); +// String storageGroupPath = upgradedFile.getParentFile().getParentFile().getParent(); +// File partitionDir = FSFactoryProducer.getFSFactory() +// .getFile(storageGroupPath, partition + ""); +// if (!partitionDir.exists()) { +// partitionDir.mkdir(); +// } +// FSFactoryProducer.getFSFactory().moveFile(upgradedFile, +// FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName())); +// upgradedResource.setFile( +// FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName())); +// // copy mods file to partition directories +// if (modificationFile.exists()) { +// Files.copy(modificationFile.toPath(), +// FSFactoryProducer.getFSFactory().getFile(partitionDir, upgradedFile.getName() +// + ModificationFile.FILE_SUFFIX).toPath()); +// } +// upgradedResource.serialize(); +// // delete tmp partition folder when it is empty +// if (upgradedFile.getParentFile().isDirectory() +// && upgradedFile.getParentFile().listFiles().length == 0) { +// Files.delete(upgradedFile.getParentFile().toPath()); +// } +// // rename all files to 0 level +// upgradedFile = upgradedResource.getTsFile(); +// File zeroMergeVersionFile = getMaxMergeVersionFile(upgradedFile); +// fsFactory.moveFile(upgradedFile, zeroMergeVersionFile); +// fsFactory.moveFile( +// fsFactory.getFile(upgradedFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX), +// fsFactory +// .getFile( +// zeroMergeVersionFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX)); +// upgradedResource.setFile(upgradedFile); +// } +// // delete old modificationFile +// if (modificationFile.exists()) { +// Files.delete(modificationFile.toPath()); +// } +// // delete upgrade folder when it is empty +// if (upgradeResource.getTsFile().getParentFile().isDirectory() +// && upgradeResource.getTsFile().getParentFile().listFiles().length == 0) { +// Files.delete(upgradeResource.getTsFile().getParentFile().toPath()); +// } +// upgradeResource.setUpgradedResources(upgradedResources); +// UpgradeLog.writeUpgradeLogFile( +// oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.UPGRADE_SUCCESS); +// upgradeResource.getUpgradeTsFileResourceCallBack().call(upgradeResource); +// } finally { +// upgradeResource.writeUnlock(); +// } +// UpgradeSevice.setCntUpgradeFileNum(UpgradeSevice.getCntUpgradeFileNum() - 1); +// logger.info("Upgrade completes, file path:{} , the remaining upgraded file num: {}", +// oldTsfilePath, UpgradeSevice.getCntUpgradeFileNum()); +// } catch (Exception e) { +// logger.error("meet error when upgrade file:{}", upgradeResource.getTsFile().getAbsolutePath(), +// e); +// } +// } +// +// private List<TsFileResource> generateUpgradedFiles() throws WriteProcessException { +// upgradeResource.readLock(); +// String oldTsfilePath = upgradeResource.getTsFile().getAbsolutePath(); +// List<TsFileResource> upgradedResources = new ArrayList<>(); +// UpgradeLog.writeUpgradeLogFile( +// oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.BEGIN_UPGRADE_FILE); +// try { +// TsFileOnlineUpgradeTool.upgradeOneTsfile(oldTsfilePath, upgradedResources); +// UpgradeLog.writeUpgradeLogFile( +// oldTsfilePath + COMMA_SEPERATOR + UpgradeCheckStatus.AFTER_UPGRADE_FILE); +// } catch (IOException e) { +// logger +// .error("generate upgrade file failed, the file to be upgraded:{}", oldTsfilePath, e); +// } finally { +// upgradeResource.readUnlock(); +// } +// return upgradedResources; +// } +// +// private File getMaxMergeVersionFile(File seqFile) { +// String[] splits = seqFile.getName().replace(TSFILE_SUFFIX, "") +// .split(IoTDBConstant.FILE_NAME_SEPARATOR); +// return fsFactory.getFile(seqFile.getParentFile(), +// splits[0] + IoTDBConstant.FILE_NAME_SEPARATOR + splits[1] +// + IoTDBConstant.FILE_NAME_SEPARATOR + (maxLevelNum - 1) + TSFILE_SUFFIX); +// } +// +//} diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java index 776fcdb..9a9eb18 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java +++ b/server/src/main/java/org/apache/iotdb/db/query/control/FileReaderManager.java @@ -33,7 +33,6 @@ import org.apache.iotdb.db.service.ServiceType; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.UnClosedTsFileReader; -import org.apache.iotdb.tsfile.v1.read.TsFileSequenceReaderForV1; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -168,10 +167,6 @@ public class FileReaderManager implements IService { else { tsFileReader = new TsFileSequenceReader(filePath); switch (tsFileReader.readVersionNumber()) { - case TSFileConfig.VERSION_NUMBER_V1: - tsFileReader.close(); - tsFileReader = new TsFileSequenceReaderForV1(filePath); - break; case TSFileConfig.VERSION_NUMBER_V2: break; default: diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index f56e7ca..60d7095 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -137,7 +137,7 @@ public class IoTDB implements IoTDBMBean { } registerManager.register(SyncServerManager.getInstance()); - registerManager.register(UpgradeSevice.getINSTANCE()); +// registerManager.register(UpgradeSevice.getINSTANCE()); registerManager.register(MergeManager.getINSTANCE()); registerManager.register(CompactionMergeTaskPoolManager.getInstance()); diff --git a/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java b/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java index b7d1b89..d3260c7 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java +++ b/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java @@ -1,122 +1,122 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.service; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.StorageEngine; -import org.apache.iotdb.db.engine.upgrade.UpgradeLog; -import org.apache.iotdb.db.engine.upgrade.UpgradeTask; -import org.apache.iotdb.db.exception.StartupException; -import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.utils.UpgradeUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class UpgradeSevice implements IService { - - private static final Logger logger = LoggerFactory.getLogger(UpgradeSevice.class); - - private static final UpgradeSevice INSTANCE = new UpgradeSevice(); - private ExecutorService upgradeThreadPool; - private AtomicInteger threadCnt = new AtomicInteger(); - private static int cntUpgradeFileNum; - - - private UpgradeSevice() { - } - - public static UpgradeSevice getINSTANCE() { - return INSTANCE; - } - - @Override - public void start() throws StartupException { - int updateThreadNum = IoTDBDescriptor.getInstance().getConfig().getUpgradeThreadNum(); - if (updateThreadNum <= 0) { - updateThreadNum = 1; - } - upgradeThreadPool = Executors.newFixedThreadPool(updateThreadNum, - r -> new Thread(r, "UpgradeThread-" + threadCnt.getAndIncrement())); - UpgradeLog.createUpgradeLog(); - countUpgradeFiles(); - if (cntUpgradeFileNum == 0) { - stop(); - return; - } - upgradeAll(); - } - - @Override - public void stop() { - UpgradeLog.closeLogWriter(); - if (upgradeThreadPool != null) { - upgradeThreadPool.shutdownNow(); - logger.info("Waiting for upgrade task pool to shut down"); - while (!upgradeThreadPool.isTerminated()) { - // wait - } - upgradeThreadPool = null; - logger.info("Upgrade service stopped"); - } - } - - @Override - public ServiceType getID() { - return ServiceType.UPGRADE_SERVICE; - } - - - public static void setCntUpgradeFileNum(int cntUpgradeFileNum) { - UpgradeUtils.getCntUpgradeFileLock().writeLock().lock(); - try { - UpgradeSevice.cntUpgradeFileNum = cntUpgradeFileNum; - } finally { - UpgradeUtils.getCntUpgradeFileLock().writeLock().unlock(); - } - } - - public static int getCntUpgradeFileNum() { - UpgradeUtils.getCntUpgradeFileLock().readLock().lock(); - try { - return cntUpgradeFileNum; - } finally { - UpgradeUtils.getCntUpgradeFileLock().readLock().unlock(); - } - } - - public void submitUpgradeTask(UpgradeTask upgradeTask) { - upgradeThreadPool.submit(upgradeTask); - } - - private static void countUpgradeFiles() { - cntUpgradeFileNum = StorageEngine.getInstance().countUpgradeFiles(); - logger.info("finish counting upgrading files, total num:{}", cntUpgradeFileNum); - } - - private static void upgradeAll() { - try { - StorageEngine.getInstance().upgradeAll(); - } catch (StorageEngineException e) { - logger.error("Cannot perform a global upgrade because", e); - } - } -} +///* +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, +// * software distributed under the License is distributed on an +// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// * KIND, either express or implied. See the License for the +// * specific language governing permissions and limitations +// * under the License. +// */ +//package org.apache.iotdb.db.service; +// +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +//import java.util.concurrent.atomic.AtomicInteger; +//import org.apache.iotdb.db.conf.IoTDBDescriptor; +//import org.apache.iotdb.db.engine.StorageEngine; +//import org.apache.iotdb.db.engine.upgrade.UpgradeLog; +//import org.apache.iotdb.db.engine.upgrade.UpgradeTask; +//import org.apache.iotdb.db.exception.StartupException; +//import org.apache.iotdb.db.exception.StorageEngineException; +//import org.apache.iotdb.db.utils.UpgradeUtils; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//public class UpgradeSevice implements IService { +// +// private static final Logger logger = LoggerFactory.getLogger(UpgradeSevice.class); +// +// private static final UpgradeSevice INSTANCE = new UpgradeSevice(); +// private ExecutorService upgradeThreadPool; +// private AtomicInteger threadCnt = new AtomicInteger(); +// private static int cntUpgradeFileNum; +// +// +// private UpgradeSevice() { +// } +// +// public static UpgradeSevice getINSTANCE() { +// return INSTANCE; +// } +// +// @Override +// public void start() throws StartupException { +// int updateThreadNum = IoTDBDescriptor.getInstance().getConfig().getUpgradeThreadNum(); +// if (updateThreadNum <= 0) { +// updateThreadNum = 1; +// } +// upgradeThreadPool = Executors.newFixedThreadPool(updateThreadNum, +// r -> new Thread(r, "UpgradeThread-" + threadCnt.getAndIncrement())); +// UpgradeLog.createUpgradeLog(); +// countUpgradeFiles(); +// if (cntUpgradeFileNum == 0) { +// stop(); +// return; +// } +// upgradeAll(); +// } +// +// @Override +// public void stop() { +// UpgradeLog.closeLogWriter(); +// if (upgradeThreadPool != null) { +// upgradeThreadPool.shutdownNow(); +// logger.info("Waiting for upgrade task pool to shut down"); +// while (!upgradeThreadPool.isTerminated()) { +// // wait +// } +// upgradeThreadPool = null; +// logger.info("Upgrade service stopped"); +// } +// } +// +// @Override +// public ServiceType getID() { +// return ServiceType.UPGRADE_SERVICE; +// } +// +// +// public static void setCntUpgradeFileNum(int cntUpgradeFileNum) { +// UpgradeUtils.getCntUpgradeFileLock().writeLock().lock(); +// try { +// UpgradeSevice.cntUpgradeFileNum = cntUpgradeFileNum; +// } finally { +// UpgradeUtils.getCntUpgradeFileLock().writeLock().unlock(); +// } +// } +// +// public static int getCntUpgradeFileNum() { +// UpgradeUtils.getCntUpgradeFileLock().readLock().lock(); +// try { +// return cntUpgradeFileNum; +// } finally { +// UpgradeUtils.getCntUpgradeFileLock().readLock().unlock(); +// } +// } +// +// public void submitUpgradeTask(UpgradeTask upgradeTask) { +// upgradeThreadPool.submit(upgradeTask); +// } +// +// private static void countUpgradeFiles() { +// cntUpgradeFileNum = StorageEngine.getInstance().countUpgradeFiles(); +// logger.info("finish counting upgrading files, total num:{}", cntUpgradeFileNum); +// } +// +// private static void upgradeAll() { +// try { +// StorageEngine.getInstance().upgradeAll(); +// } catch (StorageEngineException e) { +// logger.error("Cannot perform a global upgrade because", e); +// } +// } +//} diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java index 779fb56..b72d99a 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java @@ -105,7 +105,7 @@ public class TsFileSketchTool { } // chunkGroupFooter begins printlnBoth(pw, String.format("%20s", chunkEndPos) + "|\t[Chunk Group Footer]"); - ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupFooter(chunkEndPos, false); + ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader(chunkEndPos, false); printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 0"); printlnBoth(pw, String.format("%20s", "") + "|\t\t[deviceID] " + chunkGroupHeader.getDeviceID()); diff --git a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java index 52b7eb3..7f75ce8 100644 --- a/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java +++ b/server/src/main/java/org/apache/iotdb/db/tools/upgrade/TsFileOnlineUpgradeTool.java @@ -1,619 +1,614 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.upgrade; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import org.apache.iotdb.db.engine.StorageEngine; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.compress.IUnCompressor; -import org.apache.iotdb.tsfile.encoding.decoder.Decoder; -import org.apache.iotdb.tsfile.exception.write.PageException; -import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; -import org.apache.iotdb.tsfile.exception.write.WriteProcessException; -import org.apache.iotdb.tsfile.file.MetaMarker; -import org.apache.iotdb.tsfile.file.footer.ChunkGroupHeader; -import org.apache.iotdb.tsfile.file.header.ChunkHeader; -import org.apache.iotdb.tsfile.file.header.PageHeader; -import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; -import org.apache.iotdb.tsfile.read.common.BatchData; -import org.apache.iotdb.tsfile.read.reader.TsFileInput; -import org.apache.iotdb.tsfile.read.reader.page.PageReader; -import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; -import org.apache.iotdb.tsfile.v1.file.metadata.ChunkGroupMetaDataV1; -import org.apache.iotdb.tsfile.v1.file.metadata.TsDeviceMetadataIndexV1; -import org.apache.iotdb.tsfile.v1.file.metadata.TsDeviceMetadataV1; -import org.apache.iotdb.tsfile.v1.file.metadata.TsFileMetadataV1; -import org.apache.iotdb.tsfile.v1.file.utils.HeaderUtils; -import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; -import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; -import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TsFileOnlineUpgradeTool implements AutoCloseable { - - private static final Logger logger = LoggerFactory.getLogger(TsFileOnlineUpgradeTool.class); - - private TsFileInput tsFileInput; - private long fileMetadataPos; - private int fileMetadataSize; - private ByteBuffer markerBuffer = ByteBuffer.allocate(Byte.BYTES); - private Decoder defaultTimeDecoder = Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64); - private Decoder valueDecoder; - protected String file; - - // PartitionId -> TsFileIOWriter - private Map<Long, TsFileIOWriter> partitionWriterMap; - - /** - * Create a file reader of the given file. The reader will read the tail of the file to get the - * file metadata size.Then the reader will skip the first TSFileConfig.OLD_MAGIC_STRING.length() - * bytes of the file for preparing reading real data. - * - * @param file the data file - * @throws IOException If some I/O error occurs - */ - public TsFileOnlineUpgradeTool(String file) throws IOException { - this(file, true); - } - - /** - * construct function for TsfileOnlineUpgradeTool. - * - * @param file -given file name - * @param loadMetadataSize -load meta data size - */ - public TsFileOnlineUpgradeTool(String file, boolean loadMetadataSize) throws IOException { - this.file = file; - tsFileInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file); - partitionWriterMap = new HashMap<>(); - try { - if (loadMetadataSize) { - loadMetadataSize(); - } - } catch (Exception e) { - tsFileInput.close(); - throw e; - } - } - - /** - * upgrade a single tsfile - * - * @param tsFileName old version tsFile's absolute path - * @param upgradedResources new version tsFiles' resources - */ - public static void upgradeOneTsfile(String tsFileName, List<TsFileResource> upgradedResources) - throws IOException, WriteProcessException { - try (TsFileOnlineUpgradeTool updater = new TsFileOnlineUpgradeTool(tsFileName)) { - updater.upgradeFile(upgradedResources); - } - } - - /** - * - */ - public void loadMetadataSize() throws IOException { - ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES); - tsFileInput.read(metadataSize, - tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES); - metadataSize.flip(); - // read file metadata size and position - fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize); - fileMetadataPos = - tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES - - fileMetadataSize; - // skip the magic header - position(TSFileConfig.MAGIC_STRING.length()); - } - - public String readTailMagic() throws IOException { - long totalSize = tsFileInput.size(); - - ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length()); - tsFileInput.read(magicStringBytes, totalSize - TSFileConfig.MAGIC_STRING.length()); - magicStringBytes.flip(); - return new String(magicStringBytes.array()); - } - - /** - * whether the file is a complete TsFile: only if the head magic and tail magic string exists. - */ - public boolean isComplete() throws IOException { - return tsFileInput.size() >= TSFileConfig.MAGIC_STRING.length() * 2 && readTailMagic() - .equals(readHeadMagic()); - } - - /** - * this function does not modify the position of the file reader. - */ - public String readHeadMagic() throws IOException { - return readHeadMagic(false); - } - - /** - * @param movePosition whether move the position of the file reader after reading the magic header - * to the end of the magic head string. - */ - public String readHeadMagic(boolean movePosition) throws IOException { - ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length()); - if (movePosition) { - tsFileInput.position(0); - tsFileInput.read(magicStringBytes); - } else { - tsFileInput.read(magicStringBytes, 0); - } - magicStringBytes.flip(); - return new String(magicStringBytes.array()); - } - - /** - * this function reads version number and checks compatibility of TsFile. - */ - public String readVersionNumber() throws IOException { - ByteBuffer versionNumberBytes = ByteBuffer - .allocate(TSFileConfig.VERSION_NUMBER_V2.getBytes().length); - tsFileInput.position(TSFileConfig.MAGIC_STRING.getBytes().length); - tsFileInput.read(versionNumberBytes); - versionNumberBytes.flip(); - return new String(versionNumberBytes.array()); - } - - /** - * this function does not modify the position of the file reader. - */ - public TsFileMetadataV1 readFileMetadata() throws IOException { - return TsFileMetadataV1.deserializeFrom(readData(fileMetadataPos, fileMetadataSize)); - } - - /** - * this function does not modify the position of the file reader. - */ - public TsDeviceMetadataV1 readTsDeviceMetaData(TsDeviceMetadataIndexV1 index) throws IOException { - return TsDeviceMetadataV1.deserializeFrom(readData(index.getOffset(), index.getLen())); - } - - /** - * read data from current position of the input, and deserialize it to a CHUNK_GROUP_FOOTER. <br> - * This method is not threadsafe. - * - * @return a CHUNK_GROUP_FOOTER - * @throws IOException io error - */ - public ChunkGroupHeader readChunkGroupFooter() throws IOException { - return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true); - } - - /** - * read data from current position of the input, and deserialize it to a CHUNK_HEADER. <br> This - * method is not threadsafe. - * - * @return a CHUNK_HEADER - * @throws IOException io error - */ - public ChunkHeader readChunkHeader() throws IOException { - return HeaderUtils.deserializeChunkHeaderV1(tsFileInput.wrapAsInputStream(), true); - } - - /** - * not thread safe. - * - * @param type given tsfile data type - */ - public PageHeader readPageHeader(TSDataType type) throws IOException { - return HeaderUtils.deserializePageHeaderV1(tsFileInput.wrapAsInputStream(), type); - } - - public ByteBuffer readPage(PageHeader header, CompressionType type) - throws IOException { - ByteBuffer buffer = readData(-1, header.getCompressedSize()); - IUnCompressor unCompressor = IUnCompressor.getUnCompressor(type); - ByteBuffer uncompressedBuffer = ByteBuffer.allocate(header.getUncompressedSize()); - if (type == CompressionType.UNCOMPRESSED) { - return buffer; - } - unCompressor.uncompress(buffer.array(), buffer.position(), buffer.remaining(), - uncompressedBuffer.array(), - 0); - return uncompressedBuffer; - } - - public ByteBuffer readCompressedPage(PageHeader header) throws IOException { - return readData(-1, header.getCompressedSize()); - } - - public long position() throws IOException { - return tsFileInput.position(); - } - - public void position(long offset) throws IOException { - tsFileInput.position(offset); - } - - /** - * read one byte from the input. <br> this method is not thread safe - */ - public byte readMarker() throws IOException { - markerBuffer.clear(); - if (ReadWriteIOUtils.readAsPossible(tsFileInput, markerBuffer) == 0) { - throw new IOException("reach the end of the file."); - } - markerBuffer.flip(); - return markerBuffer.get(); - } - - public byte readMarker(long position) throws IOException { - return readData(position, Byte.BYTES).get(); - } - - public void close() throws IOException { - this.tsFileInput.close(); - } - - public String getFileName() { - return this.file; - } - - /** - * read data from tsFileInput, from the current position (if position = -1), or the given - * position. <br> if position = -1, the tsFileInput's position will be changed to the current - * position + real data size that been read. Other wise, the tsFileInput's position is not - * changed. - * - * @param position the start position of data in the tsFileInput, or the current position if - * position = -1 - * @param size the size of data that want to read - * @return data that been read. - */ - private ByteBuffer readData(long position, int size) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(size); - if (position == -1) { - if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) { - throw new IOException("reach the end of the data"); - } - } else { - if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size) != size) { - throw new IOException("reach the end of the data"); - } - } - buffer.flip(); - return buffer; - } - - /** - * upgrade file and resource - * - * @throws IOException, WriteProcessException - */ - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - public void upgradeFile(List<TsFileResource> upgradedResources) - throws IOException, WriteProcessException { - File oldTsFile = FSFactoryProducer.getFSFactory().getFile(this.file); - - // check if the old TsFile has correct header - if (!fileCheck(oldTsFile)) { - return; - } - - // ChunkGroupOffset -> version - Map<Long, Long> oldVersionInfo = getVersionInfo(); - - // start to scan chunks and chunkGroups - long startOffsetOfChunkGroup = 0; - boolean newChunkGroup = true; - long versionOfChunkGroup = 0; - int chunkGroupCount = 0; - List<List<PageHeader>> pageHeadersInChunkGroup = new ArrayList<>(); - List<List<ByteBuffer>> pageDataInChunkGroup = new ArrayList<>(); - List<List<Boolean>> pagePartitionInfoInChunkGroup = new ArrayList<>(); - byte marker; - List<MeasurementSchema> measurementSchemaList = new ArrayList<>(); - try { - while ((marker = this.readMarker()) != MetaMarker.SEPARATOR) { - switch (marker) { - case MetaMarker.CHUNK_HEADER: - // this is the first chunk of a new ChunkGroup. - if (newChunkGroup) { - newChunkGroup = false; - startOffsetOfChunkGroup = this.position() - 1; - versionOfChunkGroup = oldVersionInfo.get(startOffsetOfChunkGroup); - } - ChunkHeader header = this.readChunkHeader(); - MeasurementSchema measurementSchema = new MeasurementSchema(header.getMeasurementID(), - header.getDataType(), - header.getEncodingType(), - header.getCompressionType()); - measurementSchemaList.add(measurementSchema); - List<PageHeader> pageHeadersInChunk = new ArrayList<>(); - List<ByteBuffer> dataInChunk = new ArrayList<>(); - List<Boolean> pagePartitionInfo = new ArrayList<>(); - for (int j = 0; j < header.getNumOfPages(); j++) { - PageHeader pageHeader = readPageHeader(header.getDataType()); - boolean pageInSamePartition = checkIfPageInSameTimePartition(pageHeader); - pagePartitionInfo.add(pageInSamePartition); - ByteBuffer pageData = pageInSamePartition ? - readCompressedPage(pageHeader) - : readPage(pageHeader, header.getCompressionType()); - pageHeadersInChunk.add(pageHeader); - dataInChunk.add(pageData); - } - pageHeadersInChunkGroup.add(pageHeadersInChunk); - pageDataInChunkGroup.add(dataInChunk); - pagePartitionInfoInChunkGroup.add(pagePartitionInfo); - break; - case MetaMarker.CHUNK_GROUP_HEADER: - // this is the footer of a ChunkGroup. - ChunkGroupHeader chunkGroupHeader = this.readChunkGroupFooter(); - String deviceID = chunkGroupHeader.getDeviceID(); - rewrite(oldTsFile, deviceID, measurementSchemaList, pageHeadersInChunkGroup, - pageDataInChunkGroup, versionOfChunkGroup, pagePartitionInfoInChunkGroup); - - pageHeadersInChunkGroup.clear(); - pageDataInChunkGroup.clear(); - measurementSchemaList.clear(); - pagePartitionInfoInChunkGroup.clear(); - newChunkGroup = true; - chunkGroupCount++; - break; - - default: - // the disk file is corrupted, using this file may be dangerous - logger.error("Unrecognized marker detected, this file may be corrupted"); - return; - } - } - // close upgraded tsFiles and generate resources for them - for (TsFileIOWriter tsFileIOWriter : partitionWriterMap.values()) { - upgradedResources.add(endFileAndGenerateResource(tsFileIOWriter)); - } - } catch (IOException e2) { - logger.info("TsFile upgrade process cannot proceed at position {} after {} chunk groups " - + "recovered, because : {}", this.position(), chunkGroupCount, e2.getMessage()); - } finally { - if (tsFileInput != null) { - tsFileInput.close(); - } - } - } - - private boolean checkIfPageInSameTimePartition(PageHeader pageHeader) { - return StorageEngine.getTimePartition(pageHeader.getStartTime()) - == StorageEngine.getTimePartition(pageHeader.getEndTime()); - } - - /** - * This method is for rewriting the ChunkGroup which data is in the different time partitions. In - * this case, we have to decode the data to points, and then rewrite the data points to different - * chunkWriters, finally write chunks to their own upgraded TsFiles - */ - private void rewrite(File oldTsFile, String deviceId, List<MeasurementSchema> schemas, - List<List<PageHeader>> pageHeadersInChunkGroup, List<List<ByteBuffer>> dataInChunkGroup, - long versionOfChunkGroup, List<List<Boolean>> pagePartitionInfoInChunkGroup) - throws IOException, PageException { - Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup = new HashMap<>(); - for (int i = 0; i < schemas.size(); i++) { - MeasurementSchema schema = schemas.get(i); - List<ByteBuffer> pageDataInChunk = dataInChunkGroup.get(i); - List<PageHeader> pageHeadersInChunk = pageHeadersInChunkGroup.get(i); - List<Boolean> pagePartitionInfo = pagePartitionInfoInChunkGroup.get(i); - valueDecoder = Decoder - .getDecoderByType(schema.getEncodingType(), schema.getType()); - for (int j = 0; j < pageDataInChunk.size(); j++) { - if (Boolean.TRUE.equals(pagePartitionInfo.get(j))) { - writePageInSamePartitionToFile(oldTsFile, schema, pageHeadersInChunk.get(j), - pageDataInChunk.get(j), chunkWritersInChunkGroup); - } else { - writePageInDifferentPartitionsToFiles(oldTsFile, schema, pageDataInChunk.get(j), - chunkWritersInChunkGroup); - } - } - } - - for (Entry<Long, Map<MeasurementSchema, ChunkWriterImpl>> entry : chunkWritersInChunkGroup - .entrySet()) { - long partitionId = entry.getKey(); - TsFileIOWriter tsFileIOWriter = partitionWriterMap.get(partitionId); - tsFileIOWriter.startChunkGroup(deviceId); - // write chunks to their own upgraded tsFiles - for (IChunkWriter chunkWriter : entry.getValue().values()) { - chunkWriter.writeToFileWriter(tsFileIOWriter); - } - tsFileIOWriter.endChunkGroup(); - tsFileIOWriter.writeVersion(versionOfChunkGroup); - } - } - - private TsFileIOWriter getOrDefaultTsFileIOWriter(File oldTsFile, long partition) { - return partitionWriterMap.computeIfAbsent(partition, k -> - { - File partitionDir = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent() - + File.separator + partition); - if (!partitionDir.exists()) { - partitionDir.mkdirs(); - } - File newFile = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent() - + File.separator + partition + File.separator + oldTsFile.getName()); - try { - if (!newFile.createNewFile()) { - logger.error("The TsFile {} has been created ", newFile); - return null; - } - return new TsFileIOWriter(newFile); - } catch (IOException e) { - logger.error("Create new TsFile {} failed ", newFile); - return null; - } - } - ); - } - - private void writePageInSamePartitionToFile(File oldTsFile, MeasurementSchema schema, - PageHeader pageHeader, - ByteBuffer pageData, - Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup) - throws PageException { - long partitionId = StorageEngine.getTimePartition(pageHeader.getStartTime()); - getOrDefaultTsFileIOWriter(oldTsFile, partitionId); - Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup - .getOrDefault(partitionId, new HashMap<>()); - ChunkWriterImpl chunkWriter = chunkWriters - .getOrDefault(schema, new ChunkWriterImpl(schema)); - chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); - chunkWriters.put(schema, chunkWriter); - chunkWritersInChunkGroup.put(partitionId, chunkWriters); - } - - private void writePageInDifferentPartitionsToFiles(File oldTsFile, MeasurementSchema schema, - ByteBuffer pageData, - Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup) - throws IOException { - valueDecoder.reset(); - PageReader pageReader = new PageReader(pageData, schema.getType(), valueDecoder, - defaultTimeDecoder, null); - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - long time = batchData.currentTime(); - Object value = batchData.currentValue(); - long partitionId = StorageEngine.getTimePartition(time); - - Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup - .getOrDefault(partitionId, new HashMap<>()); - ChunkWriterImpl chunkWriter = chunkWriters - .getOrDefault(schema, new ChunkWriterImpl(schema)); - getOrDefaultTsFileIOWriter(oldTsFile, partitionId); - switch (schema.getType()) { - case INT32: - chunkWriter.write(time, (int) value); - break; - case INT64: - chunkWriter.write(time, (long) value); - break; - case FLOAT: - chunkWriter.write(time, (float) value); - break; - case DOUBLE: - chunkWriter.write(time, (double) value); - break; - case BOOLEAN: - chunkWriter.write(time, (boolean) value); - break; - case TEXT: - chunkWriter.write(time, (Binary) value); - break; - default: - throw new UnSupportedDataTypeException( - String.format("Data type %s is not supported.", schema.getType())); - } - batchData.next(); - chunkWriters.put(schema, chunkWriter); - chunkWritersInChunkGroup.put(partitionId, chunkWriters); - } - } - - /** - * check if the file to be upgraded has correct magic strings and version number - */ - private boolean fileCheck(File oldTsFile) throws IOException { - long fileSize; - if (!oldTsFile.exists()) { - logger.error("the file to be updated does not exist, file path: {}", oldTsFile.getPath()); - return false; - } else { - fileSize = oldTsFile.length(); - } - - String magic = readHeadMagic(true); - if (!magic.equals(TSFileConfig.MAGIC_STRING)) { - logger.error("the file's MAGIC STRING is incorrect, file path: {}", oldTsFile.getPath()); - return false; - } - - String versionNumber = readVersionNumber(); - if (!versionNumber.equals(TSFileConfig.VERSION_NUMBER_V1)) { - logger.error("the file's Version Number is incorrect, file path: {}", oldTsFile.getPath()); - return false; - } - - if (fileSize == TSFileConfig.MAGIC_STRING.length()) { - logger.error("the file only contains magic string, file path: {}", oldTsFile.getPath()); - return false; - } else if (!readTailMagic().equals(TSFileConfig.MAGIC_STRING)) { - logger.error("the file cannot upgrade, file path: {}", oldTsFile.getPath()); - return false; - } - return true; - } - - private Map<Long, Long> getVersionInfo() throws IOException { - Map<Long, Long> versionInfo = new HashMap<>(); - TsFileMetadataV1 fileMetadata = readFileMetadata(); - List<TsDeviceMetadataV1> oldDeviceMetadataList = new ArrayList<>(); - for (TsDeviceMetadataIndexV1 index : fileMetadata.getDeviceMap().values()) { - TsDeviceMetadataV1 oldDeviceMetadata = readTsDeviceMetaData(index); - oldDeviceMetadataList.add(oldDeviceMetadata); - } - - for (TsDeviceMetadataV1 oldTsDeviceMetadata : oldDeviceMetadataList) { - for (ChunkGroupMetaDataV1 oldChunkGroupMetadata : oldTsDeviceMetadata - .getChunkGroupMetaDataList()) { - long version = oldChunkGroupMetadata.getVersion(); - long offsetOfChunkGroup = oldChunkGroupMetadata.getStartOffsetOfChunkGroup(); - // get version informations - versionInfo.put(offsetOfChunkGroup, version); - } - } - return versionInfo; - } - - private TsFileResource endFileAndGenerateResource(TsFileIOWriter tsFileIOWriter) - throws IOException { - tsFileIOWriter.endFile(); - TsFileResource tsFileResource = new TsFileResource(tsFileIOWriter.getFile()); - Map<String, List<TimeseriesMetadata>> deviceTimeseriesMetadataMap = tsFileIOWriter - .getDeviceTimeseriesMetadataMap(); - for (Map.Entry<String, List<TimeseriesMetadata>> entry : deviceTimeseriesMetadataMap - .entrySet()) { - String device = entry.getKey(); - for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) { - tsFileResource.updateStartTime(device, timeseriesMetaData.getStatistics().getStartTime()); - tsFileResource.updateEndTime(device, timeseriesMetaData.getStatistics().getEndTime()); - } - } - tsFileResource.setClosed(true); - return tsFileResource; - } - -} \ No newline at end of file +///* +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, +// * software distributed under the License is distributed on an +// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// * KIND, either express or implied. See the License for the +// * specific language governing permissions and limitations +// * under the License. +// */ +//package org.apache.iotdb.db.tools.upgrade; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.ByteBuffer; +//import java.util.ArrayList; +//import java.util.HashMap; +//import java.util.List; +//import java.util.Map; +//import java.util.Map.Entry; +//import org.apache.iotdb.db.engine.StorageEngine; +//import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +//import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +//import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +//import org.apache.iotdb.tsfile.compress.IUnCompressor; +//import org.apache.iotdb.tsfile.encoding.decoder.Decoder; +//import org.apache.iotdb.tsfile.exception.write.PageException; +//import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; +//import org.apache.iotdb.tsfile.exception.write.WriteProcessException; +//import org.apache.iotdb.tsfile.file.MetaMarker; +//import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; +//import org.apache.iotdb.tsfile.file.header.ChunkHeader; +//import org.apache.iotdb.tsfile.file.header.PageHeader; +//import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata; +//import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +//import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +//import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +//import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; +//import org.apache.iotdb.tsfile.read.common.BatchData; +//import org.apache.iotdb.tsfile.read.reader.TsFileInput; +//import org.apache.iotdb.tsfile.read.reader.page.PageReader; +//import org.apache.iotdb.tsfile.utils.Binary; +//import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; +//import org.apache.iotdb.tsfile.write.chunk.ChunkWriterImpl; +//import org.apache.iotdb.tsfile.write.chunk.IChunkWriter; +//import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +//import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//public class TsFileOnlineUpgradeTool implements AutoCloseable { +// +// private static final Logger logger = LoggerFactory.getLogger(TsFileOnlineUpgradeTool.class); +// +// private TsFileInput tsFileInput; +// private long fileMetadataPos; +// private int fileMetadataSize; +// private ByteBuffer markerBuffer = ByteBuffer.allocate(Byte.BYTES); +// private Decoder defaultTimeDecoder = Decoder.getDecoderByType( +// TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), +// TSDataType.INT64); +// private Decoder valueDecoder; +// protected String file; +// +// // PartitionId -> TsFileIOWriter +// private Map<Long, TsFileIOWriter> partitionWriterMap; +// +// /** +// * Create a file reader of the given file. The reader will read the tail of the file to get the +// * file metadata size.Then the reader will skip the first TSFileConfig.OLD_MAGIC_STRING.length() +// * bytes of the file for preparing reading real data. +// * +// * @param file the data file +// * @throws IOException If some I/O error occurs +// */ +// public TsFileOnlineUpgradeTool(String file) throws IOException { +// this(file, true); +// } +// +// /** +// * construct function for TsfileOnlineUpgradeTool. +// * +// * @param file -given file name +// * @param loadMetadataSize -load meta data size +// */ +// public TsFileOnlineUpgradeTool(String file, boolean loadMetadataSize) throws IOException { +// this.file = file; +// tsFileInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file); +// partitionWriterMap = new HashMap<>(); +// try { +// if (loadMetadataSize) { +// loadMetadataSize(); +// } +// } catch (Exception e) { +// tsFileInput.close(); +// throw e; +// } +// } +// +// /** +// * upgrade a single tsfile +// * +// * @param tsFileName old version tsFile's absolute path +// * @param upgradedResources new version tsFiles' resources +// */ +// public static void upgradeOneTsfile(String tsFileName, List<TsFileResource> upgradedResources) +// throws IOException, WriteProcessException { +// try (TsFileOnlineUpgradeTool updater = new TsFileOnlineUpgradeTool(tsFileName)) { +// updater.upgradeFile(upgradedResources); +// } +// } +// +// /** +// * +// */ +// public void loadMetadataSize() throws IOException { +// ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES); +// tsFileInput.read(metadataSize, +// tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES); +// metadataSize.flip(); +// // read file metadata size and position +// fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize); +// fileMetadataPos = +// tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES +// - fileMetadataSize; +// // skip the magic header +// position(TSFileConfig.MAGIC_STRING.length()); +// } +// +// public String readTailMagic() throws IOException { +// long totalSize = tsFileInput.size(); +// +// ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length()); +// tsFileInput.read(magicStringBytes, totalSize - TSFileConfig.MAGIC_STRING.length()); +// magicStringBytes.flip(); +// return new String(magicStringBytes.array()); +// } +// +// /** +// * whether the file is a complete TsFile: only if the head magic and tail magic string exists. +// */ +// public boolean isComplete() throws IOException { +// return tsFileInput.size() >= TSFileConfig.MAGIC_STRING.length() * 2 && readTailMagic() +// .equals(readHeadMagic()); +// } +// +// /** +// * this function does not modify the position of the file reader. +// */ +// public String readHeadMagic() throws IOException { +// return readHeadMagic(false); +// } +// +// /** +// * @param movePosition whether move the position of the file reader after reading the magic header +// * to the end of the magic head string. +// */ +// public String readHeadMagic(boolean movePosition) throws IOException { +// ByteBuffer magicStringBytes = ByteBuffer.allocate(TSFileConfig.MAGIC_STRING.length()); +// if (movePosition) { +// tsFileInput.position(0); +// tsFileInput.read(magicStringBytes); +// } else { +// tsFileInput.read(magicStringBytes, 0); +// } +// magicStringBytes.flip(); +// return new String(magicStringBytes.array()); +// } +// +// /** +// * this function reads version number and checks compatibility of TsFile. +// */ +// public String readVersionNumber() throws IOException { +// ByteBuffer versionNumberBytes = ByteBuffer +// .allocate(TSFileConfig.VERSION_NUMBER_V2.getBytes().length); +// tsFileInput.position(TSFileConfig.MAGIC_STRING.getBytes().length); +// tsFileInput.read(versionNumberBytes); +// versionNumberBytes.flip(); +// return new String(versionNumberBytes.array()); +// } +// +// /** +// * this function does not modify the position of the file reader. +// */ +// public TsFileMetadataV1 readFileMetadata() throws IOException { +// return TsFileMetadataV1.deserializeFrom(readData(fileMetadataPos, fileMetadataSize)); +// } +// +// /** +// * this function does not modify the position of the file reader. +// */ +// public TsDeviceMetadataV1 readTsDeviceMetaData(TsDeviceMetadataIndexV1 index) throws IOException { +// return TsDeviceMetadataV1.deserializeFrom(readData(index.getOffset(), index.getLen())); +// } +// +// /** +// * read data from current position of the input, and deserialize it to a CHUNK_GROUP_FOOTER. <br> +// * This method is not threadsafe. +// * +// * @return a CHUNK_GROUP_FOOTER +// * @throws IOException io error +// */ +// public ChunkGroupHeader readChunkGroupFooter() throws IOException { +// return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true); +// } +// +// /** +// * read data from current position of the input, and deserialize it to a CHUNK_HEADER. <br> This +// * method is not threadsafe. +// * +// * @return a CHUNK_HEADER +// * @throws IOException io error +// */ +// public ChunkHeader readChunkHeader() throws IOException { +// return HeaderUtils.deserializeChunkHeaderV1(tsFileInput.wrapAsInputStream(), true); +// } +// +// /** +// * not thread safe. +// * +// * @param type given tsfile data type +// */ +// public PageHeader readPageHeader(TSDataType type) throws IOException { +// return HeaderUtils.deserializePageHeaderV1(tsFileInput.wrapAsInputStream(), type); +// } +// +// public ByteBuffer readPage(PageHeader header, CompressionType type) +// throws IOException { +// ByteBuffer buffer = readData(-1, header.getCompressedSize()); +// IUnCompressor unCompressor = IUnCompressor.getUnCompressor(type); +// ByteBuffer uncompressedBuffer = ByteBuffer.allocate(header.getUncompressedSize()); +// if (type == CompressionType.UNCOMPRESSED) { +// return buffer; +// } +// unCompressor.uncompress(buffer.array(), buffer.position(), buffer.remaining(), +// uncompressedBuffer.array(), +// 0); +// return uncompressedBuffer; +// } +// +// public ByteBuffer readCompressedPage(PageHeader header) throws IOException { +// return readData(-1, header.getCompressedSize()); +// } +// +// public long position() throws IOException { +// return tsFileInput.position(); +// } +// +// public void position(long offset) throws IOException { +// tsFileInput.position(offset); +// } +// +// /** +// * read one byte from the input. <br> this method is not thread safe +// */ +// public byte readMarker() throws IOException { +// markerBuffer.clear(); +// if (ReadWriteIOUtils.readAsPossible(tsFileInput, markerBuffer) == 0) { +// throw new IOException("reach the end of the file."); +// } +// markerBuffer.flip(); +// return markerBuffer.get(); +// } +// +// public byte readMarker(long position) throws IOException { +// return readData(position, Byte.BYTES).get(); +// } +// +// public void close() throws IOException { +// this.tsFileInput.close(); +// } +// +// public String getFileName() { +// return this.file; +// } +// +// /** +// * read data from tsFileInput, from the current position (if position = -1), or the given +// * position. <br> if position = -1, the tsFileInput's position will be changed to the current +// * position + real data size that been read. Other wise, the tsFileInput's position is not +// * changed. +// * +// * @param position the start position of data in the tsFileInput, or the current position if +// * position = -1 +// * @param size the size of data that want to read +// * @return data that been read. +// */ +// private ByteBuffer readData(long position, int size) throws IOException { +// ByteBuffer buffer = ByteBuffer.allocate(size); +// if (position == -1) { +// if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer) != size) { +// throw new IOException("reach the end of the data"); +// } +// } else { +// if (ReadWriteIOUtils.readAsPossible(tsFileInput, buffer, position, size) != size) { +// throw new IOException("reach the end of the data"); +// } +// } +// buffer.flip(); +// return buffer; +// } +// +// /** +// * upgrade file and resource +// * +// * @throws IOException, WriteProcessException +// */ +// @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning +// public void upgradeFile(List<TsFileResource> upgradedResources) +// throws IOException, WriteProcessException { +// File oldTsFile = FSFactoryProducer.getFSFactory().getFile(this.file); +// +// // check if the old TsFile has correct header +// if (!fileCheck(oldTsFile)) { +// return; +// } +// +// // ChunkGroupOffset -> version +// Map<Long, Long> oldVersionInfo = getVersionInfo(); +// +// // start to scan chunks and chunkGroups +// long startOffsetOfChunkGroup = 0; +// boolean newChunkGroup = true; +// long versionOfChunkGroup = 0; +// int chunkGroupCount = 0; +// List<List<PageHeader>> pageHeadersInChunkGroup = new ArrayList<>(); +// List<List<ByteBuffer>> pageDataInChunkGroup = new ArrayList<>(); +// List<List<Boolean>> pagePartitionInfoInChunkGroup = new ArrayList<>(); +// byte marker; +// List<MeasurementSchema> measurementSchemaList = new ArrayList<>(); +// try { +// while ((marker = this.readMarker()) != MetaMarker.SEPARATOR) { +// switch (marker) { +// case MetaMarker.CHUNK_HEADER: +// // this is the first chunk of a new ChunkGroup. +// if (newChunkGroup) { +// newChunkGroup = false; +// startOffsetOfChunkGroup = this.position() - 1; +// versionOfChunkGroup = oldVersionInfo.get(startOffsetOfChunkGroup); +// } +// ChunkHeader header = this.readChunkHeader(); +// MeasurementSchema measurementSchema = new MeasurementSchema(header.getMeasurementID(), +// header.getDataType(), +// header.getEncodingType(), +// header.getCompressionType()); +// measurementSchemaList.add(measurementSchema); +// List<PageHeader> pageHeadersInChunk = new ArrayList<>(); +// List<ByteBuffer> dataInChunk = new ArrayList<>(); +// List<Boolean> pagePartitionInfo = new ArrayList<>(); +// for (int j = 0; j < header.getNumOfPages(); j++) { +// PageHeader pageHeader = readPageHeader(header.getDataType()); +// boolean pageInSamePartition = checkIfPageInSameTimePartition(pageHeader); +// pagePartitionInfo.add(pageInSamePartition); +// ByteBuffer pageData = pageInSamePartition ? +// readCompressedPage(pageHeader) +// : readPage(pageHeader, header.getCompressionType()); +// pageHeadersInChunk.add(pageHeader); +// dataInChunk.add(pageData); +// } +// pageHeadersInChunkGroup.add(pageHeadersInChunk); +// pageDataInChunkGroup.add(dataInChunk); +// pagePartitionInfoInChunkGroup.add(pagePartitionInfo); +// break; +// case MetaMarker.CHUNK_GROUP_HEADER: +// // this is the footer of a ChunkGroup. +// ChunkGroupHeader chunkGroupHeader = this.readChunkGroupFooter(); +// String deviceID = chunkGroupHeader.getDeviceID(); +// rewrite(oldTsFile, deviceID, measurementSchemaList, pageHeadersInChunkGroup, +// pageDataInChunkGroup, versionOfChunkGroup, pagePartitionInfoInChunkGroup); +// +// pageHeadersInChunkGroup.clear(); +// pageDataInChunkGroup.clear(); +// measurementSchemaList.clear(); +// pagePartitionInfoInChunkGroup.clear(); +// newChunkGroup = true; +// chunkGroupCount++; +// break; +// +// default: +// // the disk file is corrupted, using this file may be dangerous +// logger.error("Unrecognized marker detected, this file may be corrupted"); +// return; +// } +// } +// // close upgraded tsFiles and generate resources for them +// for (TsFileIOWriter tsFileIOWriter : partitionWriterMap.values()) { +// upgradedResources.add(endFileAndGenerateResource(tsFileIOWriter)); +// } +// } catch (IOException e2) { +// logger.info("TsFile upgrade process cannot proceed at position {} after {} chunk groups " +// + "recovered, because : {}", this.position(), chunkGroupCount, e2.getMessage()); +// } finally { +// if (tsFileInput != null) { +// tsFileInput.close(); +// } +// } +// } +// +// private boolean checkIfPageInSameTimePartition(PageHeader pageHeader) { +// return StorageEngine.getTimePartition(pageHeader.getStartTime()) +// == StorageEngine.getTimePartition(pageHeader.getEndTime()); +// } +// +// /** +// * This method is for rewriting the ChunkGroup which data is in the different time partitions. In +// * this case, we have to decode the data to points, and then rewrite the data points to different +// * chunkWriters, finally write chunks to their own upgraded TsFiles +// */ +// private void rewrite(File oldTsFile, String deviceId, List<MeasurementSchema> schemas, +// List<List<PageHeader>> pageHeadersInChunkGroup, List<List<ByteBuffer>> dataInChunkGroup, +// long versionOfChunkGroup, List<List<Boolean>> pagePartitionInfoInChunkGroup) +// throws IOException, PageException { +// Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup = new HashMap<>(); +// for (int i = 0; i < schemas.size(); i++) { +// MeasurementSchema schema = schemas.get(i); +// List<ByteBuffer> pageDataInChunk = dataInChunkGroup.get(i); +// List<PageHeader> pageHeadersInChunk = pageHeadersInChunkGroup.get(i); +// List<Boolean> pagePartitionInfo = pagePartitionInfoInChunkGroup.get(i); +// valueDecoder = Decoder +// .getDecoderByType(schema.getEncodingType(), schema.getType()); +// for (int j = 0; j < pageDataInChunk.size(); j++) { +// if (Boolean.TRUE.equals(pagePartitionInfo.get(j))) { +// writePageInSamePartitionToFile(oldTsFile, schema, pageHeadersInChunk.get(j), +// pageDataInChunk.get(j), chunkWritersInChunkGroup); +// } else { +// writePageInDifferentPartitionsToFiles(oldTsFile, schema, pageDataInChunk.get(j), +// chunkWritersInChunkGroup); +// } +// } +// } +// +// for (Entry<Long, Map<MeasurementSchema, ChunkWriterImpl>> entry : chunkWritersInChunkGroup +// .entrySet()) { +// long partitionId = entry.getKey(); +// TsFileIOWriter tsFileIOWriter = partitionWriterMap.get(partitionId); +// tsFileIOWriter.startChunkGroup(deviceId); +// // write chunks to their own upgraded tsFiles +// for (IChunkWriter chunkWriter : entry.getValue().values()) { +// chunkWriter.writeToFileWriter(tsFileIOWriter); +// } +// tsFileIOWriter.endChunkGroup(); +// tsFileIOWriter.writeVersion(versionOfChunkGroup); +// } +// } +// +// private TsFileIOWriter getOrDefaultTsFileIOWriter(File oldTsFile, long partition) { +// return partitionWriterMap.computeIfAbsent(partition, k -> +// { +// File partitionDir = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent() +// + File.separator + partition); +// if (!partitionDir.exists()) { +// partitionDir.mkdirs(); +// } +// File newFile = FSFactoryProducer.getFSFactory().getFile(oldTsFile.getParent() +// + File.separator + partition + File.separator + oldTsFile.getName()); +// try { +// if (!newFile.createNewFile()) { +// logger.error("The TsFile {} has been created ", newFile); +// return null; +// } +// return new TsFileIOWriter(newFile); +// } catch (IOException e) { +// logger.error("Create new TsFile {} failed ", newFile); +// return null; +// } +// } +// ); +// } +// +// private void writePageInSamePartitionToFile(File oldTsFile, MeasurementSchema schema, +// PageHeader pageHeader, +// ByteBuffer pageData, +// Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup) +// throws PageException { +// long partitionId = StorageEngine.getTimePartition(pageHeader.getStartTime()); +// getOrDefaultTsFileIOWriter(oldTsFile, partitionId); +// Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup +// .getOrDefault(partitionId, new HashMap<>()); +// ChunkWriterImpl chunkWriter = chunkWriters +// .getOrDefault(schema, new ChunkWriterImpl(schema)); +// chunkWriter.writePageHeaderAndDataIntoBuff(pageData, pageHeader); +// chunkWriters.put(schema, chunkWriter); +// chunkWritersInChunkGroup.put(partitionId, chunkWriters); +// } +// +// private void writePageInDifferentPartitionsToFiles(File oldTsFile, MeasurementSchema schema, +// ByteBuffer pageData, +// Map<Long, Map<MeasurementSchema, ChunkWriterImpl>> chunkWritersInChunkGroup) +// throws IOException { +// valueDecoder.reset(); +// PageReader pageReader = new PageReader(pageData, schema.getType(), valueDecoder, +// defaultTimeDecoder, null); +// BatchData batchData = pageReader.getAllSatisfiedPageData(); +// while (batchData.hasCurrent()) { +// long time = batchData.currentTime(); +// Object value = batchData.currentValue(); +// long partitionId = StorageEngine.getTimePartition(time); +// +// Map<MeasurementSchema, ChunkWriterImpl> chunkWriters = chunkWritersInChunkGroup +// .getOrDefault(partitionId, new HashMap<>()); +// ChunkWriterImpl chunkWriter = chunkWriters +// .getOrDefault(schema, new ChunkWriterImpl(schema)); +// getOrDefaultTsFileIOWriter(oldTsFile, partitionId); +// switch (schema.getType()) { +// case INT32: +// chunkWriter.write(time, (int) value); +// break; +// case INT64: +// chunkWriter.write(time, (long) value); +// break; +// case FLOAT: +// chunkWriter.write(time, (float) value); +// break; +// case DOUBLE: +// chunkWriter.write(time, (double) value); +// break; +// case BOOLEAN: +// chunkWriter.write(time, (boolean) value); +// break; +// case TEXT: +// chunkWriter.write(time, (Binary) value); +// break; +// default: +// throw new UnSupportedDataTypeException( +// String.format("Data type %s is not supported.", schema.getType())); +// } +// batchData.next(); +// chunkWriters.put(schema, chunkWriter); +// chunkWritersInChunkGroup.put(partitionId, chunkWriters); +// } +// } +// +// /** +// * check if the file to be upgraded has correct magic strings and version number +// */ +// private boolean fileCheck(File oldTsFile) throws IOException { +// long fileSize; +// if (!oldTsFile.exists()) { +// logger.error("the file to be updated does not exist, file path: {}", oldTsFile.getPath()); +// return false; +// } else { +// fileSize = oldTsFile.length(); +// } +// +// String magic = readHeadMagic(true); +// if (!magic.equals(TSFileConfig.MAGIC_STRING)) { +// logger.error("the file's MAGIC STRING is incorrect, file path: {}", oldTsFile.getPath()); +// return false; +// } +// +// String versionNumber = readVersionNumber(); +// if (!versionNumber.equals(TSFileConfig.VERSION_NUMBER_V1)) { +// logger.error("the file's Version Number is incorrect, file path: {}", oldTsFile.getPath()); +// return false; +// } +// +// if (fileSize == TSFileConfig.MAGIC_STRING.length()) { +// logger.error("the file only contains magic string, file path: {}", oldTsFile.getPath()); +// return false; +// } else if (!readTailMagic().equals(TSFileConfig.MAGIC_STRING)) { +// logger.error("the file cannot upgrade, file path: {}", oldTsFile.getPath()); +// return false; +// } +// return true; +// } +// +// private Map<Long, Long> getVersionInfo() throws IOException { +// Map<Long, Long> versionInfo = new HashMap<>(); +// TsFileMetadataV1 fileMetadata = readFileMetadata(); +// List<TsDeviceMetadataV1> oldDeviceMetadataList = new ArrayList<>(); +// for (TsDeviceMetadataIndexV1 index : fileMetadata.getDeviceMap().values()) { +// TsDeviceMetadataV1 oldDeviceMetadata = readTsDeviceMetaData(index); +// oldDeviceMetadataList.add(oldDeviceMetadata); +// } +// +// for (TsDeviceMetadataV1 oldTsDeviceMetadata : oldDeviceMetadataList) { +// for (ChunkGroupMetaDataV1 oldChunkGroupMetadata : oldTsDeviceMetadata +// .getChunkGroupMetaDataList()) { +// long version = oldChunkGroupMetadata.getVersion(); +// long offsetOfChunkGroup = oldChunkGroupMetadata.getStartOffsetOfChunkGroup(); +// // get version informations +// versionInfo.put(offsetOfChunkGroup, version); +// } +// } +// return versionInfo; +// } +// +// private TsFileResource endFileAndGenerateResource(TsFileIOWriter tsFileIOWriter) +// throws IOException { +// tsFileIOWriter.endFile(); +// TsFileResource tsFileResource = new TsFileResource(tsFileIOWriter.getFile()); +// Map<String, List<TimeseriesMetadata>> deviceTimeseriesMetadataMap = tsFileIOWriter +// .getDeviceTimeseriesMetadataMap(); +// for (Map.Entry<String, List<TimeseriesMetadata>> entry : deviceTimeseriesMetadataMap +// .entrySet()) { +// String device = entry.getKey(); +// for (TimeseriesMetadata timeseriesMetaData : entry.getValue()) { +// tsFileResource.updateStartTime(device, timeseriesMetaData.getStatistics().getStartTime()); +// tsFileResource.updateEndTime(device, timeseriesMetaData.getStatistics().getEndTime()); +// } +// } +// tsFileResource.setClosed(true); +// return tsFileResource; +// } +// +//} \ No newline at end of file diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java deleted file mode 100644 index 1506d27..0000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/common/EndianType.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.tsfile.encoding.common; - -/** - * In current version, we only support BIG_ENDIAN mode. - */ -public enum EndianType { - BIG_ENDIAN, LITTLE_ENDIAN -} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java index 6bffedd..0e43cf6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java @@ -22,8 +22,6 @@ package org.apache.iotdb.tsfile.encoding.decoder; import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; - -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; @@ -50,14 +48,14 @@ public abstract class Decoder { public static Decoder getDecoderByType(TSEncoding encoding, TSDataType dataType) { switch (encoding) { case PLAIN: - return new PlainDecoder(EndianType.BIG_ENDIAN); + return new PlainDecoder(); case RLE: switch (dataType) { case BOOLEAN: case INT32: - return new IntRleDecoder(EndianType.BIG_ENDIAN); + return new IntRleDecoder(); case INT64: - return new LongRleDecoder(EndianType.BIG_ENDIAN); + return new LongRleDecoder(); case FLOAT: case DOUBLE: return new FloatDecoder(TSEncoding.valueOf(encoding.toString()), dataType); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java index eaaf729..ce0bc2a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatDecoder.java @@ -21,17 +21,14 @@ package org.apache.iotdb.tsfile.encoding.decoder; import java.io.IOException; import java.nio.ByteBuffer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.encoding.encoder.FloatEncoder; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Decoder for float or double value using rle or two diff. For more info about @@ -58,10 +55,10 @@ public class FloatDecoder extends Decoder { super(encodingType); if (encodingType == TSEncoding.RLE) { if (dataType == TSDataType.FLOAT) { - decoder = new IntRleDecoder(EndianType.BIG_ENDIAN); + decoder = new IntRleDecoder(); logger.debug("tsfile-encoding FloatDecoder: init decoder using int-rle and float"); } else if (dataType == TSDataType.DOUBLE) { - decoder = new LongRleDecoder(EndianType.BIG_ENDIAN); + decoder = new LongRleDecoder(); logger.debug("tsfile-encoding FloatDecoder: init decoder using long-rle and double"); } else { throw new TsFileDecodingException(String.format("data type %s is not supported by FloatDecoder", dataType)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java index a41512f..0166446 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoder.java @@ -21,15 +21,12 @@ package org.apache.iotdb.tsfile.encoding.decoder; import java.io.IOException; import java.nio.ByteBuffer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.encoding.bitpacking.IntPacker; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Decoder for int value using rle or bit-packing. @@ -53,8 +50,8 @@ public class IntRleDecoder extends RleDecoder { */ private IntPacker packer; - public IntRleDecoder(EndianType endianType) { - super(endianType); + public IntRleDecoder() { + super(); currentValue = 0; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java index 3f8da99..de27c0c 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoder.java @@ -21,15 +21,12 @@ package org.apache.iotdb.tsfile.encoding.decoder; import java.io.IOException; import java.nio.ByteBuffer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.encoding.bitpacking.LongPacker; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Decoder for long value using rle or bit-packing. @@ -53,8 +50,8 @@ public class LongRleDecoder extends RleDecoder { */ private LongPacker packer; - public LongRleDecoder(EndianType endianType) { - super(endianType); + public LongRleDecoder() { + super(); currentValue = 0; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java index 1fd15c1..8c31cff 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PlainDecoder.java @@ -22,31 +22,16 @@ package org.apache.iotdb.tsfile.encoding.decoder; import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; public class PlainDecoder extends Decoder { - private static final Logger logger = LoggerFactory.getLogger(PlainDecoder.class); - private EndianType endianType; - - public EndianType getEndianType() { - return endianType; - } - - public void setEndianType(EndianType endianType) { - this.endianType = endianType; - } - public PlainDecoder(EndianType endianType) { + public PlainDecoder() { super(TSEncoding.PLAIN); - this.endianType = endianType; } @Override @@ -61,7 +46,7 @@ public class PlainDecoder extends Decoder { @Override public int readInt(ByteBuffer buffer) { - return buffer.getInt(); + return ReadWriteForEncodingUtils.readVarInt(buffer); } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java index 8606537..bb5e703 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/RleDecoder.java @@ -22,10 +22,8 @@ package org.apache.iotdb.tsfile.encoding.decoder; import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; @@ -40,16 +38,6 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; */ public abstract class RleDecoder extends Decoder { - private EndianType endianType; - - public EndianType getEndianType() { - return endianType; - } - - public void setEndianType(EndianType endianType) { - this.endianType = endianType; - } - protected TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); /** * mode to indicate current encoding type 0 - RLE 1 - BIT_PACKED. @@ -87,9 +75,8 @@ public abstract class RleDecoder extends Decoder { * a constructor, init with endianType, default encoding is * <code>TSEncoding.RLE</code>. */ - public RleDecoder(EndianType endianType) { + public RleDecoder() { super(TSEncoding.RLE); - this.endianType = endianType; reset(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java index 873a4ba..1b4b558 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatEncoder.java @@ -21,8 +21,6 @@ package org.apache.iotdb.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; import java.io.IOException; - -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; @@ -66,9 +64,9 @@ public class FloatEncoder extends Encoder { isMaxPointNumberSaved = false; if (encodingType == TSEncoding.RLE) { if (dataType == TSDataType.FLOAT) { - encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); + encoder = new IntRleEncoder(); } else if (dataType == TSDataType.DOUBLE) { - encoder = new LongRleEncoder(EndianType.BIG_ENDIAN); + encoder = new LongRleEncoder(); } else { throw new TsFileEncodingException(String.format("data type %s is not supported by FloatEncoder", dataType)); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java index f3bb89a..9ed819e 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntRleEncoder.java @@ -22,10 +22,8 @@ package org.apache.iotdb.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.encoding.bitpacking.IntPacker; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; /** @@ -38,11 +36,11 @@ public class IntRleEncoder extends RleEncoder<Integer> { */ private IntPacker packer; - public IntRleEncoder(EndianType endianType) { - super(endianType); + public IntRleEncoder() { + super(); bufferedValues = new Integer[TSFileConfig.RLE_MIN_REPEATED_NUM]; preValue = 0; - values = new ArrayList<Integer>(); + values = new ArrayList<>(); } @Override diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java index dbbbfbb..f8a33f3 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongRleEncoder.java @@ -22,10 +22,8 @@ package org.apache.iotdb.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.encoding.bitpacking.LongPacker; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; /** @@ -41,8 +39,8 @@ public class LongRleEncoder extends RleEncoder<Long> { /** * Constructor of LongRleEncoder. */ - public LongRleEncoder(EndianType endianType) { - super(endianType); + public LongRleEncoder() { + super(); bufferedValues = new Long[TSFileConfig.RLE_MIN_REPEATED_NUM]; preValue = (long) 0; values = new ArrayList<Long>(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java index d30bcc8..50bff74 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PlainEncoder.java @@ -22,40 +22,27 @@ package org.apache.iotdb.tsfile.encoding.encoder; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; - -import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; +import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PlainEncoder extends Encoder { private static final Logger logger = LoggerFactory.getLogger(PlainEncoder.class); - private EndianType endianType; private TSDataType dataType; private int maxStringLength; - public PlainEncoder(EndianType endianType, TSDataType dataType, int maxStringLength) { + public PlainEncoder(TSDataType dataType, int maxStringLength) { super(TSEncoding.PLAIN); - this.endianType = endianType; this.dataType = dataType; this.maxStringLength = maxStringLength; } - public void setEndianType(EndianType endianType) { - this.endianType = endianType; - } - - public EndianType getEndianType() { - return endianType; - } - @Override public void encode(boolean value, ByteArrayOutputStream out) { if (value) { @@ -67,40 +54,19 @@ public class PlainEncoder extends Encoder { @Override public void encode(short value, ByteArrayOutputStream out) { - if (this.endianType == EndianType.LITTLE_ENDIAN) { - out.write(value & 0xFF); - out.write((value >> 8) & 0xFF); - } else if (this.endianType == EndianType.BIG_ENDIAN) { - out.write((value >> 8) & 0xFF); - out.write(value & 0xFF); - } + out.write((value >> 8) & 0xFF); + out.write(value & 0xFF); } @Override public void encode(int value, ByteArrayOutputStream out) { - if (this.endianType == EndianType.LITTLE_ENDIAN) { - out.write(value & 0xFF); - out.write((value >> 8) & 0xFF); - out.write((value >> 16) & 0xFF); - out.write((value >> 24) & 0xFF); - } else if (this.endianType == EndianType.BIG_ENDIAN) { - out.write((value >> 24) & 0xFF); - out.write((value >> 16) & 0xFF); - out.write((value >> 8) & 0xFF); - out.write(value & 0xFF); - } + ReadWriteForEncodingUtils.writeVarInt(value, out); } @Override public void encode(long value, ByteArrayOutputStream out) { - if (this.endianType == EndianType.LITTLE_ENDIAN) { - for (int i = 0; i < 8; i++) { - out.write((byte) (((value) >> (i * 8)) & 0xFF)); - } - } else if (this.endianType == EndianType.BIG_ENDIAN) { - for (int i = 7; i >= 0; i--) { - out.write((byte) (((value) >> (i * 8)) & 0xFF)); - } + for (int i = 7; i >= 0; i--) { + out.write((byte) (((value) >> (i * 8)) & 0xFF)); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java index 51003ce..51ad831 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RleEncoder.java @@ -24,17 +24,14 @@ import java.io.IOException; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Encodes values using a combination of run length encoding and bit packing, @@ -66,16 +63,6 @@ public abstract class RleEncoder<T extends Comparable<T>> extends Encoder { private static final Logger logger = LoggerFactory.getLogger(RleEncoder.class); - private EndianType endianType; - - public EndianType getEndianType() { - return endianType; - } - - public void setEndianType(EndianType endianType) { - this.endianType = endianType; - } - /** * we save all value in a list and calculate its bitwidth. */ @@ -134,9 +121,8 @@ public abstract class RleEncoder<T extends Comparable<T>> extends Encoder { /** * constructor. */ - public RleEncoder(EndianType endianType) { + public RleEncoder() { super(TSEncoding.RLE); - this.endianType = endianType; bytesBuffer = new ArrayList<>(); isBitPackRun = false; isBitWidthSaved = false; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java index 0f6cd7c..8fd3cfa 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java @@ -20,17 +20,14 @@ package org.apache.iotdb.tsfile.encoding.encoder; import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Each subclass of TSEncodingBuilder responds a enumerate value in {@linkplain TSEncoding @@ -105,7 +102,7 @@ public abstract class TSEncodingBuilder { @Override public Encoder getEncoder(TSDataType type) { - return new PlainEncoder(EndianType.BIG_ENDIAN, type, maxStringLength); + return new PlainEncoder(type, maxStringLength); } @Override @@ -137,9 +134,9 @@ public abstract class TSEncodingBuilder { switch (type) { case INT32: case BOOLEAN: - return new IntRleEncoder(EndianType.BIG_ENDIAN); + return new IntRleEncoder(); case INT64: - return new LongRleEncoder(EndianType.BIG_ENDIAN); + return new LongRleEncoder(); case FLOAT: case DOUBLE: return new FloatEncoder(TSEncoding.RLE, type, maxPointNumber); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java index fd3e50a..a966ba4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java @@ -659,7 +659,7 @@ public class TsFileSequenceReader implements AutoCloseable { * @return a CHUNK_GROUP_FOOTER * @throws IOException io error */ - public ChunkGroupHeader readChunkGroupFooter() throws IOException { + public ChunkGroupHeader readChunkGroupHeader() throws IOException { return ChunkGroupHeader.deserializeFrom(tsFileInput.wrapAsInputStream(), true); } @@ -671,7 +671,7 @@ public class TsFileSequenceReader implements AutoCloseable { * @return a CHUNK_GROUP_FOOTER * @throws IOException io error */ - public ChunkGroupHeader readChunkGroupFooter(long position, boolean markerRead) + public ChunkGroupHeader readChunkGroupHeader(long position, boolean markerRead) throws IOException { return ChunkGroupHeader.deserializeFrom(tsFileInput, position, markerRead); } @@ -968,7 +968,7 @@ public class TsFileSequenceReader implements AutoCloseable { // this is a chunk group // if there is something wrong with the ChunkGroup Footer, we will drop this ChunkGroup // because we can not guarantee the correctness of the deviceId. - ChunkGroupHeader chunkGroupHeader = this.readChunkGroupFooter(); + ChunkGroupHeader chunkGroupHeader = this.readChunkGroupHeader(); deviceID = chunkGroupHeader.getDeviceID(); if (newSchema != null) { for (MeasurementSchema tsSchema : measurementSchemaList) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java index 57549a4..f747926 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java @@ -33,7 +33,6 @@ import org.apache.iotdb.tsfile.read.reader.IPageReader; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.reader.IChunkReader; import org.apache.iotdb.tsfile.read.reader.page.PageReader; -import org.apache.iotdb.tsfile.v1.file.utils.HeaderUtils; import java.io.IOException; import java.nio.ByteBuffer; @@ -92,8 +91,7 @@ public class ChunkReader implements IChunkReader { // construct next satisfied page header while (chunkDataBuffer.remaining() > 0) { // deserialize a PageHeader from chunkDataBuffer - PageHeader pageHeader = isFromOldTsFile ? HeaderUtils.deserializePageHeaderV1(chunkDataBuffer, chunkHeader.getDataType()) : - PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); + PageHeader pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); // if the current page satisfies if (pageSatisfied(pageHeader)) { pageReaderList.add(constructPageReaderForNextPage(pageHeader)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java index 52052d4..5dbd171 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java @@ -91,7 +91,6 @@ public class RestorableTsFileIOWriter extends TsFileIOWriter { try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) { truncatedSize = reader.selfCheck(knownSchemas, chunkGroupMetadataList, versionInfo, true); - totalChunkNum = reader.getTotalChunkNum(); if (truncatedSize == TsFileCheckStatus.COMPLETE_FILE) { crashed = false; canWrite = false; diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java index bed4ee9..95b413d 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/IntRleDecoderTest.java @@ -26,30 +26,22 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import org.apache.iotdb.tsfile.encoding.common.EndianType; -import org.apache.iotdb.tsfile.encoding.decoder.IntRleDecoder; -import org.apache.iotdb.tsfile.encoding.decoder.RleDecoder; import org.apache.iotdb.tsfile.encoding.encoder.IntRleEncoder; import org.apache.iotdb.tsfile.encoding.encoder.RleEncoder; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class IntRleDecoderTest { private List<Integer> rleList; private List<Integer> bpList; private List<Integer> hybridList; - private int rleBitWidth; - private int bpBitWidth; - private int hybridWidth; @Before public void setUp() throws Exception { - rleList = new ArrayList<Integer>(); + rleList = new ArrayList<>(); int rleCount = 11; int rleNum = 18; int rleStart = 11; @@ -63,9 +55,8 @@ public class IntRleDecoderTest { rleCount += 2; rleStart *= -3; } - rleBitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(rleList); - bpList = new ArrayList<Integer>(); + bpList = new ArrayList<>(); int bpCount = 100000; int bpStart = 11; for (int i = 0; i < bpCount; i++) { @@ -76,9 +67,8 @@ public class IntRleDecoderTest { bpList.add(bpStart); } } - bpBitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(bpList); - hybridList = new ArrayList<Integer>(); + hybridList = new ArrayList<>(); int hybridCount = 11; int hybridNum = 1000; int hybridStart = 20; @@ -101,7 +91,6 @@ public class IntRleDecoderTest { } hybridCount += 2; } - hybridWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(hybridList); } @After @@ -114,17 +103,16 @@ public class IntRleDecoderTest { for (int i = 7000000; i < 10000000; i++) { list.add(i); } - int width = ReadWriteForEncodingUtils.getIntMaxBitWidth(list); - testLength(list, width, false, 1); + testLength(list, false, 1); for (int i = 1; i < 10; i++) { - testLength(list, width, false, i); + testLength(list, false, i); } } @Test public void testRleReadInt() throws IOException { for (int i = 1; i < 10; i++) { - testLength(rleList, rleBitWidth, false, i); + testLength(rleList, false, i); } } @@ -144,30 +132,29 @@ public class IntRleDecoderTest { rleCount *= 7; rleStart *= -3; } - int bitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(repeatList); for (int i = 1; i < 10; i++) { - testLength(repeatList, bitWidth, false, i); + testLength(repeatList, false, i); } } @Test public void testBitPackingReadInt() throws IOException { for (int i = 1; i < 10; i++) { - testLength(bpList, bpBitWidth, false, i); + testLength(bpList, false, i); } } @Test public void testHybridReadInt() throws IOException { for (int i = 1; i < 3; i++) { - testLength(hybridList, hybridWidth, false, i); + testLength(hybridList, false, i); } } @Test public void testHybridReadBoolean() throws IOException { for (int i = 1; i < 10; i++) { - testLength(hybridList, hybridWidth, false, i); + testLength(hybridList, false, i); } } @@ -178,44 +165,10 @@ public class IntRleDecoderTest { } } - public void testBooleanLength(List<Integer> list, int bitWidth, boolean isDebug, int repeatCount) + public void testLength(List<Integer> list, boolean isDebug, int repeatCount) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RleEncoder<Integer> encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); - for (int i = 0; i < repeatCount; i++) { - for (int value : list) { - if (value % 2 == 0) { - encoder.encode(false, baos); - } else { - encoder.encode(true, baos); - } - - } - encoder.flush(baos); - } - - ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - RleDecoder decoder = new IntRleDecoder(EndianType.BIG_ENDIAN); - for (int i = 0; i < repeatCount; i++) { - for (int value : list) { - boolean value_ = decoder.readBoolean(buffer); - if (isDebug) { - System.out.println(value_ + "/" + value); - } - if (value % 2 == 0) { - assertEquals(false, value_); - } else { - assertEquals(true, value_); - } - - } - } - } - - public void testLength(List<Integer> list, int bitWidth, boolean isDebug, int repeatCount) - throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RleEncoder<Integer> encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); + RleEncoder<Integer> encoder = new IntRleEncoder(); for (int i = 0; i < repeatCount; i++) { for (int value : list) { encoder.encode(value, baos); @@ -224,7 +177,7 @@ public class IntRleDecoderTest { } ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - RleDecoder decoder = new IntRleDecoder(EndianType.BIG_ENDIAN); + RleDecoder decoder = new IntRleDecoder(); for (int i = 0; i < repeatCount; i++) { for (int value : list) { int value_ = decoder.readInt(buffer); @@ -237,14 +190,14 @@ public class IntRleDecoderTest { } private void testBitPackedReadHeader(int num) throws IOException { - List<Integer> list = new ArrayList<Integer>(); + List<Integer> list = new ArrayList<>(); for (int i = 0; i < num; i++) { list.add(i); } ByteArrayOutputStream baos = new ByteArrayOutputStream(); int bitWidth = ReadWriteForEncodingUtils.getIntMaxBitWidth(list); - RleEncoder<Integer> encoder = new IntRleEncoder(EndianType.BIG_ENDIAN); + RleEncoder<Integer> encoder = new IntRleEncoder(); for (int value : list) { encoder.encode(value, baos); } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java index ad41ac3..153112d 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/LongRleDecoderTest.java @@ -26,17 +26,12 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import org.apache.iotdb.tsfile.encoding.common.EndianType; -import org.apache.iotdb.tsfile.encoding.decoder.LongRleDecoder; -import org.apache.iotdb.tsfile.encoding.decoder.RleDecoder; import org.apache.iotdb.tsfile.encoding.encoder.LongRleEncoder; import org.apache.iotdb.tsfile.encoding.encoder.RleEncoder; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class LongRleDecoderTest { @@ -184,7 +179,7 @@ public class LongRleDecoderTest { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int bitWidth = ReadWriteForEncodingUtils.getLongMaxBitWidth(list); - RleEncoder<Long> encoder = new LongRleEncoder(EndianType.BIG_ENDIAN); + RleEncoder<Long> encoder = new LongRleEncoder(); for (long value : list) { encoder.encode(value, baos); } @@ -206,7 +201,7 @@ public class LongRleDecoderTest { public void testLength(List<Long> list, int bitWidth, boolean isDebug, int repeatCount) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - RleEncoder<Long> encoder = new LongRleEncoder(EndianType.BIG_ENDIAN); + RleEncoder<Long> encoder = new LongRleEncoder(); for (int i = 0; i < repeatCount; i++) { for (long value : list) { encoder.encode(value, baos); @@ -215,7 +210,7 @@ public class LongRleDecoderTest { } ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - RleDecoder decoder = new LongRleDecoder(EndianType.BIG_ENDIAN); + RleDecoder decoder = new LongRleDecoder(); for (int i = 0; i < repeatCount; i++) { for (long value : list) { long value_ = decoder.readLong(buffer); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java index 0a0131c..4bcbe5b 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/DoubleStatisticsTest.java @@ -18,11 +18,11 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import org.junit.Test; + public class DoubleStatisticsTest { private static final double maxError = 0.0001d; @@ -36,7 +36,7 @@ public class DoubleStatisticsTest { assertFalse(doubleStats.isEmpty()); assertEquals(2.32d, doubleStats.getMaxValue(), maxError); assertEquals(1.34d, doubleStats.getMinValue(), maxError); - assertEquals(2.32d + 1.34d, doubleStats.getSumValue(), maxError); + assertEquals(2.32d + 1.34d, doubleStats.getSumDoubleValue(), maxError); assertEquals(1.34d, doubleStats.getFirstValue(), maxError); assertEquals(2.32d, doubleStats.getLastValue(), maxError); } @@ -60,14 +60,14 @@ public class DoubleStatisticsTest { assertFalse(doubleStats3.isEmpty()); assertEquals(100.13453d, doubleStats3.getMaxValue(), maxError); assertEquals(1.34d, doubleStats3.getMinValue(), maxError); - assertEquals(100.13453d + 1.34d, doubleStats3.getSumValue(), maxError); + assertEquals(100.13453d + 1.34d, doubleStats3.getSumDoubleValue(), maxError); assertEquals(1.34d, doubleStats3.getFirstValue(), maxError); assertEquals(100.13453d, doubleStats3.getLastValue(), maxError); doubleStats3.mergeStatistics(doubleStats2); assertEquals(200.435d, doubleStats3.getMaxValue(), maxError); assertEquals(1.34d, doubleStats3.getMinValue(), maxError); - assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSumValue(), maxError); + assertEquals(100.13453d + 1.34d + 200.435d, doubleStats3.getSumDoubleValue(), maxError); assertEquals(1.34d, doubleStats3.getFirstValue(), maxError); assertEquals(200.435d, doubleStats3.getLastValue(), maxError); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java index 8bd49c3..2867a23 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/FloatStatisticsTest.java @@ -18,11 +18,11 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import org.junit.Test; + public class FloatStatisticsTest { private static final float maxError = 0.0001f; @@ -36,7 +36,7 @@ public class FloatStatisticsTest { assertFalse(floatStats.isEmpty()); assertEquals(2.32f, (double) floatStats.getMaxValue(), maxError); assertEquals(1.34f, (double) floatStats.getMinValue(), maxError); - assertEquals(2.32f + 1.34f, (double) floatStats.getSumValue(), maxError); + assertEquals(2.32f + 1.34f, (double) floatStats.getSumDoubleValue(), maxError); assertEquals(1.34f, (double) floatStats.getFirstValue(), maxError); assertEquals(2.32f, (double) floatStats.getLastValue(), maxError); } @@ -60,14 +60,14 @@ public class FloatStatisticsTest { assertFalse(floatStats3.isEmpty()); assertEquals(100.13453f, floatStats3.getMaxValue(), maxError); assertEquals(1.34f, floatStats3.getMinValue(), maxError); - assertEquals(100.13453f + 1.34f, (float) floatStats3.getSumValue(), maxError); + assertEquals(100.13453f + 1.34f, (float) floatStats3.getSumDoubleValue(), maxError); assertEquals(1.34f, floatStats3.getFirstValue(), maxError); assertEquals(100.13453f, floatStats3.getLastValue(), maxError); floatStats3.mergeStatistics(floatStats2); assertEquals(200.435f, floatStats3.getMaxValue(), maxError); assertEquals(1.34f, floatStats3.getMinValue(), maxError); - assertEquals(100.13453f + 1.34f + 200.435f, (float) floatStats3.getSumValue(), maxError); + assertEquals(100.13453f + 1.34f + 200.435f, (float) floatStats3.getSumDoubleValue(), maxError); assertEquals(1.34f, floatStats3.getFirstValue(), maxError); assertEquals(200.435f, floatStats3.getLastValue(), maxError); @@ -87,7 +87,6 @@ public class FloatStatisticsTest { assertEquals(122.34f, floatStats3.getFirstValue(), maxError); assertEquals(125.34f, floatStats3.getLastValue(), maxError); - floatStats3.mergeStatistics(floatStats5); assertEquals(122.34f, floatStats3.getFirstValue(), maxError); assertEquals(125.34f, floatStats3.getLastValue(), maxError); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java index 9de231e..469f7b1 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/IntegerStatisticsTest.java @@ -18,11 +18,11 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import org.junit.Test; + public class IntegerStatisticsTest { @Test @@ -35,7 +35,7 @@ public class IntegerStatisticsTest { assertEquals(2, (int) intStats.getMaxValue()); assertEquals(1, (int) intStats.getMinValue()); assertEquals(1, (int) intStats.getFirstValue()); - assertEquals(3, (int) intStats.getSumValue()); + assertEquals(3, (int) intStats.getSumLongValue()); assertEquals(2, (int) intStats.getLastValue()); } @@ -59,14 +59,14 @@ public class IntegerStatisticsTest { assertEquals(100, (int) intStats3.getMaxValue()); assertEquals(1, (int) intStats3.getMinValue()); assertEquals(1, (int) intStats3.getFirstValue()); - assertEquals(1 + 100, (int) intStats3.getSumValue()); + assertEquals(1 + 100, (int) intStats3.getSumLongValue()); assertEquals(100, (int) intStats3.getLastValue()); intStats3.mergeStatistics(intStats2); assertEquals(200, (int) intStats3.getMaxValue()); assertEquals(1, (int) intStats3.getMinValue()); assertEquals(1, (int) intStats3.getFirstValue()); - assertEquals(101 + 200, (int) intStats3.getSumValue()); + assertEquals(101 + 200, (int) intStats3.getSumLongValue()); assertEquals(200, (int) intStats3.getLastValue()); // Unseq merge diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java index 914d37f..4aa0c6f 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/statistics/LongStatisticsTest.java @@ -18,11 +18,14 @@ */ package org.apache.iotdb.tsfile.file.metadata.statistics; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import org.apache.iotdb.tsfile.exception.filter.StatisticsClassException; import org.junit.Test; -import static org.junit.Assert.*; - public class LongStatisticsTest { @Test @@ -38,7 +41,7 @@ public class LongStatisticsTest { assertEquals(secondValue, (long) longStats.getMaxValue()); assertEquals(firstValue, (long) longStats.getMinValue()); assertEquals(firstValue, (long) longStats.getFirstValue()); - assertEquals(firstValue + secondValue, (long) longStats.getSumValue()); + assertEquals(firstValue + secondValue, (long) longStats.getSumDoubleValue()); assertEquals(secondValue, (long) longStats.getLastValue()); } @@ -63,14 +66,14 @@ public class LongStatisticsTest { assertFalse(longStats3.isEmpty()); assertEquals(max1, (long) longStats3.getMaxValue()); assertEquals(1, (long) longStats3.getMinValue()); - assertEquals(max1 + 1, (long) longStats3.getSumValue()); + assertEquals(max1 + 1, (long) longStats3.getSumDoubleValue()); assertEquals(1, (long) longStats3.getFirstValue()); assertEquals(max1, (long) longStats3.getLastValue()); longStats3.mergeStatistics(longStats2); assertEquals(max2, (long) longStats3.getMaxValue()); assertEquals(1, (long) longStats3.getMinValue()); - assertEquals(max2 + max1 + 1, (long) longStats3.getSumValue()); + assertEquals(max2 + max1 + 1, (long) longStats3.getSumDoubleValue()); assertEquals(1, (long) longStats3.getFirstValue()); assertEquals(max2, (long) longStats3.getLastValue()); @@ -89,7 +92,7 @@ public class LongStatisticsTest { // if not merge, the min value will not be changed by smaller value in // intStats5 assertEquals(1, (long) longStats3.getMinValue()); - assertEquals(max2 + max1 + 1, (long) longStats3.getSumValue()); + assertEquals(max2 + max1 + 1, (long) longStats3.getSumDoubleValue()); assertEquals(1, (long) longStats3.getFirstValue()); assertEquals(max2, (long) longStats3.getLastValue()); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java index 4dff2b2..cdbd50f 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/file/metadata/utils/Utils.java @@ -29,6 +29,8 @@ import org.apache.iotdb.tsfile.file.header.PageHeader; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode; import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; +import org.apache.iotdb.tsfile.file.metadata.statistics.BooleanStatistics; +import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.junit.Assert; @@ -131,11 +133,11 @@ public class Utils { public static void isPageHeaderEqual(PageHeader header1, PageHeader header2) { if (Utils.isTwoObjectsNotNULL(header1, header2, "PageHeader")) { - assertTrue(header1.getUncompressedSize() == header2.getUncompressedSize()); - assertTrue(header1.getCompressedSize() == header2.getCompressedSize()); - assertTrue(header1.getNumOfValues() == header2.getNumOfValues()); - assertTrue(header1.getEndTime() == header2.getEndTime()); - assertTrue(header1.getStartTime() == header2.getStartTime()); + assertEquals(header1.getUncompressedSize(), header2.getUncompressedSize()); + assertEquals(header1.getCompressedSize(), header2.getCompressedSize()); + assertEquals(header1.getNumOfValues(), header2.getNumOfValues()); + assertEquals(header1.getEndTime(), header2.getEndTime()); + assertEquals(header1.getStartTime(), header2.getStartTime()); if (Utils .isTwoObjectsNotNULL(header1.getStatistics(), header2.getStatistics(), "statistics")) { Utils.isStatisticsEqual(header1.getStatistics(), header2.getStatistics()); @@ -144,21 +146,23 @@ public class Utils { } public static void isStatisticsEqual(Statistics statistics1, Statistics statistics2) { - if ((statistics1 == null) ^ (statistics2 == null)) { + if ((statistics1 == null) || (statistics2 == null)) { System.out.println("error"); fail("one of statistics is null"); } - if ((statistics1 != null) && (statistics2 != null)) { - if (statistics1.isEmpty() ^ statistics2.isEmpty()) { - fail("one of statistics is empty while the other one is not"); - } - if (!statistics1.isEmpty() && !statistics2.isEmpty()) { - assertEquals(statistics1.getMinValue(), statistics2.getMinValue()); - assertEquals(statistics1.getMaxValue(), statistics2.getMaxValue()); - assertEquals(statistics1.getFirstValue(), statistics2.getFirstValue()); - assertEquals(statistics1.getSumValue(), statistics2.getSumValue(), maxError); - assertEquals(statistics1.getLastValue(), statistics2.getLastValue()); + if (statistics1.isEmpty() || statistics2.isEmpty()) { + fail("one of statistics is empty while the other one is not"); + } + if (!statistics1.isEmpty() && !statistics2.isEmpty()) { + assertEquals(statistics1.getMinValue(), statistics2.getMinValue()); + assertEquals(statistics1.getMaxValue(), statistics2.getMaxValue()); + assertEquals(statistics1.getFirstValue(), statistics2.getFirstValue()); + if (statistics1 instanceof IntegerStatistics || statistics1 instanceof BooleanStatistics) { + assertEquals(statistics1.getSumLongValue(), statistics2.getSumLongValue()); + } else { + assertEquals(statistics1.getSumDoubleValue(), statistics2.getSumDoubleValue(), maxError); } + assertEquals(statistics1.getLastValue(), statistics2.getLastValue()); } } } \ No newline at end of file diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java index bb86168..c9931fc 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTest.java @@ -80,7 +80,7 @@ public class TsFileSequenceReaderTest { } break; case MetaMarker.CHUNK_GROUP_HEADER: - ChunkGroupHeader footer = reader.readChunkGroupFooter(); + ChunkGroupHeader footer = reader.readChunkGroupHeader(); long endOffset = reader.position(); Pair<Long, Long> pair = new Pair<>(startOffset, endOffset); deviceChunkGroupMetadataOffsets.putIfAbsent(footer.getDeviceID(), new ArrayList<>()); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java index 65d86e6..4f4a681 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderTest.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.encoding.decoder.DeltaBinaryDecoder; import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV1; @@ -54,11 +53,11 @@ public class PageReaderTest { public void testLong() { LoopWriteReadTest test = new LoopWriteReadTest("Test INT64", - new LongRleEncoder(EndianType.BIG_ENDIAN), - new LongRleDecoder(EndianType.BIG_ENDIAN), TSDataType.INT64, POINTS_COUNT_IN_ONE_PAGE) { + new LongRleEncoder(), + new LongRleDecoder(), TSDataType.INT64, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return Long.valueOf(Long.MAX_VALUE - i); + return Long.MAX_VALUE - i; } }; test.test(TSDataType.INT64); @@ -67,11 +66,11 @@ public class PageReaderTest { @Test public void testBoolean() { LoopWriteReadTest test = new LoopWriteReadTest("Test Boolean", - new IntRleEncoder(EndianType.BIG_ENDIAN), - new IntRleDecoder(EndianType.BIG_ENDIAN), TSDataType.BOOLEAN, POINTS_COUNT_IN_ONE_PAGE) { + new IntRleEncoder(), + new IntRleDecoder(), TSDataType.BOOLEAN, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return i % 3 == 0 ? true : false; + return i % 3 == 0; } }; test.test(TSDataType.BOOLEAN); @@ -79,12 +78,11 @@ public class PageReaderTest { @Test public void testInt() { - LoopWriteReadTest test = new LoopWriteReadTest("Test INT32", - new IntRleEncoder(EndianType.BIG_ENDIAN), - new IntRleDecoder(EndianType.BIG_ENDIAN), TSDataType.INT32, POINTS_COUNT_IN_ONE_PAGE) { + LoopWriteReadTest test = new LoopWriteReadTest("Test INT32", new IntRleEncoder(), + new IntRleDecoder(), TSDataType.INT32, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return Integer.valueOf(i); + return i; } }; test.test(TSDataType.INT32); @@ -96,7 +94,7 @@ public class PageReaderTest { new SinglePrecisionDecoderV1(), TSDataType.FLOAT, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return Float.valueOf(i) / 10 - Float.valueOf(i) / 100; + return (float) i / 10 - (float) i / 100; } }; test.test(TSDataType.FLOAT); @@ -105,7 +103,7 @@ public class PageReaderTest { new SinglePrecisionDecoderV1(), TSDataType.FLOAT, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return Float.valueOf(i) / 100 - Float.valueOf(i) / 10; + return (float) i / 100 - (float) i / 10; } }; test2.test(TSDataType.FLOAT); @@ -117,7 +115,7 @@ public class PageReaderTest { new DoublePrecisionDecoderV1(), TSDataType.DOUBLE, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return Double.valueOf(i) / 10 - Double.valueOf(i) / 100; + return (double) i / 10 - (double) i / 100; } }; test.test(TSDataType.DOUBLE); @@ -126,7 +124,7 @@ public class PageReaderTest { new DoublePrecisionDecoderV1(), TSDataType.DOUBLE, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return Double.valueOf(i) / 1000 - Double.valueOf(i) / 100; + return (double) i / 1000 - (double) i / 100; } }; test2.test(TSDataType.DOUBLE); @@ -135,11 +133,11 @@ public class PageReaderTest { @Test public void testBinary() { LoopWriteReadTest test = new LoopWriteReadTest("Test Double", - new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.TEXT, 1000), - new PlainDecoder(EndianType.BIG_ENDIAN), TSDataType.TEXT, POINTS_COUNT_IN_ONE_PAGE) { + new PlainEncoder(TSDataType.TEXT, 1000), + new PlainDecoder(), TSDataType.TEXT, POINTS_COUNT_IN_ONE_PAGE) { @Override public Object generateValueByIndex(int i) { - return new Binary(new StringBuilder("TEST TEXT").append(i).toString()); + return new Binary("TEST TEXT" + i); } }; test.test(TSDataType.TEXT); @@ -238,26 +236,26 @@ public class PageReaderTest { } } - private void writeData() throws IOException { + private void writeData() { for (int i = 0; i < count; i++) { switch (dataType) { case BOOLEAN: - pageWriter.write(Long.valueOf(i), (Boolean) generateValueByIndex(i)); + pageWriter.write(i, (Boolean) generateValueByIndex(i)); break; case INT32: - pageWriter.write(Long.valueOf(i), (Integer) generateValueByIndex(i)); + pageWriter.write(i, (Integer) generateValueByIndex(i)); break; case INT64: - pageWriter.write(Long.valueOf(i), (Long) generateValueByIndex(i)); + pageWriter.write(i, (Long) generateValueByIndex(i)); break; case FLOAT: - pageWriter.write(Long.valueOf(i), (Float) generateValueByIndex(i)); + pageWriter.write(i, (Float) generateValueByIndex(i)); break; case DOUBLE: - pageWriter.write(Long.valueOf(i), (Double) generateValueByIndex(i)); + pageWriter.write(i, (Double) generateValueByIndex(i)); break; case TEXT: - pageWriter.write(Long.valueOf(i), (Binary) generateValueByIndex(i)); + pageWriter.write(i, (Binary) generateValueByIndex(i)); break; } @@ -270,11 +268,11 @@ public class PageReaderTest { @Test public void testPageDelete() { LoopWriteReadTest test = new LoopWriteReadTest("Test INT64", - new LongRleEncoder(EndianType.BIG_ENDIAN), - new LongRleDecoder(EndianType.BIG_ENDIAN), TSDataType.INT64, 100) { + new LongRleEncoder(), + new LongRleDecoder(), TSDataType.INT64, 100) { @Override public Object generateValueByIndex(int i) { - return Long.valueOf(Long.MAX_VALUE - i); + return Long.MAX_VALUE - i; } }; test.testDelete(TSDataType.INT64); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java index 61e4786..675a77c 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIOWriterTest.java @@ -23,7 +23,7 @@ import java.io.IOException; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.constant.TestConstant; import org.apache.iotdb.tsfile.file.MetaMarker; -import org.apache.iotdb.tsfile.file.footer.ChunkGroupFooter; +import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader; import org.apache.iotdb.tsfile.file.header.ChunkHeader; import org.apache.iotdb.tsfile.file.metadata.TimeSeriesMetadataTest; import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata; @@ -87,18 +87,17 @@ public class TsFileIOWriterTest { Assert.assertEquals(TSFileConfig.VERSION_NUMBER, reader.readVersionNumber()); Assert.assertEquals(TSFileConfig.MAGIC_STRING, reader.readTailMagic()); + // chunk group header + Assert.assertEquals(MetaMarker.CHUNK_GROUP_HEADER, reader.readMarker()); + ChunkGroupHeader footer = reader.readChunkGroupHeader(); + Assert.assertEquals(deviceId, footer.getDeviceID()); + // chunk header - reader.position(TSFileConfig.MAGIC_STRING.getBytes().length + TSFileConfig.VERSION_NUMBER - .getBytes().length); - Assert.assertEquals(MetaMarker.CHUNK_HEADER, reader.readMarker()); - ChunkHeader header = reader.readChunkHeader(); + reader.position(TSFileConfig.MAGIC_STRING.getBytes().length + 1); + Assert.assertEquals(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, reader.readMarker()); + ChunkHeader header = reader.readChunkHeader(MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER); Assert.assertEquals(TimeSeriesMetadataTest.measurementUID, header.getMeasurementID()); - // chunk group footer - Assert.assertEquals(MetaMarker.CHUNK_GROUP_FOOTER, reader.readMarker()); - ChunkGroupFooter footer = reader.readChunkGroupFooter(); - Assert.assertEquals(deviceId, footer.getDeviceID()); - // separator Assert.assertEquals(MetaMarker.VERSION, reader.readMarker()); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java index 05cfc24..3253495 100755 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/writer/PageWriterTest.java @@ -18,8 +18,12 @@ */ package org.apache.iotdb.tsfile.write.writer; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.nio.ByteBuffer; import org.apache.iotdb.tsfile.constant.TestConstant; -import org.apache.iotdb.tsfile.encoding.common.EndianType; import org.apache.iotdb.tsfile.encoding.decoder.PlainDecoder; import org.apache.iotdb.tsfile.encoding.encoder.PlainEncoder; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -28,13 +32,6 @@ import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.iotdb.tsfile.write.page.PageWriter; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; -import org.apache.iotdb.tsfile.constant.TestConstant; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; import org.junit.Test; public class PageWriterTest { @@ -42,8 +39,8 @@ public class PageWriterTest { @Test public void testWriteInt() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.LITTLE_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT32, 0)); + writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder(TSDataType.INT32, 0)); writer.initStatistics(TSDataType.INT32); int value = 1; int timeCount = 0; @@ -58,12 +55,11 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.LITTLE_ENDIAN); + PlainDecoder decoder = new PlainDecoder(); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); } decoder.reset(); - decoder.setEndianType(EndianType.BIG_ENDIAN); assertEquals(value, decoder.readInt(buffer)); } catch (IOException e) { fail(); @@ -73,8 +69,8 @@ public class PageWriterTest { @Test public void testWriteLong() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); + writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder(TSDataType.INT64, 0)); writer.initStatistics(TSDataType.INT64); long value = 123142120391L; int timeCount = 0; @@ -89,7 +85,7 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN); + PlainDecoder decoder = new PlainDecoder(); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); } @@ -103,8 +99,8 @@ public class PageWriterTest { @Test public void testWriteFloat() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.FLOAT, 0)); + writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder(TSDataType.FLOAT, 0)); writer.initStatistics(TSDataType.FLOAT); float value = 2.2f; int timeCount = 0; @@ -119,7 +115,7 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN); + PlainDecoder decoder = new PlainDecoder(); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); } @@ -133,8 +129,8 @@ public class PageWriterTest { @Test public void testWriteBoolean() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.BOOLEAN, 0)); + writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder( TSDataType.BOOLEAN, 0)); writer.initStatistics(TSDataType.BOOLEAN); boolean value = false; int timeCount = 0; @@ -149,7 +145,7 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN); + PlainDecoder decoder = new PlainDecoder(); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); } @@ -162,8 +158,8 @@ public class PageWriterTest { @Test public void testWriteBinary() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.TEXT, 0)); + writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder(TSDataType.TEXT, 0)); writer.initStatistics(TSDataType.TEXT); String value = "I have a dream"; int timeCount = 0; @@ -178,7 +174,7 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN); + PlainDecoder decoder = new PlainDecoder(); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); } @@ -192,8 +188,8 @@ public class PageWriterTest { @Test public void testWriteDouble() { PageWriter writer = new PageWriter(); - writer.setTimeEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.INT64, 0)); - writer.setValueEncoder(new PlainEncoder(EndianType.BIG_ENDIAN, TSDataType.DOUBLE, 0)); + writer.setTimeEncoder(new PlainEncoder(TSDataType.INT64, 0)); + writer.setValueEncoder(new PlainEncoder(TSDataType.DOUBLE, 0)); writer.initStatistics(TSDataType.DOUBLE); double value = 1d; int timeCount = 0; @@ -208,7 +204,7 @@ public class PageWriterTest { byte[] timeBytes = new byte[timeSize]; buffer.get(timeBytes); ByteBuffer buffer2 = ByteBuffer.wrap(timeBytes); - PlainDecoder decoder = new PlainDecoder(EndianType.BIG_ENDIAN); + PlainDecoder decoder = new PlainDecoder(); for (int i = 0; i < timeCount; i++) { assertEquals(i, decoder.readLong(buffer2)); }
