This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch h/TableModelWrite in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 240bd93b8a97dd8c157f88693ae082480fffdf1c Author: HTHou <[email protected]> AuthorDate: Wed Apr 17 18:02:40 2024 +0800 TsFileResource --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../RepairUnsortedFileCompactionPerformer.java | 4 +- .../compaction/repair/RepairDataFileScanUtil.java | 11 +- .../estimator/AbstractCompactionEstimator.java | 19 ++- .../selector/utils/TsFileResourceCandidate.java | 4 +- .../dataregion/tsfile/TsFileResource.java | 16 +- ...iceTimeIndex.java => ArrayDeviceTimeIndex.java} | 84 +++++----- .../dataregion/tsfile/timeindex/FileTimeIndex.java | 10 +- .../dataregion/tsfile/timeindex/ITimeIndex.java | 3 +- .../tsfile/timeindex/PlainDeviceTimeIndex.java | 183 +++++++++++++++++++++ .../tsfile/timeindex/TimeIndexLevel.java | 15 +- .../dataregion/utils/TsFileResourceUtils.java | 10 +- .../dataregion/LastFlushTimeMapTest.java | 8 +- .../TsFileResourceProgressIndexTest.java | 5 +- .../compaction/CompactionValidationTest.java | 18 +- .../InsertionCrossSpaceCompactionRecoverTest.java | 4 +- .../InsertionCrossSpaceCompactionSelectorTest.java | 4 +- .../dataregion/tsfile/TsFileResourceTest.java | 16 +- .../rescon/memory/ResourceManagerTest.java | 22 +-- 19 files changed, 317 insertions(+), 121 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index df068dc3420..ffc8f8a97b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -770,7 +770,7 @@ public class IoTDBConfig { * Level of TimeIndex, which records the start time and end time of TsFileResource. Currently, * DEVICE_TIME_INDEX and FILE_TIME_INDEX are supported, and could not be changed after first set. */ - private TimeIndexLevel timeIndexLevel = TimeIndexLevel.DEVICE_TIME_INDEX; + private TimeIndexLevel timeIndexLevel = TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX; // just for test // wait for 60 second by default. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java index 163987ce434..c9f61a02cbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java @@ -22,7 +22,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performe import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.RepairUnsortedFileCompactionWriter; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import java.io.File; @@ -63,7 +63,7 @@ public class RepairUnsortedFileCompactionPerformer extends ReadPointCompactionPe TsFileResource targetFile = targetFiles.get(0); Files.createLink(targetFile.getTsFile().toPath(), seqSourceFile.getTsFile().toPath()); ITimeIndex timeIndex = seqSourceFile.getTimeIndex(); - if (timeIndex instanceof DeviceTimeIndex) { + if (timeIndex instanceof ArrayDeviceTimeIndex) { targetFile.setTimeIndex(timeIndex); } else { targetFile.setTimeIndex(seqSourceFile.buildDeviceTimeIndex()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index b6a0dd5033d..1451691ea80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionChunkReader; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.compress.IUnCompressor; @@ -228,7 +228,7 @@ public class RepairDataFileScanUtil { || resource.getStatus() == TsFileResourceStatus.DELETED) { continue; } - DeviceTimeIndex deviceTimeIndex; + ArrayDeviceTimeIndex deviceTimeIndex; try { deviceTimeIndex = getDeviceTimeIndex(resource); } catch (Exception ignored) { @@ -263,10 +263,11 @@ public class RepairDataFileScanUtil { return overlapResources; } - private static DeviceTimeIndex getDeviceTimeIndex(TsFileResource resource) throws IOException { + private static ArrayDeviceTimeIndex getDeviceTimeIndex(TsFileResource resource) + throws IOException { ITimeIndex timeIndex = resource.getTimeIndex(); - if (timeIndex instanceof DeviceTimeIndex) { - return (DeviceTimeIndex) timeIndex; + if (timeIndex instanceof ArrayDeviceTimeIndex) { + return (ArrayDeviceTimeIndex) timeIndex; } return resource.buildDeviceTimeIndex(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java index e72f163a21c..2482d42e8b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java @@ -24,7 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; @@ -55,7 +55,7 @@ public abstract class AbstractCompactionEstimator { new LRUMap<>( IoTDBDescriptor.getInstance().getConfig().getGlobalCompactionFileInfoCacheSize())); protected Map<TsFileResource, FileInfo> fileInfoCache = new HashMap<>(); - protected Map<TsFileResource, DeviceTimeIndex> deviceTimeIndexCache = new HashMap<>(); + protected Map<TsFileResource, ArrayDeviceTimeIndex> deviceTimeIndexCache = new HashMap<>(); protected IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -111,15 +111,15 @@ public abstract class AbstractCompactionEstimator { protected int calculatingMaxOverlapFileNumInSubCompactionTask(List<TsFileResource> resources) throws IOException { Set<IDeviceID> devices = new HashSet<>(); - List<DeviceTimeIndex> resourceDevices = new ArrayList<>(resources.size()); + List<ArrayDeviceTimeIndex> resourceDevices = new ArrayList<>(resources.size()); for (TsFileResource resource : resources) { - DeviceTimeIndex deviceTimeIndex = getDeviceTimeIndexFromCache(resource); + ArrayDeviceTimeIndex deviceTimeIndex = getDeviceTimeIndexFromCache(resource); devices.addAll(deviceTimeIndex.getDevices()); resourceDevices.add(deviceTimeIndex); } int maxOverlapFileNumInSubCompactionTask = 1; for (IDeviceID device : devices) { - List<DeviceTimeIndex> resourcesContainsCurrentDevice = + List<ArrayDeviceTimeIndex> resourcesContainsCurrentDevice = resourceDevices.stream() .filter(resource -> !resource.definitelyNotContains(device)) .sorted(Comparator.comparingLong(resource -> resource.getStartTime(device))) @@ -130,7 +130,7 @@ public abstract class AbstractCompactionEstimator { long maxEndTimeOfCurrentDevice = Long.MIN_VALUE; int overlapFileNumOfCurrentDevice = 0; - for (DeviceTimeIndex resource : resourcesContainsCurrentDevice) { + for (ArrayDeviceTimeIndex resource : resourcesContainsCurrentDevice) { long deviceStartTimeInCurrentFile = resource.getStartTime(device); long deviceEndTimeInCurrentFile = resource.getEndTime(device); if (deviceStartTimeInCurrentFile <= maxEndTimeOfCurrentDevice) { @@ -154,7 +154,8 @@ public abstract class AbstractCompactionEstimator { return maxOverlapFileNumInSubCompactionTask; } - private DeviceTimeIndex getDeviceTimeIndexFromCache(TsFileResource resource) throws IOException { + private ArrayDeviceTimeIndex getDeviceTimeIndexFromCache(TsFileResource resource) + throws IOException { if (deviceTimeIndexCache.containsKey(resource)) { return deviceTimeIndexCache.get(resource); } @@ -162,8 +163,8 @@ public abstract class AbstractCompactionEstimator { if (timeIndex instanceof FileTimeIndex) { timeIndex = resource.buildDeviceTimeIndex(); } - deviceTimeIndexCache.put(resource, (DeviceTimeIndex) timeIndex); - return (DeviceTimeIndex) timeIndex; + deviceTimeIndexCache.put(resource, (ArrayDeviceTimeIndex) timeIndex); + return (ArrayDeviceTimeIndex) timeIndex; } public void cleanup() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java index daf8d1c86e8..de4c146fbe7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java @@ -22,7 +22,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; @@ -78,7 +78,7 @@ public class TsFileResourceCandidate { hasDetailedDeviceInfo = false; return; } - DeviceTimeIndex timeIndex = resource.buildDeviceTimeIndex(); + ArrayDeviceTimeIndex timeIndex = resource.buildDeviceTimeIndex(); for (IDeviceID deviceId : timeIndex.getDevices()) { deviceInfoMap.put( deviceId, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 5b012f6876d..72bcaa39c4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -34,9 +34,10 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.PlainDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; @@ -413,7 +414,7 @@ public class TsFileResource { return timeIndex.getDevices(file.getPath(), this); } - public DeviceTimeIndex buildDeviceTimeIndex() throws IOException { + public ArrayDeviceTimeIndex buildDeviceTimeIndex() throws IOException { readLock(); try { if (!resourceFileExists()) { @@ -424,10 +425,10 @@ public class TsFileResource { .getBufferedInputStream(file.getPath() + RESOURCE_SUFFIX)) { ReadWriteIOUtils.readByte(inputStream); ITimeIndex timeIndexFromResourceFile = ITimeIndex.createTimeIndex(inputStream); - if (!(timeIndexFromResourceFile instanceof DeviceTimeIndex)) { + if (!(timeIndexFromResourceFile instanceof ArrayDeviceTimeIndex)) { throw new IOException("cannot build DeviceTimeIndex from resource " + file.getPath()); } - return (DeviceTimeIndex) timeIndexFromResourceFile; + return (ArrayDeviceTimeIndex) timeIndexFromResourceFile; } catch (Exception e) { throw new IOException( "Can't read file " + file.getPath() + RESOURCE_SUFFIX + " from disk", e); @@ -1049,12 +1050,15 @@ public class TsFileResource { @TestOnly public void setTimeIndexType(byte type) { switch (type) { - case ITimeIndex.DEVICE_TIME_INDEX_TYPE: - this.timeIndex = new DeviceTimeIndex(); + case ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE: + this.timeIndex = new ArrayDeviceTimeIndex(); break; case ITimeIndex.FILE_TIME_INDEX_TYPE: this.timeIndex = new FileTimeIndex(); break; + case ITimeIndex.PLAIN_DEVICE_TIME_INDEX_TYPE: + this.timeIndex = new PlainDeviceTimeIndex(); + break; default: throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java similarity index 83% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index 777d092fee3..beef82716cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@ -19,14 +19,12 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; -import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.exception.PartitionViolationException; -import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; -import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID; +import org.apache.iotdb.tsfile.file.metadata.StringArrayDeviceID; import org.apache.iotdb.tsfile.utils.FilePathUtils; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.RamUsageEstimator; @@ -46,12 +44,12 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -public class DeviceTimeIndex implements ITimeIndex { +public class ArrayDeviceTimeIndex implements ITimeIndex { private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(DeviceTimeIndex.class); + RamUsageEstimator.shallowSizeOfInstance(ArrayDeviceTimeIndex.class); - private static final Logger logger = LoggerFactory.getLogger(DeviceTimeIndex.class); + private static final Logger logger = LoggerFactory.getLogger(ArrayDeviceTimeIndex.class); public static final int INIT_ARRAY_SIZE = 64; @@ -65,15 +63,15 @@ public class DeviceTimeIndex implements ITimeIndex { protected long[] endTimes; /** min start time */ - private long minStartTime = Long.MAX_VALUE; + protected long minStartTime = Long.MAX_VALUE; /** max end time */ - private long maxEndTime = Long.MIN_VALUE; + protected long maxEndTime = Long.MIN_VALUE; /** device -> index of start times array and end times array */ protected Map<IDeviceID, Integer> deviceToIndex; - public DeviceTimeIndex() { + public ArrayDeviceTimeIndex() { this.deviceToIndex = new ConcurrentHashMap<>(); this.startTimes = new long[INIT_ARRAY_SIZE]; this.endTimes = new long[INIT_ARRAY_SIZE]; @@ -81,7 +79,7 @@ public class DeviceTimeIndex implements ITimeIndex { initTimes(endTimes, Long.MIN_VALUE); } - public DeviceTimeIndex( + public ArrayDeviceTimeIndex( Map<IDeviceID, Integer> deviceToIndex, long[] startTimes, long[] endTimes) { this.startTimes = startTimes; this.endTimes = endTimes; @@ -99,16 +97,16 @@ public class DeviceTimeIndex implements ITimeIndex { ReadWriteIOUtils.write(endTimes[i], outputStream); } - for (Entry<IDeviceID, Integer> stringIntegerEntry : deviceToIndex.entrySet()) { - IDeviceID device = stringIntegerEntry.getKey(); - int index = stringIntegerEntry.getValue(); - ReadWriteIOUtils.write(((PlainDeviceID) device).toStringID(), outputStream); + for (Entry<IDeviceID, Integer> deviceIdIntegerEntry : deviceToIndex.entrySet()) { + IDeviceID device = deviceIdIntegerEntry.getKey(); + int index = deviceIdIntegerEntry.getValue(); + device.serialize(outputStream); ReadWriteIOUtils.write(index, outputStream); } } @Override - public DeviceTimeIndex deserialize(InputStream inputStream) throws IOException { + public ArrayDeviceTimeIndex deserialize(InputStream inputStream) throws IOException { int deviceNum = ReadWriteIOUtils.readInt(inputStream); startTimes = new long[deviceNum]; @@ -122,17 +120,15 @@ public class DeviceTimeIndex implements ITimeIndex { } for (int i = 0; i < deviceNum; i++) { - String path = - DataNodeDevicePathCache.getInstance() - .getDeviceId(ReadWriteIOUtils.readString(inputStream)); + StringArrayDeviceID deviceID = StringArrayDeviceID.deserialize(inputStream); int index = ReadWriteIOUtils.readInt(inputStream); - deviceToIndex.put(new PlainDeviceID(path), index); + deviceToIndex.put(deviceID, index); } return this; } @Override - public DeviceTimeIndex deserialize(ByteBuffer buffer) { + public ArrayDeviceTimeIndex deserialize(ByteBuffer buffer) { int deviceNum = buffer.getInt(); startTimes = new long[deviceNum]; endTimes = new long[deviceNum]; @@ -145,10 +141,9 @@ public class DeviceTimeIndex implements ITimeIndex { } for (int i = 0; i < deviceNum; i++) { - String path = - DataNodeDevicePathCache.getInstance().getDeviceId(ReadWriteIOUtils.readString(buffer)); + StringArrayDeviceID deviceID = StringArrayDeviceID.deserialize(buffer); int index = buffer.getInt(); - deviceToIndex.put(new PlainDeviceID(path), index); + deviceToIndex.put(deviceID, index); } return this; } @@ -179,11 +174,9 @@ public class DeviceTimeIndex implements ITimeIndex { ReadWriteIOUtils.skip(inputStream, 2L * deviceNum * ReadWriteIOUtils.LONG_LEN); Set<IDeviceID> devices = new HashSet<>(); for (int i = 0; i < deviceNum; i++) { - String path = - DataNodeDevicePathCache.getInstance() - .getDeviceId(ReadWriteIOUtils.readString(inputStream)); + StringArrayDeviceID deviceID = StringArrayDeviceID.deserialize(inputStream); ReadWriteIOUtils.skip(inputStream, ReadWriteIOUtils.INT_LEN); - devices.add(new PlainDeviceID(path)); + devices.add(deviceID); } return devices; } @@ -376,7 +369,7 @@ public class DeviceTimeIndex implements ITimeIndex { @Override public int compareDegradePriority(ITimeIndex timeIndex) { - if (timeIndex instanceof DeviceTimeIndex) { + if (timeIndex instanceof ArrayDeviceTimeIndex) { return Long.compare(getMinStartTime(), timeIndex.getMinStartTime()); } else if (timeIndex instanceof FileTimeIndex) { return -1; @@ -409,8 +402,22 @@ public class DeviceTimeIndex implements ITimeIndex { long startTime = Long.MAX_VALUE; long endTime = Long.MIN_VALUE; for (Entry<IDeviceID, Integer> entry : deviceToIndex.entrySet()) { - try { - if (deviceMatchInfo.contains(entry.getKey())) { + if (deviceMatchInfo.contains(entry.getKey())) { + hasMatchedDevice = true; + if (startTimes[entry.getValue()] < startTime) { + startTime = startTimes[entry.getValue()]; + } + if (endTimes[entry.getValue()] > endTime) { + endTime = endTimes[entry.getValue()]; + } + } else { + StringArrayDeviceID deviceID = (StringArrayDeviceID) entry.getKey(); + String[] devicePath = new String[deviceID.segmentNum()]; + for (int i = 0; i < devicePath.length; i++) { + devicePath[i] = deviceID.segment(i); + } + if (devicePattern.matchFullPath(new PartialPath(devicePath))) { + deviceMatchInfo.add(entry.getKey()); hasMatchedDevice = true; if (startTimes[entry.getValue()] < startTime) { startTime = startTimes[entry.getValue()]; @@ -418,22 +425,7 @@ public class DeviceTimeIndex implements ITimeIndex { if (endTimes[entry.getValue()] > endTime) { endTime = endTimes[entry.getValue()]; } - } else { - if (devicePattern.matchFullPath( - DataNodeDevicePathCache.getInstance() - .getPartialPath(((PlainDeviceID) entry.getKey()).toStringID()))) { - deviceMatchInfo.add(entry.getKey()); - hasMatchedDevice = true; - if (startTimes[entry.getValue()] < startTime) { - startTime = startTimes[entry.getValue()]; - } - if (endTimes[entry.getValue()] > endTime) { - endTime = endTimes[entry.getValue()]; - } - } } - } catch (IllegalPathException e) { - // won't reach here } } @@ -442,6 +434,6 @@ public class DeviceTimeIndex implements ITimeIndex { @Override public byte getTimeIndexType() { - return DEVICE_TIME_INDEX_TYPE; + return ARRAY_DEVICE_TIME_INDEX_TYPE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index 65aa8153a2a..916457c2ff0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -91,8 +91,12 @@ public class FileTimeIndex implements ITimeIndex { FSFactoryProducer.getFSFactory() .getBufferedInputStream(tsFilePath + TsFileResource.RESOURCE_SUFFIX)) { // The first byte is VERSION_NUMBER, second byte is timeIndexType. - ReadWriteIOUtils.readBytes(inputStream, 2); - return DeviceTimeIndex.getDevices(inputStream); + byte[] bytes = ReadWriteIOUtils.readBytes(inputStream, 2); + if (bytes[1] == ARRAY_DEVICE_TIME_INDEX_TYPE) { + return ArrayDeviceTimeIndex.getDevices(inputStream); + } else { + return PlainDeviceTimeIndex.getDevices(inputStream); + } } catch (NoSuchFileException e) { // deleted by ttl if (tsFileResource.isDeleted()) { @@ -211,7 +215,7 @@ public class FileTimeIndex implements ITimeIndex { @Override public int compareDegradePriority(ITimeIndex timeIndex) { - if (timeIndex instanceof DeviceTimeIndex) { + if (timeIndex instanceof ArrayDeviceTimeIndex) { return 1; } else if (timeIndex instanceof FileTimeIndex) { return Long.compare(startTime, timeIndex.getMinStartTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java index 0edbe81745d..f180ad1036a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java @@ -34,8 +34,9 @@ import java.util.Set; public interface ITimeIndex { - byte DEVICE_TIME_INDEX_TYPE = 1; + byte PLAIN_DEVICE_TIME_INDEX_TYPE = 1; byte FILE_TIME_INDEX_TYPE = 2; + byte ARRAY_DEVICE_TIME_INDEX_TYPE = 3; /** * serialize to outputStream diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/PlainDeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/PlainDeviceTimeIndex.java new file mode 100644 index 00000000000..6f5e3d3c258 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/PlainDeviceTimeIndex.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; +import org.apache.iotdb.tsfile.file.metadata.IDeviceID; +import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.utils.RamUsageEstimator; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class PlainDeviceTimeIndex extends ArrayDeviceTimeIndex implements ITimeIndex { + + private static final Logger logger = LoggerFactory.getLogger(PlainDeviceTimeIndex.class); + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ArrayDeviceTimeIndex.class); + + @Override + public void serialize(OutputStream outputStream) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public PlainDeviceTimeIndex deserialize(InputStream inputStream) throws IOException { + int deviceNum = ReadWriteIOUtils.readInt(inputStream); + + startTimes = new long[deviceNum]; + endTimes = new long[deviceNum]; + + for (int i = 0; i < deviceNum; i++) { + startTimes[i] = ReadWriteIOUtils.readLong(inputStream); + endTimes[i] = ReadWriteIOUtils.readLong(inputStream); + minStartTime = Math.min(minStartTime, startTimes[i]); + maxEndTime = Math.max(maxEndTime, endTimes[i]); + } + + for (int i = 0; i < deviceNum; i++) { + String path = + DataNodeDevicePathCache.getInstance() + .getDeviceId(ReadWriteIOUtils.readString(inputStream)); + int index = ReadWriteIOUtils.readInt(inputStream); + deviceToIndex.put(new PlainDeviceID(path), index); + } + return this; + } + + @Override + public PlainDeviceTimeIndex deserialize(ByteBuffer buffer) { + int deviceNum = buffer.getInt(); + startTimes = new long[deviceNum]; + endTimes = new long[deviceNum]; + + for (int i = 0; i < deviceNum; i++) { + startTimes[i] = buffer.getLong(); + endTimes[i] = buffer.getLong(); + minStartTime = Math.min(minStartTime, startTimes[i]); + maxEndTime = Math.max(maxEndTime, endTimes[i]); + } + + for (int i = 0; i < deviceNum; i++) { + String path = + DataNodeDevicePathCache.getInstance().getDeviceId(ReadWriteIOUtils.readString(buffer)); + int index = buffer.getInt(); + deviceToIndex.put(new PlainDeviceID(path), index); + } + return this; + } + + /** + * Deserialize TimeIndex and get devices only. + * + * @param inputStream inputStream + * @return device name + */ + public static Set<IDeviceID> getDevices(InputStream inputStream) throws IOException { + int deviceNum = ReadWriteIOUtils.readInt(inputStream); + ReadWriteIOUtils.skip(inputStream, 2L * deviceNum * ReadWriteIOUtils.LONG_LEN); + Set<IDeviceID> devices = new HashSet<>(); + for (int i = 0; i < deviceNum; i++) { + String path = + DataNodeDevicePathCache.getInstance() + .getDeviceId(ReadWriteIOUtils.readString(inputStream)); + ReadWriteIOUtils.skip(inputStream, ReadWriteIOUtils.INT_LEN); + devices.add(new PlainDeviceID(path)); + } + return devices; + } + + @Override + public Pair<Long, Long> getPossibleStartTimeAndEndTime( + PartialPath devicePattern, Set<IDeviceID> deviceMatchInfo) { + boolean hasMatchedDevice = false; + long startTime = Long.MAX_VALUE; + long endTime = Long.MIN_VALUE; + for (Map.Entry<IDeviceID, Integer> entry : deviceToIndex.entrySet()) { + try { + if (deviceMatchInfo.contains(entry.getKey())) { + hasMatchedDevice = true; + if (startTimes[entry.getValue()] < startTime) { + startTime = startTimes[entry.getValue()]; + } + if (endTimes[entry.getValue()] > endTime) { + endTime = endTimes[entry.getValue()]; + } + } else { + if (devicePattern.matchFullPath( + DataNodeDevicePathCache.getInstance() + .getPartialPath(((PlainDeviceID) entry.getKey()).toStringID()))) { + deviceMatchInfo.add(entry.getKey()); + hasMatchedDevice = true; + if (startTimes[entry.getValue()] < startTime) { + startTime = startTimes[entry.getValue()]; + } + if (endTimes[entry.getValue()] > endTime) { + endTime = endTimes[entry.getValue()]; + } + } + } + } catch (IllegalPathException e) { + // won't reach here + } + } + + return hasMatchedDevice ? new Pair<>(startTime, endTime) : null; + } + + @Override + public int compareDegradePriority(ITimeIndex timeIndex) { + if (timeIndex instanceof ArrayDeviceTimeIndex) { + return Long.compare(getMinStartTime(), timeIndex.getMinStartTime()); + } else if (timeIndex instanceof FileTimeIndex) { + return -1; + } else { + logger.error("Wrong timeIndex type {}", timeIndex.getClass().getName()); + throw new RuntimeException("Wrong timeIndex type " + timeIndex.getClass().getName()); + } + } + + @Override + public long calculateRamSize() { + return INSTANCE_SIZE + + RamUsageEstimator.sizeOfMap( + deviceToIndex, RamUsageEstimator.shallowSizeOfInstance(Integer.class)) + + RamUsageEstimator.sizeOf(startTimes) + + RamUsageEstimator.sizeOf(endTimes); + } + + @Override + public byte getTimeIndexType() { + return PLAIN_DEVICE_TIME_INDEX_TYPE; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/TimeIndexLevel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/TimeIndexLevel.java index e958f053e5f..3a4a5900ba2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/TimeIndexLevel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/TimeIndexLevel.java @@ -23,21 +23,26 @@ public enum TimeIndexLevel { /** v0.12 file to time index (small memory foot print) */ V012_FILE_TIME_INDEX, - /** device to time index (large memory foot print) */ - DEVICE_TIME_INDEX, + /** plain device to time index (large memory foot print) */ + PLAIN_DEVICE_TIME_INDEX, /** file to time index (small memory foot print) */ - FILE_TIME_INDEX; + FILE_TIME_INDEX, + + /** array device to time index (large memory foot print) */ + ARRAY_DEVICE_TIME_INDEX; public ITimeIndex getTimeIndex() { switch (this) { case V012_FILE_TIME_INDEX: throw new IllegalStateException("V012_FILE_TIME_INDEX should never appear"); + case PLAIN_DEVICE_TIME_INDEX: + return new PlainDeviceTimeIndex(); case FILE_TIME_INDEX: return new FileTimeIndex(); - case DEVICE_TIME_INDEX: + case ARRAY_DEVICE_TIME_INDEX: default: - return new DeviceTimeIndex(); + return new ArrayDeviceTimeIndex(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java index eca984c01bd..589aa751b96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java @@ -21,7 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.utils; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -67,13 +67,13 @@ public class TsFileResourceUtils { } public static boolean validateTsFileResourceCorrectness(TsFileResource resource) { - DeviceTimeIndex timeIndex; + ArrayDeviceTimeIndex timeIndex; try { if (resource.getTimeIndexType() != 1) { // if time index is not device time index, then deserialize it from resource file timeIndex = resource.buildDeviceTimeIndex(); } else { - timeIndex = (DeviceTimeIndex) resource.getTimeIndex(); + timeIndex = (ArrayDeviceTimeIndex) resource.getTimeIndex(); } if (timeIndex == null) { logger.error("{} {} time index is null", resource.getTsFilePath(), VALIDATE_FAILED); @@ -345,7 +345,7 @@ public class TsFileResourceUtils { // deviceID -> <TsFileResource, last end time> Map<IDeviceID, Pair<TsFileResource, Long>> lastEndTimeMap = new HashMap<>(); for (TsFileResource resource : resources) { - DeviceTimeIndex timeIndex; + ArrayDeviceTimeIndex timeIndex; if (resource.getTimeIndexType() != 1) { // if time index is not device time index, then deserialize it from resource file try { @@ -355,7 +355,7 @@ public class TsFileResourceUtils { continue; } } else { - timeIndex = (DeviceTimeIndex) resource.getTimeIndex(); + timeIndex = (ArrayDeviceTimeIndex) resource.getTimeIndex(); } if (timeIndex == null) { return false; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/LastFlushTimeMapTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/LastFlushTimeMapTest.java index f8f6cb7647f..4486587bd99 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/LastFlushTimeMapTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/LastFlushTimeMapTest.java @@ -27,7 +27,7 @@ import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; @@ -201,7 +201,7 @@ public class LastFlushTimeMapTest { IDeviceID device = new PlainDeviceID("root.vehicle.d0"); File unseqResourceFile1 = new File(unseqDirPath + File.separator + "4-4-0-0.tsfile.resource"); TsFileResource unseqResource1 = new TsFileResource(); - unseqResource1.setTimeIndex(new DeviceTimeIndex()); + unseqResource1.setTimeIndex(new ArrayDeviceTimeIndex()); unseqResource1.setFile(unseqResourceFile1); unseqResource1.updateStartTime(device, 1); unseqResource1.updateEndTime(device, 100); @@ -209,7 +209,7 @@ public class LastFlushTimeMapTest { File unseqResourceFile2 = new File(unseqDirPath + File.separator + "5-5-0-0.tsfile.resource"); TsFileResource unseqResource2 = new TsFileResource(); - unseqResource2.setTimeIndex(new DeviceTimeIndex()); + unseqResource2.setTimeIndex(new ArrayDeviceTimeIndex()); unseqResource2.setFile(unseqResourceFile2); unseqResource2.updateStartTime(device, 1); unseqResource2.updateEndTime(device, 10); @@ -217,7 +217,7 @@ public class LastFlushTimeMapTest { File unseqResourceFile3 = new File(unseqDirPath + File.separator + "6-6-0-0.tsfile.resource"); TsFileResource unseqResource3 = new TsFileResource(); - unseqResource3.setTimeIndex(new DeviceTimeIndex()); + unseqResource3.setTimeIndex(new ArrayDeviceTimeIndex()); unseqResource3.setFile(unseqResourceFile3); unseqResource3.updateStartTime(device, 1); unseqResource3.updateEndTime(device, 70); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileResourceProgressIndexTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileResourceProgressIndexTest.java index e149e67ba44..d9214b5bb3f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileResourceProgressIndexTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/TsFileResourceProgressIndexTest.java @@ -29,7 +29,7 @@ import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID; @@ -72,7 +72,8 @@ public class TsFileResourceProgressIndexTest { public void setUp() { IntStream.range(0, DEVICE_NUM) .forEach(i -> deviceToIndex.put(new PlainDeviceID("root.sg.d" + i), i)); - DeviceTimeIndex deviceTimeIndex = new DeviceTimeIndex(deviceToIndex, startTimes, endTimes); + ArrayDeviceTimeIndex deviceTimeIndex = + new ArrayDeviceTimeIndex(deviceToIndex, startTimes, endTimes); IntStream.range(0, DEVICE_NUM) .forEach( i -> { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java index b2d81c36a31..c1d90ed85b9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.iotdb.db.utils.constant.TestConstant; @@ -215,7 +215,7 @@ public class CompactionValidationTest { public void testTsFileResourceIsDeletedByOtherCompactionTaskWhenValidateOverlap1() { TsFileResource resource1 = new TsFileResource(); resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile")); - resource1.setTimeIndex(new DeviceTimeIndex()); + resource1.setTimeIndex(new ArrayDeviceTimeIndex()); resource1.updateStartTime(new PlainDeviceID("d1"), 1); resource1.updateEndTime(new PlainDeviceID("d1"), 2); @@ -225,7 +225,7 @@ public class CompactionValidationTest { TsFileResource resource3 = new TsFileResource(); resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile")); - resource3.setTimeIndex(new DeviceTimeIndex()); + resource3.setTimeIndex(new ArrayDeviceTimeIndex()); resource3.updateStartTime(new PlainDeviceID("d1"), 4); resource3.updateEndTime(new PlainDeviceID("d1"), 5); @@ -237,7 +237,7 @@ public class CompactionValidationTest { @Test public void testTsFileResourceIsDeletedByOtherCompactionTaskWhenValidateOverlap2() { TsFileResource resource1 = new TsFileResource(); - resource1.setTimeIndex(new DeviceTimeIndex()); + resource1.setTimeIndex(new ArrayDeviceTimeIndex()); resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile")); resource1.updateStartTime(new PlainDeviceID("d1"), 1); resource1.updateEndTime(new PlainDeviceID("d1"), 2); @@ -248,7 +248,7 @@ public class CompactionValidationTest { TsFileResource resource3 = new TsFileResource(); resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile")); - resource3.setTimeIndex(new DeviceTimeIndex()); + resource3.setTimeIndex(new ArrayDeviceTimeIndex()); resource3.updateStartTime(new PlainDeviceID("d1"), 1); resource3.updateEndTime(new PlainDeviceID("d1"), 5); @@ -260,7 +260,7 @@ public class CompactionValidationTest { @Test public void testTsFileResourceIsBrokenWhenValidateOverlap1() throws IOException { TsFileResource resource1 = new TsFileResource(); - resource1.setTimeIndex(new DeviceTimeIndex()); + resource1.setTimeIndex(new ArrayDeviceTimeIndex()); resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile")); resource1.updateStartTime(new PlainDeviceID("d1"), 1); resource1.updateEndTime(new PlainDeviceID("d1"), 2); @@ -275,7 +275,7 @@ public class CompactionValidationTest { TsFileResource resource3 = new TsFileResource(); resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile")); - resource3.setTimeIndex(new DeviceTimeIndex()); + resource3.setTimeIndex(new ArrayDeviceTimeIndex()); resource3.updateStartTime(new PlainDeviceID("d1"), 4); resource3.updateEndTime(new PlainDeviceID("d1"), 5); @@ -287,7 +287,7 @@ public class CompactionValidationTest { @Test public void testTsFileResourceIsBrokenWhenValidateOverlap2() throws IOException { TsFileResource resource1 = new TsFileResource(); - resource1.setTimeIndex(new DeviceTimeIndex()); + resource1.setTimeIndex(new ArrayDeviceTimeIndex()); resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile")); resource1.updateStartTime(new PlainDeviceID("d1"), 1); resource1.updateEndTime(new PlainDeviceID("d1"), 2); @@ -302,7 +302,7 @@ public class CompactionValidationTest { TsFileResource resource3 = new TsFileResource(); resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile")); - resource3.setTimeIndex(new DeviceTimeIndex()); + resource3.setTimeIndex(new ArrayDeviceTimeIndex()); resource3.updateStartTime(new PlainDeviceID("d1"), 1); resource3.updateEndTime(new PlainDeviceID("d1"), 5); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java index edd27847c1f..ef73fe05837 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java @@ -33,7 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionF import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID; @@ -348,7 +348,7 @@ public class InsertionCrossSpaceCompactionRecoverTest extends AbstractCompaction private TsFileResource createTsFileResource(String name, boolean seq) { String filePath = (seq ? SEQ_DIRS : UNSEQ_DIRS) + File.separator + name; TsFileResource resource = new TsFileResource(); - resource.setTimeIndex(new DeviceTimeIndex()); + resource.setTimeIndex(new ArrayDeviceTimeIndex()); resource.setFile(new File(filePath)); resource.setStatusForTest(TsFileResourceStatus.NORMAL); resource.setSeq(seq); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java index 6999394eeda..289047b3b53 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java @@ -30,7 +30,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils.Cr import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.utils.InsertionCrossCompactionTaskResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; import org.apache.iotdb.tsfile.file.metadata.PlainDeviceID; @@ -2339,7 +2339,7 @@ public class InsertionCrossSpaceCompactionSelectorTest extends AbstractCompactio private TsFileResource createTsFileResource(String name, boolean seq) { String filePath = (seq ? SEQ_DIRS : UNSEQ_DIRS) + File.separator + name; TsFileResource resource = new TsFileResource(); - resource.setTimeIndex(new DeviceTimeIndex()); + resource.setTimeIndex(new ArrayDeviceTimeIndex()); resource.setFile(new File(filePath)); resource.setStatusForTest(TsFileResourceStatus.NORMAL); resource.setSeq(seq); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java index c73a591d90f..045cc05704d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceTest.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; @@ -50,13 +50,17 @@ public class TsFileResourceTest { @Before public void setUp() { IntStream.range(0, DEVICE_NUM) - .forEach(i -> deviceToIndex.put(new PlainDeviceID("root.sg.d" + i), i)); - DeviceTimeIndex deviceTimeIndex = new DeviceTimeIndex(deviceToIndex, startTimes, endTimes); + .forEach( + i -> deviceToIndex.put(IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d" + i), i)); + ArrayDeviceTimeIndex deviceTimeIndex = + new ArrayDeviceTimeIndex(deviceToIndex, startTimes, endTimes); IntStream.range(0, DEVICE_NUM) .forEach( i -> { - deviceTimeIndex.updateStartTime(new PlainDeviceID("root.sg.d" + i), i); - deviceTimeIndex.updateEndTime(new PlainDeviceID("root.sg.d" + i), i + 1); + deviceTimeIndex.updateStartTime( + IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d" + i), i); + deviceTimeIndex.updateEndTime( + IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d" + i), i + 1); }); tsFileResource.setTimeIndex(deviceTimeIndex); tsFileResource.setStatusForTest(TsFileResourceStatus.NORMAL); @@ -84,7 +88,7 @@ public class TsFileResourceTest { @Test public void testDegradeAndFileTimeIndex() { - Assert.assertEquals(ITimeIndex.DEVICE_TIME_INDEX_TYPE, tsFileResource.getTimeIndexType()); + Assert.assertEquals(ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE, tsFileResource.getTimeIndexType()); tsFileResource.degradeTimeIndex(); Assert.assertEquals(ITimeIndex.FILE_TIME_INDEX_TYPE, tsFileResource.getTimeIndexType()); Assert.assertEquals(deviceToIndex.keySet(), tsFileResource.getDevices()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java index 3c169f6ed9b..f98339563b0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java @@ -182,7 +182,7 @@ public class ResourceManagerTest { prepareFile(tsFileResource, 0, ptNum, 0); long previousRamSize = tsFileResource.calculateRamSize(); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); long reducedMemory = tsFileResource.degradeTimeIndex(); assertEquals(previousRamSize - tsFileResource.calculateRamSize(), reducedMemory); @@ -208,7 +208,7 @@ public class ResourceManagerTest { tsFileResource.updatePlanIndexes((long) 0); prepareFile(tsFileResource, 0, ptNum, 0); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); long curTimeIndexMemoryThreshold = 322; tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold); @@ -235,7 +235,7 @@ public class ResourceManagerTest { tsFileResource.updatePlanIndexes((long) 0); prepareFile(tsFileResource, 0, ptNum, 0); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); long previousRamSize = tsFileResource.calculateRamSize(); long curTimeIndexMemoryThreshold = 3221; @@ -243,7 +243,7 @@ public class ResourceManagerTest { tsFileResourceManager.registerSealedTsFileResource(tsFileResource); assertEquals(0, previousRamSize - tsFileResource.calculateRamSize()); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); } @@ -265,13 +265,13 @@ public class ResourceManagerTest { tsFileResource1.updatePlanIndexes((long) 0); prepareFile(tsFileResource1, 0, ptNum, 0); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource1.getTimeIndexType())); long curTimeIndexMemoryThreshold = 3221; tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold); tsFileResourceManager.registerSealedTsFileResource(tsFileResource1); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource1.getTimeIndexType())); File file2 = new File( @@ -289,13 +289,13 @@ public class ResourceManagerTest { tsFileResource2.updatePlanIndexes((long) 1); prepareFile(tsFileResource2, ptNum, ptNum, 0); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource2.getTimeIndexType())); tsFileResourceManager.registerSealedTsFileResource(tsFileResource2); assertEquals( TimeIndexLevel.FILE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource1.getTimeIndexType())); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource2.getTimeIndexType())); } @@ -319,7 +319,7 @@ public class ResourceManagerTest { tsFileResource.setStatusForTest(TsFileResourceStatus.NORMAL); tsFileResource.updatePlanIndexes((long) i); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0); @@ -334,7 +334,7 @@ public class ResourceManagerTest { TimeIndexLevel.valueOf(seqResources.get(i).getTimeIndexType())); } else { assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(seqResources.get(i).getTimeIndexType())); } } @@ -358,7 +358,7 @@ public class ResourceManagerTest { + ".tsfile")); TsFileResource tsFileResource = new TsFileResource(file); assertEquals( - TimeIndexLevel.DEVICE_TIME_INDEX, + TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); seqResources.add(tsFileResource); prepareFile(tsFileResource, i * ptNum, ptNum, 0);
