This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rm_TVListAllocator in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5b9e762241dd55961ae7cd83b15ecf08d0db09c4 Author: HTHou <[email protected]> AuthorDate: Wed Dec 8 14:29:51 2021 +0800 [IOTDB-2072] Remove TVListAllocator to reduce the TVList mem cost --- .../engine/memtable/AlignedWritableMemChunk.java | 5 +- .../iotdb/db/engine/memtable/WritableMemChunk.java | 5 +- .../db/engine/storagegroup/TsFileProcessor.java | 20 ++--- .../apache/iotdb/db/rescon/TVListAllocator.java | 98 ---------------------- .../iotdb/db/rescon/TVListAllocatorMBean.java | 24 ------ .../java/org/apache/iotdb/db/service/IoTDB.java | 2 - .../db/utils/datastructure/AlignedTVList.java | 21 +++-- .../iotdb/db/utils/datastructure/TVList.java | 17 ++-- 8 files changed, 36 insertions(+), 156 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java index 6cfed35..c3539d2 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.engine.memtable; -import org.apache.iotdb.db.rescon.TVListAllocator; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; @@ -55,7 +54,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk { measurementIndexMap.put(schemaList.get(i).getMeasurementId(), i); dataTypeList.add(schemaList.get(i).getType()); } - this.list = TVListAllocator.getInstance().allocate(dataTypeList); + AlignedTVList.newAlignedList(dataTypeList); } public Set<String> getAllMeasurements() { @@ -339,7 +338,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk { @Override public void release() { if (list.getReferenceCount() == 0) { - TVListAllocator.getInstance().release(list); + list.clear(); } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java index df107ce..a6c4355 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunk.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.engine.memtable; -import org.apache.iotdb.db.rescon.TVListAllocator; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; @@ -42,7 +41,7 @@ public class WritableMemChunk implements IWritableMemChunk { public WritableMemChunk(IMeasurementSchema schema) { this.schema = schema; - this.list = TVListAllocator.getInstance().allocate(schema.getType()); + this.list = TVList.newList(schema.getType()); } @Override @@ -348,7 +347,7 @@ public class WritableMemChunk implements IWritableMemChunk { @Override public void release() { if (list.getReferenceCount() == 0) { - TVListAllocator.getInstance().release(list); + list.clear(); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index e52a43d..06b1edf 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -352,14 +352,14 @@ public class TsFileProcessor { chunkMetadataIncrement += ChunkMetadata.calculateRamSize( insertRowPlan.getMeasurements()[i], insertRowPlan.getDataTypes()[i]); - memTableIncrement += TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i]); + memTableIncrement += TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i]); } else { // here currentChunkPointNum >= 1 long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, insertRowPlan.getMeasurements()[i]); memTableIncrement += (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0 - ? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i]) + ? TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i]) : 0; } // TEXT data mem size @@ -385,14 +385,14 @@ public class TsFileProcessor { chunkMetadataIncrement += ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR) * insertRowPlan.getDataTypes().length; - memTableIncrement += AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes()); + memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes()); } else { // here currentChunkPointNum >= 1 long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER); memTableIncrement += (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0 - ? AlignedTVList.alignedTvListArrayMemSize(insertRowPlan.getDataTypes()) + ? AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes()) : 0; alignedMemChunk = ((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId)) @@ -484,19 +484,19 @@ public class TsFileProcessor { memIncrements[2] += ChunkMetadata.calculateRamSize(measurement, dataType); memIncrements[0] += ((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1) - * TVList.tvListArrayMemSize(dataType); + * TVList.tvListArrayMemCost(dataType); } else { long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, measurement); if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) { memIncrements[0] += ((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1) - * TVList.tvListArrayMemSize(dataType); + * TVList.tvListArrayMemCost(dataType); } else { long acquireArray = (end - start - 1 + (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE)) / PrimitiveArrayManager.ARRAY_SIZE; memIncrements[0] += - acquireArray == 0 ? 0 : acquireArray * TVList.tvListArrayMemSize(dataType); + acquireArray == 0 ? 0 : acquireArray * TVList.tvListArrayMemCost(dataType); } } // TEXT data size @@ -523,14 +523,14 @@ public class TsFileProcessor { * ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); memIncrements[0] += ((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1) - * AlignedTVList.alignedTvListArrayMemSize(dataTypes); + * AlignedTVList.alignedTvListArrayMemCost(dataTypes); } else { int currentChunkPointNum = (int) workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER); if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) { memIncrements[0] += ((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1) - * AlignedTVList.alignedTvListArrayMemSize(dataTypes); + * AlignedTVList.alignedTvListArrayMemCost(dataTypes); } else { int acquireArray = (end - start - 1 + (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE)) @@ -538,7 +538,7 @@ public class TsFileProcessor { memIncrements[0] += acquireArray == 0 ? 0 - : acquireArray * AlignedTVList.alignedTvListArrayMemSize(dataTypes); + : acquireArray * AlignedTVList.alignedTvListArrayMemCost(dataTypes); } vectorMemChunk = ((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceId)) diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java deleted file mode 100644 index 6d0c4e7..0000000 --- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.rescon; - -import org.apache.iotdb.db.conf.IoTDBConstant; -import org.apache.iotdb.db.exception.StartupException; -import org.apache.iotdb.db.service.IService; -import org.apache.iotdb.db.service.JMXService; -import org.apache.iotdb.db.service.ServiceType; -import org.apache.iotdb.db.utils.datastructure.AlignedTVList; -import org.apache.iotdb.db.utils.datastructure.TVList; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; - -import java.util.ArrayDeque; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; -import java.util.Queue; - -public class TVListAllocator implements TVListAllocatorMBean, IService { - - private Map<TSDataType, Queue<TVList>> tvListCache = new EnumMap<>(TSDataType.class); - private String mbeanName = - String.format( - "%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE, getID().getJmxName()); - - public static TVListAllocator getInstance() { - return InstanceHolder.INSTANCE; - } - - private static class InstanceHolder { - private static final TVListAllocator INSTANCE = new TVListAllocator(); - - private InstanceHolder() {} - } - - public synchronized TVList allocate(TSDataType dataType) { - Queue<TVList> tvLists = tvListCache.computeIfAbsent(dataType, k -> new ArrayDeque<>()); - TVList list = tvLists.poll(); - return list != null ? list : TVList.newList(dataType); - } - - public synchronized AlignedTVList allocate(List<TSDataType> dataTypes) { - return AlignedTVList.newAlignedList(dataTypes); - } - - public synchronized void release(TVList list) { - list.clear(tvListCache); - } - - @Override - public int getNumberOfTVLists() { - int number = 0; - for (Queue<TVList> queue : tvListCache.values()) { - number += queue.size(); - } - return number; - } - - @Override - public void start() throws StartupException { - try { - JMXService.registerMBean(InstanceHolder.INSTANCE, mbeanName); - } catch (Exception e) { - throw new StartupException(this.getID().getName(), e.getMessage()); - } - } - - @Override - public void stop() { - JMXService.deregisterMBean(mbeanName); - for (Queue<TVList> queue : tvListCache.values()) { - queue.clear(); - } - } - - @Override - public ServiceType getID() { - return ServiceType.TVLIST_ALLOCATOR_SERVICE; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java deleted file mode 100644 index fed0015..0000000 --- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.rescon; - -public interface TVListAllocatorMBean { - - int getNumberOfTVLists(); -} diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index 32bc1a3..93c4cf2 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@ -41,7 +41,6 @@ import org.apache.iotdb.db.query.udf.service.UDFClassLoaderManager; import org.apache.iotdb.db.query.udf.service.UDFRegistrationService; import org.apache.iotdb.db.rescon.PrimitiveArrayManager; import org.apache.iotdb.db.rescon.SystemInfo; -import org.apache.iotdb.db.rescon.TVListAllocator; import org.apache.iotdb.db.rest.RestService; import org.apache.iotdb.db.sync.receiver.SyncServerManager; import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager; @@ -120,7 +119,6 @@ public class IoTDB implements IoTDBMBean { registerManager.register(FlushManager.getInstance()); registerManager.register(MultiFileLogNodeManager.getInstance()); registerManager.register(Measurement.INSTANCE); - registerManager.register(TVListAllocator.getInstance()); registerManager.register(CacheHitRatioMonitor.getInstance()); registerManager.register(MergeManager.getINSTANCE()); registerManager.register(CompactionTaskManager.getInstance()); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 120c9f8..6f42171 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -34,10 +34,10 @@ import org.apache.iotdb.tsfile.utils.TsPrimitiveType; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Queue; import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; public class AlignedTVList extends TVList { @@ -850,26 +850,29 @@ public class AlignedTVList extends TVList { } /** - * Get the single alignedTVList array size by give types. + * Get the single alignedTVList array mem cost by give types. * * @param types the types in the vector * @return AlignedTvListArrayMemSize */ - public static long alignedTvListArrayMemSize(TSDataType[] types) { + public static long alignedTvListArrayMemCost(TSDataType[] types) { long size = 0; - // time size + // time array mem size size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L; - // index size + // index array mem size size += (long) PrimitiveArrayManager.ARRAY_SIZE * 4L; - // value size + // value array mem size for (TSDataType type : types) { size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); } + // array headers mem size + size += NUM_BYTES_ARRAY_HEADER * (2 + types.length); + // Object references size in ArrayList + size += NUM_BYTES_OBJECT_REF * (2 + types.length); return size; } - @Override - public void clear(Map<TSDataType, Queue<TVList>> tvListCache) { + public void clear() { size = 0; sorted = true; minTime = Long.MAX_VALUE; diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 0b2cf6f..ec383ea 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -32,11 +32,11 @@ import org.apache.iotdb.tsfile.utils.BitMap; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Queue; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; +import static org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF; public abstract class TVList { @@ -84,12 +84,16 @@ public abstract class TVList { return null; } - public static long tvListArrayMemSize(TSDataType type) { + public static long tvListArrayMemCost(TSDataType type) { long size = 0; - // time size + // time array mem size size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L; - // value size + // value array mem size size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long) type.getDataTypeSize(); + // two array headers mem size + size += NUM_BYTES_ARRAY_HEADER * 2; + // Object references size in ArrayList + size += NUM_BYTES_OBJECT_REF * 2; return size; } @@ -283,7 +287,7 @@ public abstract class TVList { cloneList.minTime = minTime; } - public void clear(Map<TSDataType, Queue<TVList>> tvListCache) { + public void clear() { size = 0; sorted = true; minTime = Long.MAX_VALUE; @@ -292,7 +296,6 @@ public abstract class TVList { clearValue(); clearSortedValue(); - tvListCache.get(getDataType()).add(this); } protected void clearTime() {
