This is an automated email from the ASF dual-hosted git repository.
chaow pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new e29dd22 [To rel/0.12] [IOTDB-2072] Remove TVListAllocator to reduce
the TVList mem cost (#4530)
e29dd22 is described below
commit e29dd229b7f3b00212586a352283d87ce8823a2f
Author: Haonan <[email protected]>
AuthorDate: Wed Dec 8 14:37:16 2021 +0800
[To rel/0.12] [IOTDB-2072] Remove TVListAllocator to reduce the TVList mem
cost (#4530)
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 3 +-
.../db/engine/memtable/PrimitiveMemTable.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 10 +--
.../apache/iotdb/db/rescon/TVListAllocator.java | 94 ----------------------
.../iotdb/db/rescon/TVListAllocatorMBean.java | 24 ------
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 -
.../iotdb/db/utils/datastructure/TVList.java | 12 ++-
7 files changed, 17 insertions(+), 132 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 6bf79e6..7ca6afc 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -341,7 +340,7 @@ public abstract class AbstractMemTable implements IMemTable
{
for (Entry<String, IWritableMemChunk> subEntry :
entry.getValue().entrySet()) {
TVList list = subEntry.getValue().getTVList();
if (list.getReferenceCount() == 0) {
- TVListAllocator.getInstance().release(list);
+ list.clear();
}
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
index 254d722..70c5abd 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java
@@ -19,7 +19,7 @@
package org.apache.iotdb.db.engine.memtable;
-import org.apache.iotdb.db.rescon.TVListAllocator;
+import org.apache.iotdb.db.utils.datastructure.TVList;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import java.util.HashMap;
@@ -39,7 +39,7 @@ public class PrimitiveMemTable extends AbstractMemTable {
@Override
protected IWritableMemChunk genMemSeries(MeasurementSchema schema) {
- return new WritableMemChunk(schema,
TVListAllocator.getInstance().allocate(schema.getType()));
+ return new WritableMemChunk(schema, TVList.newList(schema.getType()));
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 2fba8da..0404ce9 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -312,14 +312,14 @@ public class TsFileProcessor {
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(
insertRowPlan.getMeasurements()[i],
insertRowPlan.getDataTypes()[i]);
- memTableIncrement +=
TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i]);
+ memTableIncrement +=
TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i]);
} else {
// here currentChunkPointNum >= 1
int currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId,
insertRowPlan.getMeasurements()[i]);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
- ? TVList.tvListArrayMemSize(insertRowPlan.getDataTypes()[i])
+ ? TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i])
: 0;
}
// TEXT data mem size
@@ -372,19 +372,19 @@ public class TsFileProcessor {
memIncrements[2] += ChunkMetadata.calculateRamSize(measurement,
dataType);
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * TVList.tvListArrayMemSize(dataType);
+ * TVList.tvListArrayMemCost(dataType);
} else {
int currentChunkPointNum =
workMemTable.getCurrentChunkPointNum(deviceId, measurement);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * TVList.tvListArrayMemSize(dataType);
+ * TVList.tvListArrayMemCost(dataType);
} else {
int acquireArray =
(end - start - 1 + (currentChunkPointNum %
PrimitiveArrayManager.ARRAY_SIZE))
/ PrimitiveArrayManager.ARRAY_SIZE;
memIncrements[0] +=
- acquireArray == 0 ? 0 : acquireArray *
TVList.tvListArrayMemSize(dataType);
+ acquireArray == 0 ? 0 : acquireArray *
TVList.tvListArrayMemCost(dataType);
}
}
// TEXT data size
diff --git
a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
deleted file mode 100644
index 492de33..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.rescon;
-
-import org.apache.iotdb.db.conf.IoTDBConstant;
-import org.apache.iotdb.db.exception.StartupException;
-import org.apache.iotdb.db.service.IService;
-import org.apache.iotdb.db.service.JMXService;
-import org.apache.iotdb.db.service.ServiceType;
-import org.apache.iotdb.db.utils.datastructure.TVList;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.util.ArrayDeque;
-import java.util.EnumMap;
-import java.util.Map;
-import java.util.Queue;
-
-public class TVListAllocator implements TVListAllocatorMBean, IService {
-
- private Map<TSDataType, Queue<TVList>> tvListCache = new
EnumMap<>(TSDataType.class);
- private String mbeanName =
- String.format(
- "%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE,
getID().getJmxName());
-
- private static final TVListAllocator INSTANCE = new TVListAllocator();
-
- public static TVListAllocator getInstance() {
- return INSTANCE;
- }
-
- public synchronized TVList allocate(TSDataType dataType) {
- Queue<TVList> tvLists = tvListCache.computeIfAbsent(dataType, k -> new
ArrayDeque<>());
- TVList list = tvLists.poll();
- return list != null ? list : TVList.newList(dataType);
- }
-
- public synchronized void release(TSDataType dataType, TVList list) {
- list.clear();
- tvListCache.get(dataType).add(list);
- }
-
- public synchronized void release(TVList list) {
- list.clear();
- tvListCache.get(list.getDataType()).add(list);
- }
-
- @Override
- public int getNumberOfTVLists() {
- int number = 0;
- for (Queue<TVList> queue : tvListCache.values()) {
- number += queue.size();
- }
- return number;
- }
-
- @Override
- public void start() throws StartupException {
- try {
- JMXService.registerMBean(INSTANCE, mbeanName);
- } catch (Exception e) {
- throw new StartupException(this.getID().getName(), e.getMessage());
- }
- }
-
- @Override
- public void stop() {
- JMXService.deregisterMBean(mbeanName);
- for (Queue<TVList> queue : tvListCache.values()) {
- queue.clear();
- }
- }
-
- @Override
- public ServiceType getID() {
- return ServiceType.TVLIST_ALLOCATOR_SERVICE;
- }
-}
diff --git
a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
deleted file mode 100644
index fed0015..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.rescon;
-
-public interface TVListAllocatorMBean {
-
- int getNumberOfTVLists();
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index d29dfc3..a35f70b 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -37,7 +37,6 @@ import
org.apache.iotdb.db.query.udf.service.UDFClassLoaderManager;
import org.apache.iotdb.db.query.udf.service.UDFRegistrationService;
import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.rescon.SystemInfo;
-import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
@@ -113,7 +112,6 @@ public class IoTDB implements IoTDBMBean {
registerManager.register(FlushManager.getInstance());
registerManager.register(MultiFileLogNodeManager.getInstance());
registerManager.register(Measurement.INSTANCE);
- registerManager.register(TVListAllocator.getInstance());
registerManager.register(CacheHitRatioMonitor.getInstance());
registerManager.register(MergeManager.getINSTANCE());
registerManager.register(CompactionMergeTaskPoolManager.getInstance());
diff --git
a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 8ce6295..f7d9823 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -34,6 +34,8 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.iotdb.db.rescon.PrimitiveArrayManager.ARRAY_SIZE;
+import static
org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static
org.apache.iotdb.tsfile.utils.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
public abstract class TVList {
@@ -79,12 +81,16 @@ public abstract class TVList {
return null;
}
- public static long tvListArrayMemSize(TSDataType type) {
+ public static long tvListArrayMemCost(TSDataType type) {
long size = 0;
- // time size
+ // time array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * 8L;
- // value size
+ // value array mem size
size += (long) PrimitiveArrayManager.ARRAY_SIZE * (long)
type.getDataTypeSize();
+ // two array headers mem size
+ size += NUM_BYTES_ARRAY_HEADER * 2;
+ // Object references size in ArrayList
+ size += NUM_BYTES_OBJECT_REF * 2;
return size;
}
