This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch preallocate_array_list
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/preallocate_array_list by this
push:
new eb46237e8b1 Revert "add memtable cache"
eb46237e8b1 is described below
commit eb46237e8b13fe389b61599c394316ce858677fa
Author: Tian Jiang <[email protected]>
AuthorDate: Wed Apr 23 09:44:53 2025 +0800
Revert "add memtable cache"
This reverts commit e702afe02d17af7ca64d153390e6aecdaf503957.
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 ----------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 25 ++++++++++------------
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 17 +++++++--------
.../db/storageengine/dataregion/DataRegion.java | 21 ------------------
.../dataregion/memtable/AbstractMemTable.java | 8 -------
.../memtable/AbstractWritableMemChunk.java | 16 ++++++--------
.../memtable/AlignedWritableMemChunk.java | 15 +------------
.../memtable/AlignedWritableMemChunkGroup.java | 5 -----
.../dataregion/memtable/IMemTable.java | 3 ---
.../dataregion/memtable/IWritableMemChunk.java | 3 ---
.../memtable/IWritableMemChunkGroup.java | 3 ---
.../dataregion/memtable/TsFileProcessor.java | 15 +++++--------
.../dataregion/memtable/WritableMemChunk.java | 15 +------------
.../dataregion/memtable/WritableMemChunkGroup.java | 7 ------
14 files changed, 32 insertions(+), 140 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index d3594d03a37..b5d5d186953 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1154,9 +1154,6 @@ public class IoTDBConfig {
private boolean enableGCThrottle = false;
private int gcThrottleTimeMs = 1000;
- private boolean enableMemTableCache = true;
- private int maxCachedMemTablePerRegion = 1;
-
IoTDBConfig() {}
public int getMaxLogEntriesNumPerBatch() {
@@ -4101,20 +4098,4 @@ public class IoTDBConfig {
public void setEnableGCThrottle(boolean enableGCThrottle) {
this.enableGCThrottle = enableGCThrottle;
}
-
- public boolean isEnableMemTableCache() {
- return enableMemTableCache;
- }
-
- public void setEnableMemTableCache(boolean enableMemTableCache) {
- this.enableMemTableCache = enableMemTableCache;
- }
-
- public int getMaxCachedMemTablePerRegion() {
- return maxCachedMemTablePerRegion;
- }
-
- public void setMaxCachedMemTablePerRegion(int maxCachedMemTablePerRegion) {
- this.maxCachedMemTablePerRegion = maxCachedMemTablePerRegion;
- }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 702807607d9..e6e5377009d 100755
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1074,23 +1074,20 @@ public class IoTDBDescriptor {
"detail_container_min_degrade_memory_in_bytes",
String.valueOf(conf.getDetailContainerMinDegradeMemoryInBytes()))));
- conf.setEnableGCThrottle(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_gc_throttle",
String.valueOf(conf.isEnableGCThrottle()))));
+ conf.setEnableGCThrottle(Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_gc_throttle",
+ String.valueOf(conf.isEnableGCThrottle())
+ )
+ ));
conf.setGcThrottleTimeMs(
Integer.parseInt(
properties.getProperty(
- "gc_throttle_time_ms",
String.valueOf(conf.getGcThrottleTimeMs()))));
-
- conf.setEnableMemTableCache(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_memtable_cache",
String.valueOf(conf.isEnableGCThrottle()))));
- conf.setMaxCachedMemTablePerRegion(
- Integer.parseInt(
- properties.getProperty(
- "max_cached_memtable_per_region",
String.valueOf(conf.getGcThrottleTimeMs()))));
+ "gc_throttle_time_ms",
+ String.valueOf(conf.getGcThrottleTimeMs())
+ )
+ )
+ );
loadIoTConsensusProps(properties);
loadIoTConsensusV2Props(properties);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
index 6f72f46ae08..e4b9f4514dc 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.protocol.thrift.impl;
+import java.util.Random;
import org.apache.iotdb.common.rpc.thrift.TAggregationType;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
@@ -214,7 +215,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
-import java.util.Random;
import java.util.concurrent.TimeUnit;
import static org.apache.iotdb.commons.partition.DataPartition.NOT_ASSIGNED;
@@ -3065,7 +3065,6 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
}
private static final Random throttleRandom = new Random();
-
public static void gcThrottle() {
if (!IoTDBDescriptor.getInstance().getConfig().isEnableGCThrottle()) {
return;
@@ -3073,13 +3072,13 @@ public class ClientRPCServiceImpl implements
IClientRPCServiceWithHandler {
int minimumGcTimePercentToThrottle = 20;
long gcTimePercentage =
JvmGcMonitorMetrics.getInstance().getGcData().getGcTimePercentage();
// if gc time is above 20%, pause the request with the probability equals
to gc time percent
- if (gcTimePercentage > minimumGcTimePercentToThrottle
- && throttleRandom.nextInt(100) < gcTimePercentage) {
- try {
-
Thread.sleep(IoTDBDescriptor.getInstance().getConfig().getGcThrottleTimeMs());
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ if (gcTimePercentage > minimumGcTimePercentToThrottle &&
throttleRandom.nextInt(100) < gcTimePercentage) {
+ try {
+
Thread.sleep(IoTDBDescriptor.getInstance().getConfig().getGcThrottleTimeMs());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
}
- }
+
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 90a03098501..8880f4f1723 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -176,8 +176,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
@@ -335,10 +333,6 @@ public class DataRegion implements IDataRegionForQuery {
private ILoadDiskSelector ordinaryLoadDiskSelector;
private ILoadDiskSelector pipeAndIoTV2LoadDiskSelector;
- private BlockingQueue<IMemTable> memTableCache =
- new ArrayBlockingQueue<>(
-
IoTDBDescriptor.getInstance().getConfig().getMaxCachedMemTablePerRegion());
-
/**
* Construct a database processor.
*
@@ -3948,19 +3942,4 @@ public class DataRegion implements IDataRegionForQuery {
return DataNodeTTLCache.getInstance().getTTLForTable(databaseName,
insertNode.getTableName());
}
}
-
- public IMemTable getCachedMemTable() {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableMemTableCache()) {
- return memTableCache.poll();
- } else {
- return null;
- }
- }
-
- @SuppressWarnings({"ResultOfMethodCallIgnored", "java:S899"})
- public void cacheMemTable(IMemTable memTable) {
- if (IoTDBDescriptor.getInstance().getConfig().isEnableMemTableCache()) {
- memTableCache.offer(memTable);
- }
- }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index a664dba57cd..d550781858f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -411,14 +411,6 @@ public abstract class AbstractMemTable implements
IMemTable {
@Override
public void clear() {
memTableMap.clear();
- reset();
- }
-
- @Override
- public void reset() {
- for (IWritableMemChunkGroup value : memTableMap.values()) {
- value.reset();
- }
memSize = 0;
seriesNumber = 0;
totalPointsNum = 0;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
index cdcc24978d7..ac71e3da6bb 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java
@@ -50,13 +50,14 @@ public abstract class AbstractWritableMemChunk implements
IWritableMemChunk {
* method must ensure success because it's part of flushing.
*
* @param tvList
- * @return whether the TvList is cleared (exclusively held by the MemTable)
*/
- protected boolean maybeReleaseTvList(TVList tvList) {
+ protected void maybeReleaseTvList(TVList tvList) {
long startTimeInMs = System.currentTimeMillis();
- while (true) {
+ boolean succeed = false;
+ while (!succeed) {
try {
- return tryReleaseTvList(tvList);
+ tryReleaseTvList(tvList);
+ succeed = true;
} catch (MemoryNotEnoughException ex) {
long waitQueryInMs = System.currentTimeMillis() - startTimeInMs;
if (waitQueryInMs > MAX_WAIT_QUERY_MS) {
@@ -87,15 +88,11 @@ public abstract class AbstractWritableMemChunk implements
IWritableMemChunk {
}
}
- /**
- * @return whether the list is cleared.
- */
- private boolean tryReleaseTvList(TVList tvList) {
+ private void tryReleaseTvList(TVList tvList) {
tvList.lockQueryList();
try {
if (tvList.getQueryContextSet().isEmpty()) {
tvList.clear();
- return true;
} else {
QueryContext firstQuery =
tvList.getQueryContextSet().iterator().next();
// transfer memory from write process to read process. Here it
reserves read memory and
@@ -107,7 +104,6 @@ public abstract class AbstractWritableMemChunk implements
IWritableMemChunk {
}
// update current TVList owner to first query in the list
tvList.setOwnerQuery(firstQuery);
- return false;
}
} finally {
tvList.unlockQueryList();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 5c63b317215..d20866d0c79 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -697,25 +697,12 @@ public class AlignedWritableMemChunk extends
AbstractWritableMemChunk {
@Override
public void release() {
- boolean listCleared = maybeReleaseTvList(list);
- if (!listCleared) {
- // if the list is not cleared, we cannot reuse it
- list = null;
- }
+ maybeReleaseTvList(list);
for (AlignedTVList alignedTvList : sortedList) {
maybeReleaseTvList(alignedTvList);
}
}
- @Override
- public void reset() {
- sortedList.clear();
- if (list == null) {
- list = AlignedTVList.newAlignedList(dataTypes);
- }
- sortedRowCount = 0;
- }
-
@Override
public long getFirstPoint() {
if (rowCount() == 0) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
index f15fffa62e2..36acdedf9bc 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
@@ -72,11 +72,6 @@ public class AlignedWritableMemChunkGroup implements
IWritableMemChunkGroup {
memChunk.release();
}
- @Override
- public void reset() {
- memChunk.reset();
- }
-
@Override
public long count() {
return memChunk.count();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
index 91fb831d955..44098b69a56 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java
@@ -140,9 +140,6 @@ public interface IMemTable extends WALEntryValue {
/** putBack all the memory resources. */
void clear();
- /** Prepare the memtable for new insertions, keeping the inner structures
for reuse. */
- void reset();
-
boolean isEmpty();
/**
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
index 2f2e819c9e8..13f09a05d8a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
@@ -116,9 +116,6 @@ public interface IWritableMemChunk extends WALEntryValue {
void release();
- /** Prepare the chunk for the next round of insertion. */
- void reset();
-
long getFirstPoint();
long getLastPoint();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
index f46c7ebd8b9..62bf7700218 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
@@ -44,9 +44,6 @@ public interface IWritableMemChunkGroup extends WALEntryValue
{
void release();
- /** Prepare the chunk group for the next round of insertion. */
- void reset();
-
long count();
boolean contains(String measurement);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index ce9cdf5bd23..861b72c923a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -453,14 +453,11 @@ public class TsFileProcessor {
}
private void createNewWorkingMemTable() {
- workMemTable = dataRegionInfo.getDataRegion().getCachedMemTable();
- if (workMemTable == null) {
- workMemTable =
- MemTableManager.getInstance()
- .getAvailableMemTable(
- dataRegionInfo.getDataRegion().getDatabaseName(),
- dataRegionInfo.getDataRegion().getDataRegionId());
- }
+ workMemTable =
+ MemTableManager.getInstance()
+ .getAvailableMemTable(
+ dataRegionInfo.getDataRegion().getDatabaseName(),
+ dataRegionInfo.getDataRegion().getDataRegionId());
walNode.onMemTableCreated(workMemTable, tsFileResource.getTsFilePath());
}
@@ -1635,8 +1632,6 @@ public class TsFileProcessor {
for (FlushListener flushListener : flushListeners) {
flushListener.onMemTableFlushed(memTableToFlush);
}
- memTableToFlush.reset();
- dataRegionInfo.getDataRegion().cacheMemTable(memTableToFlush);
// Retry to avoid unnecessary read-only mode
int retryCnt = 0;
while (shouldClose && flushingMemTables.isEmpty() && writer != null) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index 89ee4445dbd..1f4078dfc7d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -504,25 +504,12 @@ public class WritableMemChunk extends
AbstractWritableMemChunk {
@Override
public void release() {
- boolean listCleared = maybeReleaseTvList(list);
- if (!listCleared) {
- // the list cannot be reused if not cleared
- list = null;
- }
+ maybeReleaseTvList(list);
for (TVList tvList : sortedList) {
maybeReleaseTvList(tvList);
}
}
- @Override
- public void reset() {
- sortedRowCount = 0;
- sortedList.clear();
- if (list == null) {
- list = TVList.newList(schema.getType());
- }
- }
-
@Override
public int serializedSize() {
int serializedSize = schema.serializedSize() + list.serializedSize();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
index 762ca3ed13c..854ca9c8d2c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
@@ -80,13 +80,6 @@ public class WritableMemChunkGroup implements
IWritableMemChunkGroup {
}
}
- @Override
- public void reset() {
- for (IWritableMemChunk memChunk : memChunkMap.values()) {
- memChunk.reset();
- }
- }
-
@Override
public long count() {
long count = 0;
