This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ty/TableModelGrammar by this
push:
new 532e7430426 [Table Model] Implement table device cache (#12702)
532e7430426 is described below
commit 532e74304265775af5ca93d96edbc98fe250530a
Author: Marcos_Zyk <[email protected]>
AuthorDate: Tue Jun 11 16:47:24 2024 +0800
[Table Model] Implement table device cache (#12702)
---
.../fetcher/cache/TableDeviceCacheEntry.java | 55 ++++++++++++
.../metadata/fetcher/cache/TableDeviceId.java | 62 +++++++++++++
.../fetcher/cache/TableDeviceSchemaCache.java | 86 ++++++++++++++++++
.../relational/metadata/fetcher/cache/TableId.java | 67 ++++++++++++++
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 100 +++++++++++++++++++++
.../apache/iotdb/commons/schema/MemUsageUtil.java | 2 +-
6 files changed, 371 insertions(+), 1 deletion(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
new file mode 100644
index 00000000000..9c75010451d
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
+
+import org.apache.iotdb.commons.schema.MemUsageUtil;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class TableDeviceCacheEntry {
+
+ private final Map<String, String> attributeMap;
+
+ public TableDeviceCacheEntry() {
+ attributeMap = new ConcurrentHashMap<>();
+ }
+
+ public TableDeviceCacheEntry(Map<String, String> attributeMap) {
+ this.attributeMap = new HashMap<>(attributeMap);
+ }
+
+ public String getAttribute(String key) {
+ return attributeMap.get(key);
+ }
+
+ public Map<String, String> getAttributeMap() {
+ return attributeMap;
+ }
+
+ public int estimateSize() {
+ int size = 8; // map reference
+ for (Map.Entry<String, String> entry : attributeMap.entrySet()) {
+ size += (int) MemUsageUtil.computeKVMemUsageInMap(entry.getKey(),
entry.getValue());
+ }
+ return size;
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceId.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceId.java
new file mode 100644
index 00000000000..210e22ad672
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceId.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
+
+import org.apache.iotdb.commons.schema.MemUsageUtil;
+
+import java.util.Arrays;
+
+public class TableDeviceId {
+
+ private final String[] idValues;
+
+ public TableDeviceId(String[] idValues) {
+ this.idValues = idValues;
+ }
+
+ public String getIdValue(int index) {
+ return idValues[index];
+ }
+
+ public String[] getIdValues() {
+ return idValues;
+ }
+
+ public int estimateSize() {
+ int size = 8 + 8 + 8 + 4; // object header + reference + String[] header +
String.length
+ for (String node : idValues) {
+ size += (int) MemUsageUtil.computeStringMemUsage(node);
+ }
+ return size;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof TableDeviceId)) return false;
+ TableDeviceId that = (TableDeviceId) o;
+ return Arrays.equals(idValues, that.idValues);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(idValues);
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java
new file mode 100644
index 00000000000..f7c09001c08
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.IDualKeyCache;
+import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCacheBuilder;
+import
org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy;
+
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class TableDeviceSchemaCache {
+
+ private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+
+ private final IDualKeyCache<TableId, TableDeviceId, TableDeviceCacheEntry>
dualKeyCache;
+
+ private final ReentrantReadWriteLock readWriteLock = new
ReentrantReadWriteLock(false);
+
+ public TableDeviceSchemaCache() {
+ DualKeyCacheBuilder<TableId, TableDeviceId, TableDeviceCacheEntry>
dualKeyCacheBuilder =
+ new DualKeyCacheBuilder<>();
+ dualKeyCache =
+ dualKeyCacheBuilder
+ .cacheEvictionPolicy(
+
DualKeyCachePolicy.valueOf(config.getDataNodeSchemaCacheEvictionPolicy()))
+ .memoryCapacity(config.getAllocateMemoryForSchemaCache())
+ .firstKeySizeComputer(TableId::estimateSize)
+ .secondKeySizeComputer(TableDeviceId::estimateSize)
+ .valueSizeComputer(TableDeviceCacheEntry::estimateSize)
+ .build();
+ }
+
+ public Map<String, String> getDeviceAttribute(
+ String database, String tableName, String[] deviceId) {
+ readWriteLock.readLock().lock();
+ try {
+ TableDeviceCacheEntry entry =
+ dualKeyCache.get(new TableId(database, tableName), new
TableDeviceId(deviceId));
+ return entry == null ? null : entry.getAttributeMap();
+ } finally {
+ readWriteLock.readLock().unlock();
+ }
+ }
+
+ public void put(
+ String database, String tableName, String[] deviceId, Map<String,
String> attributeMap) {
+ readWriteLock.readLock().lock();
+ try {
+ dualKeyCache.put(
+ new TableId(database, tableName),
+ new TableDeviceId(deviceId),
+ new TableDeviceCacheEntry(attributeMap));
+ } finally {
+ readWriteLock.readLock().unlock();
+ }
+ }
+
+ public void invalidate(String database, String tableName) {
+ readWriteLock.writeLock().lock();
+ try {
+ dualKeyCache.invalidateAll();
+ } finally {
+ readWriteLock.writeLock().unlock();
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableId.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableId.java
new file mode 100644
index 00000000000..288ecbe57ac
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableId.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
+
+import org.apache.iotdb.commons.schema.MemUsageUtil;
+
+import java.util.Objects;
+
+public class TableId {
+
+ private final String database;
+
+ private final String tableName;
+
+ public TableId(String database, String tableName) {
+ this.database = database;
+ this.tableName = tableName;
+ }
+
+ public String getDatabase() {
+ return database;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public int estimateSize() {
+ return 8
+ + 8
+ + 8
+ + (int)
+ (MemUsageUtil.computeStringMemUsage(database)
+ + MemUsageUtil.computeStringMemUsage(tableName));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof TableId)) return false;
+ TableId tableId = (TableId) o;
+ return Objects.equals(database, tableId.database)
+ && Objects.equals(tableName, tableId.tableName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(database, tableName);
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java
new file mode 100644
index 00000000000..1c092638193
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class TableDeviceSchemaCacheTest {
+
+ private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+
+ private long originMemConfig;
+
+ @Before
+ public void setup() {
+ originMemConfig = config.getAllocateMemoryForSchemaCache();
+ config.setAllocateMemoryForSchemaCache(1500L);
+ }
+
+ @After
+ public void rollback() {
+ config.setAllocateMemoryForSchemaCache(originMemConfig);
+ }
+
+ @Test
+ public void testDeviceCache() {
+ TableDeviceSchemaCache cache = new TableDeviceSchemaCache();
+
+ String database = "db";
+ String table1 = "t1";
+
+ Map<String, String> attributeMap = new HashMap<>();
+ attributeMap.put("type", "new");
+ attributeMap.put("cycle", "monthly");
+ cache.put(database, table1, new String[] {"hebei", "p_1", "d_0"}, new
HashMap<>(attributeMap));
+ Assert.assertEquals(
+ attributeMap,
+ cache.getDeviceAttribute(database, table1, new String[] {"hebei",
"p_1", "d_0"}));
+ Assert.assertNull(
+ cache.getDeviceAttribute(database, table1, new String[] {"hebei",
"p_1", "d_1"}));
+
+ attributeMap.put("type", "old");
+ cache.put(database, table1, new String[] {"hebei", "p_1", "d_1"}, new
HashMap<>(attributeMap));
+ Assert.assertEquals(
+ attributeMap,
+ cache.getDeviceAttribute(database, table1, new String[] {"hebei",
"p_1", "d_1"}));
+
+ attributeMap.put("cycle", "daily");
+ cache.put(
+ database, table1, new String[] {"shandong", "p_1", "d_1"}, new
HashMap<>(attributeMap));
+ Assert.assertNull(
+ cache.getDeviceAttribute(database, table1, new String[] {"hebei",
"p_1", "d_0"}));
+ Assert.assertEquals(
+ attributeMap,
+ cache.getDeviceAttribute(database, table1, new String[] {"shandong",
"p_1", "d_1"}));
+
+ String table2 = "t1";
+ attributeMap.put("type", "new");
+ attributeMap.put("cycle", "monthly");
+ cache.put(database, table2, new String[] {"hebei", "p_1", "d_0"}, new
HashMap<>(attributeMap));
+ Assert.assertEquals(
+ attributeMap,
+ cache.getDeviceAttribute(database, table2, new String[] {"hebei",
"p_1", "d_0"}));
+ Assert.assertNull(
+ cache.getDeviceAttribute(database, table1, new String[] {"hebei",
"p_1", "d_1"}));
+
+ attributeMap.put("type", "old");
+ cache.put(database, table2, new String[] {"hebei", "p_1", "d_1"}, new
HashMap<>(attributeMap));
+ Assert.assertEquals(
+ attributeMap,
+ cache.getDeviceAttribute(database, table2, new String[] {"hebei",
"p_1", "d_1"}));
+ Assert.assertNull(
+ cache.getDeviceAttribute(database, table1, new String[] {"shandong",
"p_1", "d_1"}));
+ }
+}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java
index 530b9a790c2..afc25156dfb 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java
@@ -56,6 +56,6 @@ public class MemUsageUtil {
*/
private static long estimateStringSize(String string) {
// each char takes 2B in Java
- return string == null ? 0 : 32 + 2 * string.length();
+ return string == null ? 0 : 32 + 2L * string.length();
}
}
