This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new be7bd7e5 Added memory calculation for tablet (#636)
be7bd7e5 is described below
commit be7bd7e56c4c609f529dd8b396bd0d52213c594f
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 13 11:15:43 2025 +0800
Added memory calculation for tablet (#636)
* fx
* fix
* dl
---
.../org/apache/tsfile/utils/RamUsageEstimator.java | 76 ++++++++++++++++++
.../org/apache/tsfile/write/record/Tablet.java | 89 +++++++++++++++++++---
2 files changed, 154 insertions(+), 11 deletions(-)
diff --git
a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
index af7a8cff..29b9d085 100644
--- a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
+++ b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java
@@ -27,6 +27,7 @@ import java.security.AccessController;
import java.security.PrivilegedAction;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -35,6 +36,7 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -99,6 +101,11 @@ public final class RamUsageEstimator {
/** Sizes of primitive classes. */
public static final Map<Class<?>, Integer> primitiveSizes;
+ public static final long LOCAL_DATE_ARRAY_SIZE;
+ public static final long LOCAL_DATE_SIZE;
+ public static final long BIT_MAP_SIZE;
+ public static final long SIZE_OF_ARRAYLIST;
+
static {
Map<Class<?>, Integer> primitiveSizesMap = new IdentityHashMap<>();
primitiveSizesMap.put(boolean.class, 1);
@@ -111,6 +118,11 @@ public final class RamUsageEstimator {
primitiveSizesMap.put(long.class, Long.BYTES);
primitiveSizes = Collections.unmodifiableMap(primitiveSizesMap);
+
+ LOCAL_DATE_ARRAY_SIZE = RamUsageEstimator.shallowSizeOf(LocalDate[].class);
+ LOCAL_DATE_SIZE = RamUsageEstimator.shallowSizeOf(LocalDate.class);
+ BIT_MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(BitMap.class);
+ SIZE_OF_ARRAYLIST =
RamUsageEstimator.shallowSizeOfInstance(ArrayList.class);
}
/** JVMs typically cache small longs. This tries to find out what the range
is. */
@@ -654,4 +666,68 @@ public final class RamUsageEstimator {
public static long sizeOfObjectArray(int length) {
return alignObjectSize(NUM_BYTES_ARRAY_HEADER + (long)
NUM_BYTES_OBJECT_REF * length);
}
+
+ private static long sizeOf(final Binary binary) {
+ return Objects.nonNull(binary) ? binary.ramBytesUsed() : 0L;
+ }
+
+ public static long sizeOf(final Binary[] binaries) {
+ if (binaries == null) {
+ return 0L;
+ }
+
+ long size = 0L;
+ for (Binary binary : binaries) {
+ size += sizeOf(binary);
+ }
+
+ return size + RamUsageEstimator.shallowSizeOf(binaries);
+ }
+
+ public static long sizeOfStringArray(final String[] values) {
+ return Objects.nonNull(values) ? RamUsageEstimator.sizeOf(values) : 0L;
+ }
+
+ public static long sizeOf(BitMap[] bitMaps) {
+ if (bitMaps == null) {
+ return 0L;
+ }
+ long size =
+ RamUsageEstimator.alignObjectSize(
+ NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF *
bitMaps.length);
+ for (BitMap bitMap : bitMaps) {
+ size += sizeOf(bitMap);
+ }
+ return size;
+ }
+
+ private static long sizeOf(final BitMap bitMap) {
+ if (bitMap == null) {
+ return 0L;
+ }
+ long size = BIT_MAP_SIZE;
+
+ size +=
+ RamUsageEstimator.alignObjectSize(NUM_BYTES_ARRAY_HEADER +
bitMap.getByteArray().length);
+ return size;
+ }
+
+ public static long sizeOf(final LocalDate[] input) {
+ if (Objects.isNull(input)) {
+ return 0;
+ }
+ long size =
+ RamUsageEstimator.alignObjectSize(
+ LOCAL_DATE_ARRAY_SIZE + (long) input.length *
NUM_BYTES_OBJECT_REF);
+ for (final LocalDate date : input) {
+ if (Objects.nonNull(date)) {
+ size += LOCAL_DATE_SIZE;
+ }
+ }
+ return size;
+ }
+
+ public static long shallowSizeOfList(final List<?> input) {
+ return alignObjectSize(SIZE_OF_ARRAYLIST + (long) input.size() *
NUM_BYTES_OBJECT_REF);
+ }
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index 8336d2b7..89b6f2a6 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -27,11 +27,13 @@ import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.utils.Accountable;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.UnSupportedDataTypeException;
import org.apache.tsfile.write.schema.IMeasurementSchema;
@@ -48,6 +50,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
+import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfList;
+
/**
* A tablet data of one device, the tablet contains multiple measurements of
this device that share
* the same time column.
@@ -59,8 +63,8 @@ import java.util.Objects;
* <p>Notice: The tablet should not have empty cell, please use BitMap to
denote null value
*/
@SuppressWarnings("SuspiciousSystemArraycopy")
-public class Tablet {
-
+public class Tablet implements Accountable {
+ private static final long TABLET_SIZE =
RamUsageEstimator.shallowSizeOfInstance(Tablet.class);
private static final int DEFAULT_SIZE = 1024;
private static final String NOT_SUPPORT_DATATYPE = "Data type %s is not
supported.";
private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1);
@@ -78,7 +82,7 @@ public class Tablet {
private List<ColumnCategory> columnCategories;
/** Columns in the list are all ID columns. */
- private List<Integer> idColumnIndexes = new ArrayList<>();
+ private final List<Integer> tagColumnIndexes = new ArrayList<>();
/** MeasurementId->indexOf({@link MeasurementSchema}) */
private final Map<String, Integer> measurementIndex;
@@ -869,8 +873,7 @@ public class Tablet {
Object[] values = new Object[schemaSize];
boolean isValuesNotNull =
BytesUtils.byteToBool(ReadWriteIOUtils.readByte(byteBuffer));
if (isValuesNotNull) {
- values =
- readTabletValuesFromBuffer(byteBuffer, dataTypes, columnCategories,
schemaSize, rowSize);
+ values = readvaluesFromBuffer(byteBuffer, dataTypes, columnCategories,
schemaSize, rowSize);
}
Tablet tablet =
@@ -899,7 +902,7 @@ public class Tablet {
* @param columns column number
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
- public static Object[] readTabletValuesFromBuffer(
+ public static Object[] readvaluesFromBuffer(
ByteBuffer byteBuffer,
TSDataType[] types,
List<ColumnCategory> columnCategories,
@@ -1237,10 +1240,10 @@ public class Tablet {
*/
@TableModel
public IDeviceID getDeviceID(int i) {
- String[] idArray = new String[idColumnIndexes.size() + 1];
+ String[] idArray = new String[tagColumnIndexes.size() + 1];
idArray[0] = getTableName();
- for (int j = 0; j < idColumnIndexes.size(); j++) {
- final Object value = getValue(i, idColumnIndexes.get(j));
+ for (int j = 0; j < tagColumnIndexes.size(); j++) {
+ final Object value = getValue(i, tagColumnIndexes.get(j));
idArray[j + 1] = value != null ? value.toString() : null;
}
return new StringArrayDeviceID(idArray);
@@ -1248,11 +1251,11 @@ public class Tablet {
public void setColumnCategories(List<ColumnCategory> columnCategories) {
this.columnCategories = columnCategories;
- idColumnIndexes.clear();
+ tagColumnIndexes.clear();
for (int i = 0; i < columnCategories.size(); i++) {
ColumnCategory columnCategory = columnCategories.get(i);
if (columnCategory.equals(ColumnCategory.TAG)) {
- idColumnIndexes.add(i);
+ tagColumnIndexes.add(i);
}
}
}
@@ -1493,4 +1496,68 @@ public class Tablet {
}
}
}
+
+ @Override
+ public long ramBytesUsed() {
+ long totalSizeInBytes =
+ TABLET_SIZE
+ + RamUsageEstimator.sizeOf(insertTargetName)
+ + RamUsageEstimator.sizeOf(timestamps)
+ + shallowSizeOfList(columnCategories)
+ + RamUsageEstimator.sizeOf(bitMaps)
+ + RamUsageEstimator.shallowSizeOfList(tagColumnIndexes)
+ + RamUsageEstimator.sizeOfMap(measurementIndex);
+
+ // values
+ final List<IMeasurementSchema> timeSeries = schemas;
+
+ if (timeSeries != null) {
+ totalSizeInBytes += RamUsageEstimator.shallowSizeOfList(timeSeries);
+ for (int column = 0; column < timeSeries.size(); column++) {
+ final IMeasurementSchema measurementSchema = timeSeries.get(column);
+ if (measurementSchema == null) {
+ continue;
+ }
+ // Measurement schema size
+ totalSizeInBytes += 75;
+
+ final TSDataType tsDataType = measurementSchema.getType();
+ if (tsDataType == null) {
+ continue;
+ }
+
+ if (values == null || values.length <= column) {
+ continue;
+ }
+ switch (tsDataType) {
+ case INT64:
+ case TIMESTAMP:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((long[])
values[column]);
+ break;
+ case DATE:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((LocalDate[])
values[column]);
+ break;
+ case INT32:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((int[])
values[column]);
+ break;
+ case DOUBLE:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((double[])
values[column]);
+ break;
+ case FLOAT:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((float[])
values[column]);
+ break;
+ case BOOLEAN:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((boolean[])
values[column]);
+ break;
+ case STRING:
+ case TEXT:
+ case BLOB:
+ totalSizeInBytes += RamUsageEstimator.sizeOf((Binary[])
values[column]);
+ break;
+ }
+ }
+ }
+
+ return totalSizeInBytes;
+ }
}
