This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new e807770f Support Object type (#613)
e807770f is described below
commit e807770f379470c9e82d32bb33f35c492df285d2
Author: Haonan <[email protected]>
AuthorDate: Thu Oct 30 11:58:13 2025 +0800
Support Object type (#613)
---
.../java/org/apache/tsfile/enums/TSDataType.java | 31 ++++-
.../org/apache/tsfile/utils/TsPrimitiveType.java | 2 +
.../apache/tsfile/common/conf/TSFileConfig.java | 2 +
.../tsfile/encoding/encoder/PlainEncoder.java | 1 +
.../tsfile/file/metadata/TimeseriesMetadata.java | 30 +++--
.../tsfile/file/metadata/enums/TSEncoding.java | 2 +
.../file/metadata/statistics/ObjectStatistics.java | 126 +++++++++++++++++++++
.../file/metadata/statistics/Statistics.java | 4 +
.../apache/tsfile/read/TsFileSequenceReader.java | 61 ++++++----
.../org/apache/tsfile/read/common/BatchData.java | 6 +
.../java/org/apache/tsfile/read/common/Chunk.java | 2 +
.../tsfile/read/common/DescReadWriteBatchData.java | 3 +
.../java/org/apache/tsfile/read/common/Field.java | 8 ++
.../apache/tsfile/read/common/block/TsBlock.java | 1 +
.../tsfile/read/common/block/TsBlockBuilder.java | 2 +
.../read/common/block/column/ColumnFactory.java | 1 +
.../read/common/block/column/NullColumn.java | 1 +
.../apache/tsfile/read/common/type/ObjectType.java | 79 +++++++++++++
.../apache/tsfile/read/common/type/TypeEnum.java | 3 +-
.../tsfile/read/common/type/TypeFactory.java | 2 +
.../query/dataset/DataSetWithoutTimeGenerator.java | 1 +
.../tsfile/read/reader/TsFileLastReader.java | 16 +--
.../apache/tsfile/read/reader/page/PageReader.java | 2 +
.../tsfile/read/reader/page/ValuePageReader.java | 6 +
.../java/org/apache/tsfile/utils/BytesUtils.java | 21 ++++
.../write/chunk/AlignedChunkGroupWriterImpl.java | 3 +
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 2 +
.../chunk/NonAlignedChunkGroupWriterImpl.java | 1 +
.../org/apache/tsfile/write/record/Tablet.java | 15 +++
.../tsfile/write/record/datapoint/DataPoint.java | 1 +
.../write/v4/AbstractTableModelTsFileWriter.java | 23 +++-
.../tsfile/write/v4/DeviceTableModelWriter.java | 7 ++
.../file/metadata/statistics/StatisticsTest.java | 5 +
.../java/org/apache/tsfile/utils/TypeCastTest.java | 4 +
34 files changed, 428 insertions(+), 46 deletions(-)
diff --git a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
index 492d1bab..4fb01f86 100644
--- a/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
+++ b/java/common/src/main/java/org/apache/tsfile/enums/TSDataType.java
@@ -71,7 +71,10 @@ public enum TSDataType {
BLOB((byte) 10),
/** STRING */
- STRING((byte) 11);
+ STRING((byte) 11),
+
+ /** OBJECT */
+ OBJECT((byte) 12);
private final byte type;
private static final Map<TSDataType, Set<TSDataType>> compatibleTypes;
@@ -139,6 +142,8 @@ public enum TSDataType {
stringCompatibleTypes.add(DATE);
stringCompatibleTypes.add(TIMESTAMP);
compatibleTypes.put(STRING, stringCompatibleTypes);
+
+ compatibleTypes.put(OBJECT, Collections.emptySet());
}
TSDataType(byte type) {
@@ -185,6 +190,8 @@ public enum TSDataType {
return TSDataType.BLOB;
case 11:
return TSDataType.STRING;
+ case 12:
+ return TSDataType.OBJECT;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
@@ -306,6 +313,12 @@ public enum TSDataType {
} else {
break;
}
+ case OBJECT:
+ if (sourceType == TSDataType.OBJECT) {
+ return value;
+ } else {
+ break;
+ }
case VECTOR:
case UNKNOWN:
default:
@@ -447,6 +460,12 @@ public enum TSDataType {
} else {
break;
}
+ case OBJECT:
+ if (sourceType == TSDataType.OBJECT) {
+ return array;
+ } else {
+ break;
+ }
case VECTOR:
case UNKNOWN:
default:
@@ -494,6 +513,7 @@ public enum TSDataType {
case DOUBLE:
case VECTOR:
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
return 8;
@@ -532,6 +552,7 @@ public enum TSDataType {
case BOOLEAN:
case TEXT:
case VECTOR:
+ case OBJECT:
return false;
default:
throw new UnSupportedDataTypeException(this.toString());
@@ -558,6 +579,7 @@ public enum TSDataType {
return true;
case VECTOR:
case BLOB:
+ case OBJECT:
return false;
default:
throw new UnSupportedDataTypeException(this.toString());
@@ -565,7 +587,12 @@ public enum TSDataType {
}
public boolean isBinary() {
- return this == TEXT || this == STRING || this == BLOB;
+ return this == TEXT || this == STRING || this == BLOB || this == OBJECT;
+ }
+
+ // Indicating the statistics don't contain values, such as first, last, min,
max...
+ public boolean hasNoValueInStatistics() {
+ return this == BLOB || this == OBJECT;
}
public static String getDateStringValue(int value) {
diff --git
a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java
b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java
index c3356fc5..4850d1ad 100644
--- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java
+++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java
@@ -48,6 +48,7 @@ public abstract class TsPrimitiveType implements Serializable
{
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return new TsPrimitiveType.TsBinary();
case VECTOR:
return new TsPrimitiveType.TsVector();
@@ -79,6 +80,7 @@ public abstract class TsPrimitiveType implements Serializable
{
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return new TsPrimitiveType.TsBinary((Binary) v);
case VECTOR:
return new TsPrimitiveType.TsVector((TsPrimitiveType[]) v);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
index eb49b718..201e6f9e 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java
@@ -385,6 +385,7 @@ public class TSFileConfig implements Serializable {
case STRING:
case BLOB:
case TEXT:
+ case OBJECT:
default:
return textEncoding;
}
@@ -413,6 +414,7 @@ public class TSFileConfig implements Serializable {
case STRING:
case BLOB:
case TEXT:
+ case OBJECT:
compressionName = textCompression;
break;
default:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java
b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java
index 09c28cbe..0b97005d 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/PlainEncoder.java
@@ -128,6 +128,7 @@ public class PlainEncoder extends Encoder {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
// refer to encode(Binary,ByteArrayOutputStream)
return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength;
default:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
index 25b1a127..544489ed 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java
@@ -125,7 +125,9 @@ public class TimeseriesMetadata implements
ITimeSeriesMetadata {
}
public static TimeseriesMetadata deserializeFrom(
- ByteBuffer buffer, boolean needChunkMetadataForNonBlob, boolean
needChunkMetadataForBlob) {
+ ByteBuffer buffer,
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) {
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(buffer));
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer));
@@ -133,8 +135,10 @@ public class TimeseriesMetadata implements
ITimeSeriesMetadata {
int chunkMetaDataListDataSize =
ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize);
timeseriesMetaData.setStatistics(Statistics.deserialize(buffer,
timeseriesMetaData.dataType));
- if ((timeseriesMetaData.getTsDataType() != TSDataType.BLOB &&
needChunkMetadataForNonBlob)
- || (timeseriesMetaData.getTsDataType() == TSDataType.BLOB &&
needChunkMetadataForBlob)) {
+ if ((!timeseriesMetaData.getTsDataType().hasNoValueInStatistics()
+ && needChunkMetadataForDataTypeWithValuesInStatistics)
+ || (timeseriesMetaData.getTsDataType().hasNoValueInStatistics()
+ && needChunkMetadataForDataTypeWithoutValuesInStatistics)) {
ByteBuffer byteBuffer = buffer.slice();
byteBuffer.limit(chunkMetaDataListDataSize);
timeseriesMetaData.chunkMetadataList = new ArrayList<>();
@@ -156,8 +160,8 @@ public class TimeseriesMetadata implements
ITimeSeriesMetadata {
public static TimeseriesMetadata deserializeFrom(
TsFileInput tsFileInput,
- boolean needChunkMetadataForNonBlob,
- boolean needChunkMetadataForBlob)
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics)
throws IOException {
InputStream inputStream = tsFileInput.wrapAsInputStream();
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
@@ -169,8 +173,10 @@ public class TimeseriesMetadata implements
ITimeSeriesMetadata {
timeseriesMetaData.setStatistics(
Statistics.deserialize(inputStream, timeseriesMetaData.dataType));
long startOffset = tsFileInput.position();
- if ((timeseriesMetaData.getTsDataType() != TSDataType.BLOB &&
needChunkMetadataForNonBlob)
- || (timeseriesMetaData.getTsDataType() == TSDataType.BLOB &&
needChunkMetadataForBlob)) {
+ if ((!timeseriesMetaData.getTsDataType().hasNoValueInStatistics()
+ && needChunkMetadataForDataTypeWithValuesInStatistics)
+ || (timeseriesMetaData.getTsDataType().hasNoValueInStatistics()
+ && needChunkMetadataForDataTypeWithoutValuesInStatistics)) {
timeseriesMetaData.chunkMetadataList = new ArrayList<>();
while (tsFileInput.position() < startOffset + chunkMetaDataListDataSize)
{
timeseriesMetaData.chunkMetadataList.add(
@@ -196,8 +202,8 @@ public class TimeseriesMetadata implements
ITimeSeriesMetadata {
public static TimeseriesMetadata deserializeFrom(
ByteBuffer buffer,
Set<String> excludedMeasurements,
- boolean needChunkMetadataForNonBlob,
- boolean needChunkMetadataForBlob) {
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) {
byte timeseriesType = ReadWriteIOUtils.readByte(buffer);
String measurementID = ReadWriteIOUtils.readVarIntString(buffer);
TSDataType tsDataType = ReadWriteIOUtils.readDataType(buffer);
@@ -212,8 +218,10 @@ public class TimeseriesMetadata implements
ITimeSeriesMetadata {
timeseriesMetaData.setStatistics(statistics);
if (!excludedMeasurements.contains(measurementID)
- && ((tsDataType != TSDataType.BLOB && needChunkMetadataForNonBlob)
- || (tsDataType == TSDataType.BLOB && needChunkMetadataForBlob))) {
+ && ((!tsDataType.hasNoValueInStatistics()
+ && needChunkMetadataForDataTypeWithValuesInStatistics)
+ || (tsDataType.hasNoValueInStatistics()
+ && needChunkMetadataForDataTypeWithoutValuesInStatistics))) {
// measurement is not in the excluded set and need chunk metadata
ByteBuffer byteBuffer = buffer.slice();
byteBuffer.limit(chunkMetaDataListDataSize);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
index 6c02cb1f..f85c0d5e 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/enums/TSEncoding.java
@@ -96,6 +96,8 @@ public enum TSEncoding {
Set<TSEncoding> blobSet = new HashSet<>();
blobSet.add(TSEncoding.PLAIN);
TYPE_SUPPORTED_ENCODINGS.put(TSDataType.BLOB, blobSet);
+
+ TYPE_SUPPORTED_ENCODINGS.put(TSDataType.OBJECT, blobSet);
}
TSEncoding(byte type) {
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/ObjectStatistics.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/ObjectStatistics.java
new file mode 100644
index 00000000..f054a267
--- /dev/null
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/ObjectStatistics.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.file.metadata.statistics;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.filter.StatisticsClassException;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.RamUsageEstimator;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public class ObjectStatistics extends Statistics<Binary> {
+
+ public static final long INSTANCE_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(ObjectStatistics.class);
+
+ // no statistics for object data type
+
+ @Override
+ public TSDataType getType() {
+ return TSDataType.OBJECT;
+ }
+
+ /** The output of this method should be identical to the method
"serializeStats(outputStream)". */
+ @Override
+ public int getStatsSize() {
+ return 0;
+ }
+
+ @Override
+ public long getRetainedSizeInBytes() {
+ return INSTANCE_SIZE;
+ }
+
+ @Override
+ int serializeStats(OutputStream outputStream) throws IOException {
+ return 0;
+ }
+
+ public void updateStats(Binary value) {
+ if (isEmpty) {
+ isEmpty = false;
+ }
+ }
+
+ @Override
+ public void deserialize(InputStream inputStream) throws IOException {
+ // do nothing
+ }
+
+ @Override
+ public void deserialize(ByteBuffer byteBuffer) {
+ // do nothing
+ }
+
+ @Override
+ public Binary getMinValue() {
+ throw new StatisticsClassException(
+ String.format(STATS_UNSUPPORTED_MSG, TSDataType.OBJECT, "min"));
+ }
+
+ @Override
+ public Binary getMaxValue() {
+ throw new StatisticsClassException(
+ String.format(STATS_UNSUPPORTED_MSG, TSDataType.OBJECT, "max"));
+ }
+
+ @Override
+ public Binary getFirstValue() {
+ throw new StatisticsClassException(
+ String.format(STATS_UNSUPPORTED_MSG, TSDataType.OBJECT, "first"));
+ }
+
+ @Override
+ public Binary getLastValue() {
+ throw new StatisticsClassException(
+ String.format(STATS_UNSUPPORTED_MSG, TSDataType.OBJECT, "last"));
+ }
+
+ @Override
+ public double getSumDoubleValue() {
+ throw new StatisticsClassException(
+ String.format(STATS_UNSUPPORTED_MSG, TSDataType.OBJECT, "sum"));
+ }
+
+ @Override
+ public long getSumLongValue() {
+
+ throw new StatisticsClassException(
+ String.format(STATS_UNSUPPORTED_MSG, TSDataType.OBJECT, "sum"));
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ // do nothing
+ if (isEmpty) {
+ isEmpty = false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "ObjectStatistics{}";
+ }
+}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
index b1c2c463..0378a509 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
@@ -126,6 +126,8 @@ public abstract class Statistics<T extends Serializable> {
return new StringStatistics();
case BLOB:
return new BlobStatistics();
+ case OBJECT:
+ return new ObjectStatistics();
default:
throw new UnknownColumnTypeException(type.toString());
}
@@ -155,6 +157,8 @@ public abstract class Statistics<T extends Serializable> {
return StringStatistics.INSTANCE_SIZE;
case BLOB:
return BlobStatistics.INSTANCE_SIZE;
+ case OBJECT:
+ return ObjectStatistics.INSTANCE_SIZE;
default:
throw new UnknownColumnTypeException(type.toString());
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
index 7a572b9f..bc742dff 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java
@@ -1455,8 +1455,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
IDeviceID deviceId,
MetadataIndexNodeType type,
Map<IDeviceID, List<TimeseriesMetadata>> timeseriesMetadataMap,
- boolean needChunkMetadataForNonBlob,
- boolean needChunkMetadataForBlob)
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics)
throws IOException {
try {
if (type.equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
@@ -1464,7 +1464,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
while (buffer.hasRemaining()) {
timeseriesMetadataList.add(
TimeseriesMetadata.deserializeFrom(
- buffer, needChunkMetadataForNonBlob,
needChunkMetadataForBlob));
+ buffer,
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics));
}
timeseriesMetadataMap
.computeIfAbsent(deviceId, k -> new ArrayList<>())
@@ -1493,8 +1495,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
deviceId,
metadataIndexNode.getNodeType(),
timeseriesMetadataMap,
- needChunkMetadataForNonBlob,
- needChunkMetadataForBlob);
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics);
} else {
// when the buffer length is over than Integer.MAX_VALUE,
// using tsFileInput to get timeseriesMetadataList
@@ -1505,8 +1507,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
deviceId,
metadataIndexNode.getNodeType(),
timeseriesMetadataMap,
- needChunkMetadataForNonBlob,
- needChunkMetadataForBlob);
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics);
}
}
}
@@ -1545,8 +1547,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
IDeviceID deviceId,
MetadataIndexNodeType type,
Map<IDeviceID, List<TimeseriesMetadata>> timeseriesMetadataMap,
- boolean needChunkMetadataForNonBlob,
- boolean needChunkMetadataForBlob)
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics)
throws IOException {
try {
tsFileInput.position(start);
@@ -1555,7 +1557,9 @@ public class TsFileSequenceReader implements
AutoCloseable {
while (tsFileInput.position() < end) {
timeseriesMetadataList.add(
TimeseriesMetadata.deserializeFrom(
- tsFileInput, needChunkMetadataForNonBlob,
needChunkMetadataForBlob));
+ tsFileInput,
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics));
}
timeseriesMetadataMap
.computeIfAbsent(deviceId, k -> new ArrayList<>())
@@ -1582,8 +1586,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
deviceId,
metadataIndexNode.getNodeType(),
timeseriesMetadataMap,
- needChunkMetadataForNonBlob,
- needChunkMetadataForBlob);
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics);
}
}
} catch (StopReadTsFileByInterruptException e) {
@@ -1636,8 +1640,12 @@ public class TsFileSequenceReader implements
AutoCloseable {
}
public Iterator<Pair<IDeviceID, List<TimeseriesMetadata>>>
iterAllTimeseriesMetadata(
- boolean needChunkMetadataForNonBlob, boolean needChunkMetadataForBlob)
throws IOException {
- return new TimeseriesMetadataIterator(needChunkMetadataForNonBlob,
needChunkMetadataForBlob);
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics)
+ throws IOException {
+ return new TimeseriesMetadataIterator(
+ needChunkMetadataForDataTypeWithoutValuesInStatistics,
+ needChunkMetadataForDataTypeWithValuesInStatistics);
}
/* This method will only deserialize the TimeseriesMetadata, not including
chunk metadata list */
@@ -2371,6 +2379,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkStatistics.update(timeStamp, value.getBinary());
break;
default:
@@ -2410,6 +2419,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkStatistics.update(batchData.currentTime(),
batchData.getBinary());
break;
default:
@@ -2640,6 +2650,7 @@ public class TsFileSequenceReader implements
AutoCloseable {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkStatistics.update(batchData.currentTime(),
batchData.getBinary());
break;
default:
@@ -3140,16 +3151,20 @@ public class TsFileSequenceReader implements
AutoCloseable {
implements Iterator<Pair<IDeviceID, List<TimeseriesMetadata>>> {
private final Deque<MetadataIndexNode> nodeStack = new ArrayDeque<>();
- private final boolean needChunkMetadataForNonBlob;
- private final boolean needCHunkMetadataForBlob;
+ private final boolean needChunkMetadataForDataTypeWithValuesInStatistics;
+ private final boolean
needChunkMetadataForDataTypeWithoutValuesInStatistics;
private Pair<IDeviceID, List<TimeseriesMetadata>> nextValue;
private MetadataIndexNode currentLeafDeviceNode;
private int currentLeafDeviceNodeIndex;
public TimeseriesMetadataIterator(
- boolean needChunkMetadataForNonBlob, boolean needChunkMetadataForBlob)
throws IOException {
- this.needChunkMetadataForNonBlob = needChunkMetadataForNonBlob;
- this.needCHunkMetadataForBlob = needChunkMetadataForBlob;
+ boolean needChunkMetadataForDataTypeWithValuesInStatistics,
+ boolean needChunkMetadataForDataTypeWithoutValuesInStatistics)
+ throws IOException {
+ this.needChunkMetadataForDataTypeWithValuesInStatistics =
+ needChunkMetadataForDataTypeWithValuesInStatistics;
+ this.needChunkMetadataForDataTypeWithoutValuesInStatistics =
+ needChunkMetadataForDataTypeWithoutValuesInStatistics;
if (tsFileMetaData == null) {
readFileMetadata();
}
@@ -3245,8 +3260,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
deviceId,
currentLeafDeviceNode.getNodeType(),
nextValueMap,
- needChunkMetadataForNonBlob,
- needCHunkMetadataForBlob);
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics);
} else {
// when the buffer length is over than Integer.MAX_VALUE,
// using tsFileInput to get timeseriesMetadataList
@@ -3257,8 +3272,8 @@ public class TsFileSequenceReader implements
AutoCloseable {
deviceId,
currentLeafDeviceNode.getNodeType(),
nextValueMap,
- needChunkMetadataForNonBlob,
- needCHunkMetadataForBlob);
+ needChunkMetadataForDataTypeWithValuesInStatistics,
+ needChunkMetadataForDataTypeWithoutValuesInStatistics);
}
currentLeafDeviceNodeIndex++;
Entry<IDeviceID, List<TimeseriesMetadata>> entry =
nextValueMap.entrySet().iterator().next();
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java
index 504f7a9c..1a7e080b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/BatchData.java
@@ -144,6 +144,7 @@ public class BatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return getBinary();
case VECTOR:
return getVector();
@@ -169,6 +170,7 @@ public class BatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return new TsBinary(getBinary());
case VECTOR:
return new TsVector(getVector());
@@ -231,6 +233,7 @@ public class BatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
binaryRet = new ArrayList<>();
binaryRet.add(new Binary[capacity]);
break;
@@ -575,6 +578,7 @@ public class BatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
putBinary(t, (Binary) v);
break;
case VECTOR:
@@ -699,6 +703,7 @@ public class BatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
for (int i = 0; i < length(); i++) {
outputStream.writeLong(getTimeByIndex(i));
Binary binary = getBinaryByIndex(i);
@@ -744,6 +749,7 @@ public class BatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary binary = value.getBinary();
outputStream.writeInt(binary.getLength());
outputStream.write(binary.getValues());
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
index a7e2e9ea..c560050d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Chunk.java
@@ -284,6 +284,7 @@ public class Chunk {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
chunkWriter.write(
timestamp,
convertedValue == null ? Binary.EMPTY_VALUE : (Binary)
convertedValue,
@@ -358,6 +359,7 @@ public class Chunk {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
chunkWriter.write(timestamp, (Binary) convertedValue);
break;
default:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java
index 1fd65740..b667c537 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/DescReadWriteBatchData.java
@@ -75,6 +75,7 @@ public class DescReadWriteBatchData extends DescReadBatchData
{
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
binaryRet = new LinkedList<>();
binaryRet.add(new Binary[capacity]);
break;
@@ -440,6 +441,7 @@ public class DescReadWriteBatchData extends
DescReadBatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
for (int i = length() - 1; i >= 0; i--) {
outputStream.writeLong(getTimeByIndex(i));
Binary binary = getBinaryByIndex(i);
@@ -485,6 +487,7 @@ public class DescReadWriteBatchData extends
DescReadBatchData {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary binary = value.getBinary();
outputStream.writeInt(binary.getLength());
outputStream.write(binary.getValues());
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java
index 01d4dcdb..a8de28a7 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/Field.java
@@ -29,6 +29,8 @@ import org.apache.tsfile.write.UnSupportedDataTypeException;
import java.time.LocalDate;
+import static org.apache.tsfile.utils.BytesUtils.parseObjectByteArrayToString;
+
/**
* Field is component of one {@code RowRecord} which stores a value in
specific data type. The value
* type of Field is primitive(int long, float, double, binary, boolean).
@@ -71,6 +73,7 @@ public class Field {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
out.setBinaryV(field.getBinaryV());
break;
default:
@@ -183,6 +186,8 @@ public class Field {
case TEXT:
case STRING:
return binaryV.toString();
+ case OBJECT:
+ return parseObjectByteArrayToString(binaryV.getValues());
case BLOB:
return BytesUtils.parseBlobByteArrayToString(binaryV.getValues());
default:
@@ -216,6 +221,7 @@ public class Field {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return getBinaryV();
default:
throw new UnSupportedDataTypeException(dataType.toString());
@@ -248,6 +254,7 @@ public class Field {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
field.setBinaryV((Binary) value);
break;
default:
@@ -278,6 +285,7 @@ public class Field {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
field.setBinaryV(value.getBinary());
break;
default:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
index b5224580..7e31d5d4 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlock.java
@@ -592,6 +592,7 @@ public class TsBlock {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueColumns[i].isNull()[updateIdx] = false;
valueColumns[i].getBinaries()[updateIdx] =
sourceTsBlock.getValueColumns()[i].getBinary(sourceIndex);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
index 0e96fca5..e06f8c37 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java
@@ -135,6 +135,7 @@ public class TsBlockBuilder {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueColumnBuilders[i] =
new BinaryColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries);
@@ -205,6 +206,7 @@ public class TsBlockBuilder {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueColumnBuilders[i] =
new BinaryColumnBuilder(
tsBlockBuilderStatus.createColumnBuilderStatus(),
initialExpectedEntries);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
index 88a40113..fc9774a1 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
@@ -35,6 +35,7 @@ public class ColumnFactory {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
return new BinaryColumn(initialCapacity);
case INT32:
case DATE:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
index d91359ca..30031822 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/NullColumn.java
@@ -154,6 +154,7 @@ public class NullColumn implements Column {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return new
RunLengthEncodedColumn(BinaryColumnBuilder.NULL_VALUE_BLOCK, positionCount);
default:
throw new IllegalArgumentException("Unknown data type: " + dataType);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java
new file mode 100644
index 00000000..b30b621f
--- /dev/null
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/ObjectType.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.common.type;
+
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder;
+import org.apache.tsfile.utils.Binary;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ObjectType extends AbstractType {
+
+ public static final ObjectType OBJECT = new ObjectType();
+
+ private ObjectType() {}
+
+ @Override
+ public Binary getBinary(Column c, int position) {
+ return c.getBinary(position);
+ }
+
+ @Override
+ public void writeBinary(ColumnBuilder builder, Binary value) {
+ builder.writeBinary(value);
+ }
+
+ @Override
+ public ColumnBuilder createColumnBuilder(int expectedEntries) {
+ return new BinaryColumnBuilder(null, expectedEntries);
+ }
+
+ @Override
+ public TypeEnum getTypeEnum() {
+ return TypeEnum.OBJECT;
+ }
+
+ @Override
+ public String getDisplayName() {
+ return "OBJECT";
+ }
+
+ @Override
+ public boolean isComparable() {
+ return false;
+ }
+
+ @Override
+ public boolean isOrderable() {
+ return false;
+ }
+
+ @Override
+ public List<Type> getTypeParameters() {
+ return Collections.emptyList();
+ }
+
+ public static ObjectType getInstance() {
+ return OBJECT;
+ }
+}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
index 184a64af..52db17a3 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java
@@ -38,5 +38,6 @@ public enum TypeEnum {
TIMESTAMP,
DATE,
BLOB,
- STRING
+ STRING,
+ OBJECT
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
index 41474542..cf23572d 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java
@@ -49,6 +49,8 @@ public class TypeFactory {
return StringType.getInstance();
case BLOB:
return BlobType.getInstance();
+ case OBJECT:
+ return ObjectType.getInstance();
default:
throw new UnsupportedOperationException(
String.format("Invalid TSDataType for TypeFactory: %s",
tsDataType));
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java
index 02f8ffd4..20df8887 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/DataSetWithoutTimeGenerator.java
@@ -179,6 +179,7 @@ public class DataSetWithoutTimeGenerator extends
QueryDataSet {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
field.setBinaryV(col.getBinary());
break;
case VECTOR:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java
index 312b73dd..3957083d 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/TsFileLastReader.java
@@ -58,8 +58,8 @@ public class TsFileLastReader
private final TsFileSequenceReader sequenceReader;
private boolean asyncIO = true;
- // when true, blob series will return a null TimeValuePair
- private boolean ignoreBlob = false;
+ // when true, series without last value statistics will return a null
TimeValuePair
+ private boolean ignoreTypesWithoutStatistics = false;
private Iterator<Pair<IDeviceID, List<TimeseriesMetadata>>>
timeseriesMetadataIter;
private Pair<IDeviceID, List<Pair<String, TimeValuePair>>> nextValue;
@@ -73,13 +73,14 @@ public class TsFileLastReader
/**
* @param filePath path of the TsFile
* @param asyncIO use asynchronous IO or not
- * @param ignoreBlob whether to ignore series with blob type (the returned
TimeValuePair will be
- * null)
+ * @param ignoreTypesWithoutStatistics whether to ignore series with blob
type or object type (the
+ * returned TimeValuePair will be null)
*/
- public TsFileLastReader(String filePath, boolean asyncIO, boolean
ignoreBlob) throws IOException {
+ public TsFileLastReader(String filePath, boolean asyncIO, boolean
ignoreTypesWithoutStatistics)
+ throws IOException {
this(filePath);
this.asyncIO = asyncIO;
- this.ignoreBlob = ignoreBlob;
+ this.ignoreTypesWithoutStatistics = ignoreTypesWithoutStatistics;
}
@Override
@@ -270,7 +271,8 @@ public class TsFileLastReader
}
private void init() throws IOException {
- timeseriesMetadataIter = sequenceReader.iterAllTimeseriesMetadata(false,
!ignoreBlob);
+ timeseriesMetadataIter =
+ sequenceReader.iterAllTimeseriesMetadata(false,
!ignoreTypesWithoutStatistics);
if (asyncIO) {
int queueCapacity = 1024;
lastValueQueue = new ArrayBlockingQueue<>(queueCapacity);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
index 7ba234cf..6d56530c 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/PageReader.java
@@ -196,6 +196,7 @@ public class PageReader implements IPageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary aBinary = valueDecoder.readBinary(valueBuffer);
if (!isDeleted(timestamp)
&& (allSatisfy || recordFilter.satisfyBinary(timestamp,
aBinary))) {
@@ -339,6 +340,7 @@ public class PageReader implements IPageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
while (timeDecoder.hasNext(timeBuffer)) {
long timestamp = timeDecoder.readLong(timeBuffer);
Binary aBinary = valueDecoder.readBinary(valueBuffer);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
index 998b2117..ff34c98b 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java
@@ -159,6 +159,7 @@ public class ValuePageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary aBinary = valueDecoder.readBinary(valueBuffer);
if (!isDeleted(timestamp)
&& (filter == null || filter.satisfyBinary(timestamp, aBinary)))
{
@@ -214,6 +215,7 @@ public class ValuePageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary aBinary = valueDecoder.readBinary(valueBuffer);
if (!isDeleted(timestamp)) {
resultValue = new TsPrimitiveType.TsBinary(aBinary);
@@ -276,6 +278,7 @@ public class ValuePageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary aBinary = valueDecoder.readBinary(valueBuffer);
if (!isDeleted(timeBatch[i])) {
valueBatch[i] = new TsPrimitiveType.TsBinary(aBinary);
@@ -376,6 +379,7 @@ public class ValuePageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary aBinary = valueDecoder.readBinary(valueBuffer);
if (keepCurrentRow[i]) {
if (isDeleted[i]) {
@@ -454,6 +458,7 @@ public class ValuePageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary aBinary = valueDecoder.readBinary(valueBuffer);
if (keepCurrentRow[i]) {
columnBuilder.writeBinary(aBinary);
@@ -577,6 +582,7 @@ public class ValuePageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
// skip useless data
for (int i = 0; i < readStartIndex; i++) {
if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java
b/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java
index 82acc0b4..051c259a 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/BytesUtils.java
@@ -37,6 +37,14 @@ import java.util.List;
*/
public class BytesUtils {
+ private static final String BYTE_OBJECT_STRING_FORMAT = "(Object) %d B";
+ private static final double KB_SIZE = 1024;
+ private static final String KB_OBJECT_STRING_FORMAT = "(Object) %.2f KB";
+ private static final double MB_SIZE = 1024.0 * 1024;
+ private static final String MB_OBJECT_STRING_FORMAT = "(Object) %.2f MB";
+ private static final double GB_SIZE = 1024.0 * 1024 * 1024;
+ private static final String GB_OBJECT_STRING_FORMAT = "(Object) %.2f GB";
+
private BytesUtils() {}
private static final Logger LOG = LoggerFactory.getLogger(BytesUtils.class);
@@ -951,4 +959,17 @@ public class BytesUtils {
}
return hexString.toString();
}
+
+ public static String parseObjectByteArrayToString(byte[] input) {
+ long size = BytesUtils.bytesToLong(input, 8);
+ if (size < KB_SIZE) {
+ return String.format(BYTE_OBJECT_STRING_FORMAT, size);
+ } else if (size < MB_SIZE) {
+ return String.format(KB_OBJECT_STRING_FORMAT, size / KB_SIZE);
+ } else if (size < GB_SIZE) {
+ return String.format(MB_OBJECT_STRING_FORMAT, size / MB_SIZE);
+ } else {
+ return String.format(GB_OBJECT_STRING_FORMAT, size / GB_SIZE);
+ }
+ }
}
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index 3207c409..5fd9dde7 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -191,6 +191,7 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueChunkWriter.write(time, (Binary) point.getValue(), isNull);
break;
default:
@@ -283,6 +284,7 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueChunkWriter.write(time, ((Binary[])
tablet.getValues()[columnIndex])[row], isNull);
break;
default:
@@ -376,6 +378,7 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueChunkWriter.write(-1, null, true);
break;
default:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
index 2ec4bd8f..27761e67 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkWriterImpl.java
@@ -325,6 +325,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter
{
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
writer.write(
time,
point != null ? point.getBinary() : new
Binary("".getBytes(StandardCharsets.UTF_8)),
@@ -369,6 +370,7 @@ public class AlignedChunkWriterImpl implements IChunkWriter
{
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkWriter.write(times, column.getBinaries(), column.isNull(),
batchSize, arrayOffset);
break;
case DOUBLE:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
index 58de2ffb..10e66b9a 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/NonAlignedChunkGroupWriterImpl.java
@@ -163,6 +163,7 @@ public class NonAlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkWriters
.get(measurementId)
.write(time, ((Binary[]) tablet.getValues()[column])[row]);
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index c984b422..8336d2b7 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -557,6 +557,20 @@ public class Tablet {
updateBitMap(rowIndex, columnIndex, false);
}
+ public void addValue(int rowIndex, int columnIndex, boolean isEOF, long
offset, byte[] content) {
+ if (!(values[columnIndex] instanceof Binary[])) {
+ throw new IllegalArgumentException(
+ "The data type of column index " + columnIndex + " is not OBJECT");
+ }
+ final Binary[] sensor = (Binary[]) values[columnIndex];
+ byte[] val = new byte[content.length + 9];
+ val[0] = (byte) (isEOF ? 1 : 0);
+ System.arraycopy(BytesUtils.longToBytes(offset), 0, val, 1, 8);
+ System.arraycopy(content, 0, val, 9, content.length);
+ sensor[rowIndex] = new Binary(val);
+ updateBitMap(rowIndex, columnIndex, false);
+ }
+
private int getColumnIndexByMeasurement(String measurement) {
if (measurement == null) {
throw new IllegalArgumentException("measurement should be non null
value");
@@ -660,6 +674,7 @@ public class Tablet {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valueColumn = new Binary[capacity];
break;
case DATE:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
index 7bd126c5..ed0fbddc 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DataPoint.java
@@ -91,6 +91,7 @@ public abstract class DataPoint {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
dataPoint =
new StringDataPoint(measurementId, new Binary(value,
TSFileConfig.STRING_CHARSET));
break;
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
index 260d4264..2df52df5 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/AbstractTableModelTsFileWriter.java
@@ -32,6 +32,7 @@ import
org.apache.tsfile.write.chunk.NonAlignedChunkGroupWriterImpl;
import org.apache.tsfile.write.chunk.TableChunkGroupWriterImpl;
import org.apache.tsfile.write.schema.Schema;
import org.apache.tsfile.write.writer.TsFileIOWriter;
+import org.apache.tsfile.write.writer.TsFileOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,7 +84,7 @@ abstract class AbstractTableModelTsFileWriter implements
ITsFileWriter {
protected AbstractTableModelTsFileWriter(File file, long
chunkGroupSizeThreshold)
throws IOException {
this(
- file,
+ new TsFileIOWriter(file),
chunkGroupSizeThreshold,
new EncryptParameter(config.getEncryptType(), config.getEncryptKey()));
}
@@ -92,9 +93,27 @@ abstract class AbstractTableModelTsFileWriter implements
ITsFileWriter {
protected AbstractTableModelTsFileWriter(
File file, long chunkGroupSizeThreshold, EncryptParameter
firstEncryptParam)
throws IOException {
+ this(new TsFileIOWriter(file), chunkGroupSizeThreshold, firstEncryptParam);
+ }
+
+ @TsFileApi
+ protected AbstractTableModelTsFileWriter(TsFileOutput output, long
chunkGroupSizeThreshold)
+ throws IOException {
+ this(
+ new TsFileIOWriter(output),
+ chunkGroupSizeThreshold,
+ new EncryptParameter(config.getEncryptType(), config.getEncryptKey()));
+ }
+
+ @TsFileApi
+ protected AbstractTableModelTsFileWriter(
+ TsFileIOWriter tsFileIOWriter,
+ long chunkGroupSizeThreshold,
+ EncryptParameter firstEncryptParam)
+ throws IOException {
Schema schema = new Schema();
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- this.fileWriter = new TsFileIOWriter(file);
+ this.fileWriter = tsFileIOWriter;
fileWriter.setSchema(schema);
this.pageSize = conf.getPageSizeInByte();
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
index b2ae6b1f..0397b42d 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/v4/DeviceTableModelWriter.java
@@ -34,6 +34,7 @@ import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.datapoint.DataPoint;
import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileOutput;
import java.io.File;
import java.io.IOException;
@@ -52,6 +53,12 @@ public class DeviceTableModelWriter extends
AbstractTableModelTsFileWriter {
registerTableSchema(tableSchema);
}
+ public DeviceTableModelWriter(
+ TsFileOutput tsFileOutput, TableSchema tableSchema, long
memoryThreshold) throws IOException {
+ super(tsFileOutput, memoryThreshold);
+ registerTableSchema(tableSchema);
+ }
+
/**
* Write the tablet in to the TsFile with the table-view. The method will
try to split the tablet
* by device.
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java
index 06aeca38..ba12c635 100644
---
a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java
+++
b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java
@@ -108,6 +108,7 @@ public class StatisticsTest {
statistics.getLastValue());
break;
case BLOB:
+ case OBJECT:
break;
default:
throw new IllegalArgumentException(statistics.getType().toString());
@@ -162,6 +163,10 @@ public class StatisticsTest {
BlobStatistics blobStat = new BlobStatistics();
result = blobStat;
break;
+ case OBJECT:
+ ObjectStatistics objectStat = new ObjectStatistics();
+ result = objectStat;
+ break;
case DATE:
DateStatistics dateStat = new DateStatistics();
dateStat.initializeStats(val, val, val, val, val);
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
index 4d1d0006..6398eac7 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/utils/TypeCastTest.java
@@ -42,6 +42,7 @@ public class TypeCastTest {
Collections.addAll(dataTypes, TSDataType.values());
dataTypes.remove(TSDataType.VECTOR);
dataTypes.remove(TSDataType.UNKNOWN);
+ dataTypes.remove(TSDataType.OBJECT);
for (TSDataType from : dataTypes) {
for (TSDataType to : dataTypes) {
@@ -75,6 +76,7 @@ public class TypeCastTest {
Collections.addAll(dataTypes, TSDataType.values());
dataTypes.remove(TSDataType.VECTOR);
dataTypes.remove(TSDataType.UNKNOWN);
+ dataTypes.remove(TSDataType.OBJECT);
for (TSDataType from : dataTypes) {
for (TSDataType to : dataTypes) {
@@ -195,6 +197,7 @@ public class TypeCastTest {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
return new Binary(Integer.toString(i), StandardCharsets.UTF_8);
case UNKNOWN:
case VECTOR:
@@ -220,6 +223,7 @@ public class TypeCastTest {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
return new Binary[] {
new Binary(Integer.toString(1), StandardCharsets.UTF_8),
new Binary(Integer.toString(2), StandardCharsets.UTF_8),
