This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch iotdbBranch in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 0dfeac670085ad5cbb9158b2654ab0becd6cb2b9 Author: JackieTien97 <[email protected]> AuthorDate: Sat Apr 20 09:15:50 2024 +0800 support for iotdb table model --- common/pom.xml | 2 +- examples/pom.xml | 4 +- pom.xml | 2 +- tsfile/pom.xml | 4 +- .../apache/tsfile/common/conf/TSFileConfig.java | 2 + .../org/apache/tsfile/file/metadata/IDeviceID.java | 55 +++++ .../apache/tsfile/file/metadata/PlainDeviceID.java | 15 ++ .../tsfile/file/metadata/StringArrayDeviceID.java | 273 +++++++++++++++++++++ .../apache/tsfile/read/common/type/BinaryType.java | 29 ++- .../tsfile/read/common/type/BooleanType.java | 27 +- .../apache/tsfile/read/common/type/DoubleType.java | 27 +- .../apache/tsfile/read/common/type/FloatType.java | 27 +- .../apache/tsfile/read/common/type/IntType.java | 27 +- .../apache/tsfile/read/common/type/LongType.java | 27 +- .../apache/tsfile/read/common/type/RowType.java | 168 +++++++++++++ .../org/apache/tsfile/read/common/type/Type.java | 15 ++ .../apache/tsfile/read/common/type/TypeEnum.java | 6 +- .../tsfile/read/common/type/TypeFactory.java | 22 ++ .../tsfile/read/common/type/UnknownType.java | 87 +++++++ .../tsfile/read/filter/factory/FilterFactory.java | 18 ++ .../type/TypeEnum.java => utils/WriteUtils.java} | 26 +- 21 files changed, 831 insertions(+), 32 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 8e61f807..ad785213 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-SNAPSHOT</version> + <version>1.0.1-table-SNAPSHOT</version> </parent> <artifactId>common</artifactId> <name>TsFile: Common API</name> diff --git a/examples/pom.xml b/examples/pom.xml index aa9f958d..a5e32728 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-SNAPSHOT</version> + <version>1.0.1-table-SNAPSHOT</version> </parent> <artifactId>examples</artifactId> <packaging>pom</packaging> @@ -33,7 +33,7 @@ <dependency> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile</artifactId> - <version>1.0.1-SNAPSHOT</version> + <version>1.0.1-table-SNAPSHOT</version> </dependency> </dependencies> <build> diff --git a/pom.xml b/pom.xml index ce766e3c..f3b0df28 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ </parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-SNAPSHOT</version> + <version>1.0.1-table-SNAPSHOT</version> <packaging>pom</packaging> <name>Apache TsFile Project Parent POM</name> <modules> diff --git a/tsfile/pom.xml b/tsfile/pom.xml index a8fb83af..86dada8c 100644 --- a/tsfile/pom.xml +++ b/tsfile/pom.xml @@ -24,7 +24,7 @@ <parent> <groupId>org.apache.tsfile</groupId> <artifactId>tsfile-parent</artifactId> - <version>1.0.1-SNAPSHOT</version> + <version>1.0.1-table-SNAPSHOT</version> </parent> <artifactId>tsfile</artifactId> <name>TsFile: TsFile</name> @@ -38,7 +38,7 @@ <dependency> <groupId>org.apache.tsfile</groupId> <artifactId>common</artifactId> - <version>1.0.1-SNAPSHOT</version> + <version>1.0.1-table-SNAPSHOT</version> </dependency> <dependency> <groupId>com.github.luben</groupId> diff --git a/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java index 23132894..0f2100cf 100644 --- a/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java +++ b/tsfile/src/main/java/org/apache/tsfile/common/conf/TSFileConfig.java @@ -73,6 +73,8 @@ public class TSFileConfig implements Serializable { /** The primitive array capacity threshold. */ public static final int ARRAY_CAPACITY_THRESHOLD = 1000; + + public static final int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3; /** Memory size threshold for flushing to disk, default value is 128MB. */ private int groupSizeInByte = 128 * 1024 * 1024; /** The memory size for each series writer to pack page, default value is 64KB. */ diff --git a/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java b/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java index 04d73c19..cbcaa7cb 100644 --- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java +++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/IDeviceID.java @@ -22,6 +22,10 @@ package org.apache.tsfile.file.metadata; import org.apache.tsfile.utils.Accountable; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -30,6 +34,8 @@ import java.nio.ByteBuffer; /** Device id interface. */ public interface IDeviceID extends Comparable<IDeviceID>, Accountable { + Logger LOGGER = LoggerFactory.getLogger(IDeviceID.class); + int serialize(ByteBuffer byteBuffer); int serialize(OutputStream outputStream) throws IOException; @@ -38,11 +44,60 @@ public interface IDeviceID extends Comparable<IDeviceID>, Accountable { boolean isEmpty(); + /** + * @return the table name associated with the device. For a path-DeviceId, like "root.a.b.c.d", it + * is converted according to a fixed rule, like assuming the first three levels ("root.a.b") + * as the table name; for a tuple-deviceId, like "(table1, beijing, turbine)", it is the first + * element in the deviceId, namely "table1". + */ + String getTableName(); + + /** + * @return how many segments this DeviceId consists of. For a path-DeviceId, like "root.a.b.c.d", + * it is 5; fot a tuple-DeviceId, like "(table1, beijing, turbine)", it is 3. + */ + int segmentNum(); + + /** + * @param i the sequence number of the segment that should be returned. + * @return i-th segment in this DeviceId. + * @throws ArrayIndexOutOfBoundsException if i >= segmentNum(). + */ + Object segment(int i); + + default int serializedSize() { + LOGGER.debug( + "Using default inefficient implementation of serialized size by {}", this.getClass()); + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + serialize(baos); + return baos.size(); + } catch (IOException e) { + LOGGER.error("Failed to serialize device ID: {}", this, e); + return -1; + } + } + + interface Deserializer { + IDeviceID deserializeFrom(ByteBuffer byteBuffer); + + IDeviceID deserializeFrom(InputStream inputStream) throws IOException; + + Deserializer DEFAULT_DESERIALIZER = StringArrayDeviceID.getDESERIALIZER(); + } + + interface Factory { + IDeviceID create(String deviceIdString); + + Factory DEFAULT_FACTORY = StringArrayDeviceID.getFACTORY(); + } + static IDeviceID deserializeFrom(ByteBuffer byteBuffer) { + // TODO return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(byteBuffer)); } static IDeviceID deserializeFrom(InputStream inputStream) throws IOException { + // TODO return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(inputStream)); } } diff --git a/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java b/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java index eb922521..b9fc6625 100644 --- a/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java +++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/PlainDeviceID.java @@ -86,6 +86,21 @@ public class PlainDeviceID implements IDeviceID { return deviceID.isEmpty(); } + @Override + public String getTableName() { + throw new UnsupportedOperationException(); + } + + @Override + public int segmentNum() { + throw new UnsupportedOperationException(); + } + + @Override + public Object segment(int i) { + throw new UnsupportedOperationException(); + } + @Override public long ramBytesUsed() { return INSTANCE_SIZE + sizeOfCharArray(deviceID.length()); diff --git a/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java b/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java new file mode 100644 index 00000000..522aee0a --- /dev/null +++ b/tsfile/src/main/java/org/apache/tsfile/file/metadata/StringArrayDeviceID.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.file.metadata; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.exception.TsFileRuntimeException; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.utils.WriteUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; + +public class StringArrayDeviceID implements IDeviceID { + + private static final Deserializer DESERIALIZER = + new Deserializer() { + @Override + public IDeviceID deserializeFrom(ByteBuffer byteBuffer) { + return deserialize(byteBuffer); + } + + @Override + public IDeviceID deserializeFrom(InputStream inputStream) throws IOException { + return deserialize(inputStream); + } + }; + + private static final Factory FACTORY = + new Factory() { + @Override + public IDeviceID create(String deviceIdString) { + return new StringArrayDeviceID(deviceIdString); + } + }; + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(StringArrayDeviceID.class); + + // TODO: change to Object[] and rename to just ArrayDeviceID + // or we can just use a tuple like Relational DB. + private final String[] segments; + + public StringArrayDeviceID(String... segments) { + this.segments = segments; + } + + public StringArrayDeviceID(String deviceIdString) { + this.segments = splitDeviceIdString(deviceIdString); + } + + @SuppressWarnings("java:S125") // confusing comments with codes + private static String[] splitDeviceIdString(String deviceIdString) { + int lastSeparatorPos = -1; + int currPos = 0; + int segmentCnt = 1; + // split the string with '.', stop when finding enough segments to form a table name + // String.split is not used here to avoid unnecessary string copy + for (; + currPos < deviceIdString.length() + && segmentCnt < TSFileConfig.DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1; + currPos++) { + if (deviceIdString.charAt(currPos) == TsFileConstant.PATH_SEPARATOR_CHAR) { + lastSeparatorPos = currPos; + segmentCnt++; + } + } + + String tableName; + String[] segments; + // assuming DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3 + if (segmentCnt < TSFileConfig.DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1) { + // "root" -> {"", "root"} + // "root.a" -> {"root", "a"} + // "root.a.b" -> {"root.a", "b"} + tableName = segmentCnt == 1 ? "" : deviceIdString.substring(0, lastSeparatorPos); + segments = new String[2]; + segments[0] = tableName; + segments[1] = deviceIdString.substring(lastSeparatorPos + 1); + } else { + // "root.a.b.c" -> {"root.a.b", "c"} + // "root.a.b.c.d" -> {"root.a.b", "c", "d"} + tableName = deviceIdString.substring(0, lastSeparatorPos); + String[] idSegments = + deviceIdString + .substring(lastSeparatorPos + 1) + .split(TsFileConstant.PATH_SEPARATER_NO_REGEX); + segments = new String[idSegments.length + 1]; + segments[0] = tableName; + System.arraycopy(idSegments, 0, segments, 1, idSegments.length); + } + + return segments; + } + + public static Deserializer getDESERIALIZER() { + return DESERIALIZER; + } + + public static Factory getFACTORY() { + return FACTORY; + } + + @Override + public int serialize(ByteBuffer byteBuffer) { + int cnt = 0; + cnt += ReadWriteIOUtils.write(segments.length, byteBuffer); + for (String segment : segments) { + cnt += ReadWriteIOUtils.write(segment, byteBuffer); + } + return cnt; + } + + @Override + public int serialize(OutputStream outputStream) throws IOException { + int cnt = 0; + cnt += ReadWriteIOUtils.write(segments.length, outputStream); + for (String segment : segments) { + cnt += ReadWriteIOUtils.write(segment, outputStream); + } + return cnt; + } + + public static StringArrayDeviceID deserialize(ByteBuffer byteBuffer) { + final int cnt = byteBuffer.getInt(); + String[] segments = new String[cnt]; + for (int i = 0; i < cnt; i++) { + final int stringSize = byteBuffer.getInt(); + byte[] stringBytes = new byte[stringSize]; + byteBuffer.get(stringBytes); + segments[i] = new String(stringBytes, TSFileConfig.STRING_CHARSET); + } + return new StringArrayDeviceID(segments); + } + + public static StringArrayDeviceID deserialize(InputStream stream) throws IOException { + final int cnt = ReadWriteIOUtils.readInt(stream); + if (cnt == 0) { + return new StringArrayDeviceID(new String[] {""}); + } + + String[] segments = new String[cnt]; + for (int i = 0; i < cnt; i++) { + final int stringSize = ReadWriteIOUtils.readInt(stream); + byte[] stringBytes = new byte[stringSize]; + final int readCnt = stream.read(stringBytes); + if (readCnt != stringSize) { + throw new IOException(String.format("Expected %d bytes but read %d", stringSize, readCnt)); + } + segments[i] = new String(stringBytes, TSFileConfig.STRING_CHARSET); + } + return new StringArrayDeviceID(segments); + } + + @Override + public byte[] getBytes() { + ByteArrayOutputStream publicBAOS = new ByteArrayOutputStream(256); + for (String segment : segments) { + try { + publicBAOS.write(segment.getBytes(TSFileConfig.STRING_CHARSET)); + } catch (IOException e) { + throw new TsFileRuntimeException(e.getMessage()); + } + } + return publicBAOS.toByteArray(); + } + + @Override + public boolean isEmpty() { + return segments == null || segments.length == 0; + } + + @Override + public String getTableName() { + return segments[0]; + } + + @Override + public int segmentNum() { + return segments.length; + } + + @Override + public String segment(int i) { + return segments[i]; + } + + @Override + public int compareTo(IDeviceID o) { + int thisSegmentNum = segmentNum(); + int otherSegmentNum = o.segmentNum(); + for (int i = 0; i < thisSegmentNum; i++) { + if (i >= otherSegmentNum) { + // the other ID is a prefix of this one + return 1; + } + final int comp = + Objects.compare(this.segment(i), ((String) o.segment(i)), WriteUtils::compareStrings); + if (comp != 0) { + // the partial comparison has a result + return comp; + } + } + + if (thisSegmentNum < otherSegmentNum) { + // this ID is a prefix of the other one + return -1; + } + + // two ID equal + return 0; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + RamUsageEstimator.sizeOf(segments); + } + + @Override + public int serializedSize() { + int cnt = Integer.BYTES; + for (String segment : segments) { + cnt += Integer.BYTES; + cnt += segment.getBytes(TSFileConfig.STRING_CHARSET).length; + } + return cnt; + } + + @Override + public String toString() { + return String.join(".", segments); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StringArrayDeviceID deviceID = (StringArrayDeviceID) o; + return Objects.deepEquals(segments, deviceID.segments); + } + + @Override + public int hashCode() { + return Arrays.hashCode(segments); + } +} diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java index bcbd2a0b..e3619e42 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/BinaryType.java @@ -24,8 +24,11 @@ import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; import org.apache.tsfile.utils.Binary; +import java.util.Collections; +import java.util.List; + public class BinaryType implements Type { - private static final BinaryType INSTANCE = new BinaryType(); + public static final BinaryType TEXT = new BinaryType(); private BinaryType() {} @@ -46,10 +49,30 @@ public class BinaryType implements Type { @Override public TypeEnum getTypeEnum() { - return TypeEnum.BINARY; + return TypeEnum.TEXT; + } + + @Override + public String getDisplayName() { + return "TEXT"; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); } public static BinaryType getInstance() { - return INSTANCE; + return TEXT; } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java index 67cf86cc..53c6d21a 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/BooleanType.java @@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; +import java.util.Collections; +import java.util.List; + public class BooleanType implements Type { - private static final BooleanType INSTANCE = new BooleanType(); + public static final BooleanType BOOLEAN = new BooleanType(); private BooleanType() {} @@ -49,7 +52,27 @@ public class BooleanType implements Type { return TypeEnum.BOOLEAN; } + @Override + public String getDisplayName() { + return "BOOLEAN"; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); + } + public static BooleanType getInstance() { - return INSTANCE; + return BOOLEAN; } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java index 317dfd96..37b129bd 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/DoubleType.java @@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.DoubleColumnBuilder; +import java.util.Collections; +import java.util.List; + public class DoubleType implements Type { - private static final DoubleType INSTANCE = new DoubleType(); + public static final DoubleType DOUBLE = new DoubleType(); private DoubleType() {} @@ -79,7 +82,27 @@ public class DoubleType implements Type { return TypeEnum.DOUBLE; } + @Override + public String getDisplayName() { + return "DOUBLE"; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); + } + public static DoubleType getInstance() { - return INSTANCE; + return DOUBLE; } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java index 51e7e5b2..ee079455 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/FloatType.java @@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.FloatColumnBuilder; +import java.util.Collections; +import java.util.List; + public class FloatType implements Type { - private static final FloatType INSTANCE = new FloatType(); + public static final FloatType FLOAT = new FloatType(); private FloatType() {} @@ -79,7 +82,27 @@ public class FloatType implements Type { return TypeEnum.FLOAT; } + @Override + public String getDisplayName() { + return "FLOAT"; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); + } + public static FloatType getInstance() { - return INSTANCE; + return FLOAT; } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java index 1cead88d..a48d1983 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/IntType.java @@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.IntColumnBuilder; +import java.util.Collections; +import java.util.List; + public class IntType implements Type { - private static final IntType INSTANCE = new IntType(); + public static final IntType INT32 = new IntType(); private IntType() {} @@ -79,7 +82,27 @@ public class IntType implements Type { return TypeEnum.INT32; } + @Override + public String getDisplayName() { + return "INT32"; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); + } + public static IntType getInstance() { - return INSTANCE; + return INT32; } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java index 7be7b792..679d908d 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/LongType.java @@ -23,9 +23,12 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.block.column.LongColumnBuilder; +import java.util.Collections; +import java.util.List; + public class LongType implements Type { - private static final LongType INSTANCE = new LongType(); + public static final LongType INT64 = new LongType(); private LongType() {} @@ -79,7 +82,27 @@ public class LongType implements Type { return TypeEnum.INT64; } + @Override + public String getDisplayName() { + return "INT64"; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); + } + public static LongType getInstance() { - return INSTANCE; + return INT64; } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java new file mode 100644 index 00000000..d3f700ec --- /dev/null +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/RowType.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.common.type; + +import org.apache.tsfile.block.column.ColumnBuilder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; +import static org.apache.tsfile.read.common.type.TypeEnum.ROW; + +public class RowType implements Type { + + private final List<Field> fields; + private final List<Type> fieldTypes; + private final boolean comparable; + private final boolean orderable; + + private RowType(List<Field> originalFields) { + + this.fields = new ArrayList<>(originalFields); + this.fieldTypes = fields.stream().map(Field::getType).collect(Collectors.toList()); + + this.comparable = fields.stream().allMatch(field -> field.getType().isComparable()); + this.orderable = fields.stream().allMatch(field -> field.getType().isOrderable()); + } + + public static RowType from(List<Field> fields) { + return new RowType(fields); + } + + public static RowType anonymous(List<Type> types) { + List<Field> fields = + types.stream().map(type -> new Field(Optional.empty(), type)).collect(Collectors.toList()); + + return new RowType(fields); + } + + public static RowType rowType(Field... field) { + return from(Arrays.asList(field)); + } + + public static RowType anonymousRow(Type... types) { + return anonymous(Arrays.asList(types)); + } + + // Only RowParametricType.createType should call this method + public static RowType createWithTypeSignature(List<Field> fields) { + return new RowType(fields); + } + + public static Field field(String name, Type type) { + return new Field(Optional.of(name), type); + } + + public static Field field(Type type) { + return new Field(Optional.empty(), type); + } + + @Override + public ColumnBuilder createColumnBuilder(int expectedEntries) { + throw new UnsupportedOperationException(); + } + + @Override + public TypeEnum getTypeEnum() { + return ROW; + } + + @Override + public String getDisplayName() { + // Convert to standard sql name + StringBuilder result = new StringBuilder(); + result.append("ROW").append('('); + for (Field field : fields) { + String typeDisplayName = field.getType().getDisplayName(); + if (field.getName().isPresent()) { + // TODO: names are already canonicalized, so they should be printed as delimited identifiers + result.append(field.getName().get()).append(' ').append(typeDisplayName); + } else { + result.append(typeDisplayName); + } + result.append(", "); + } + result.setLength(result.length() - 2); + result.append(')'); + return result.toString(); + } + + @Override + public List<Type> getTypeParameters() { + return fieldTypes; + } + + public List<Field> getFields() { + return fields; + } + + public static class Field { + private final Type type; + private final Optional<String> name; + + public Field(Optional<String> name, Type type) { + this.type = requireNonNull(type, "type is null"); + this.name = requireNonNull(name, "name is null"); + } + + public Type getType() { + return type; + } + + public Optional<String> getName() { + return name; + } + } + + @Override + public boolean isComparable() { + return comparable; + } + + @Override + public boolean isOrderable() { + return orderable; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RowType rowType = (RowType) o; + return comparable == rowType.comparable + && orderable == rowType.orderable + && Objects.equals(fields, rowType.fields) + && Objects.equals(fieldTypes, rowType.fieldTypes); + } + + @Override + public int hashCode() { + return Objects.hash(fields, fieldTypes, comparable, orderable); + } +} diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java index 5e5ed501..cb6b6094 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/Type.java @@ -23,6 +23,8 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.utils.Binary; +import java.util.List; + public interface Type { /** Gets a boolean at {@code position}. */ @@ -54,6 +56,7 @@ public interface Type { default Binary getBinary(Column c, int position) { throw new UnsupportedOperationException(getClass().getName()); } + /** Gets a Object at {@code position}. */ default Object getObject(Column c, int position) { return c.getObject(position); @@ -101,4 +104,16 @@ public interface Type { ColumnBuilder createColumnBuilder(int expectedEntries); TypeEnum getTypeEnum(); + + /** Returns the name of this type that should be displayed to end-users. */ + String getDisplayName(); + + /** True if the type supports equalTo and hash. */ + boolean isComparable(); + + /** True if the type supports compareTo. */ + boolean isOrderable(); + + /** For parameterized types returns the list of parameters. */ + List<Type> getTypeParameters(); } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java index 5a4489de..1524a681 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java @@ -30,5 +30,9 @@ public enum TypeEnum { BOOLEAN, - BINARY + TEXT, + + ROW, + + UNKNOWN } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java index 62282280..87cf3f0f 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeFactory.java @@ -46,4 +46,26 @@ public class TypeFactory { String.format("Invalid TSDataType for TypeFactory: %s", tsDataType)); } } + + public static Type getType(TypeEnum typeEnum) { + switch (typeEnum) { + case INT32: + return IntType.getInstance(); + case INT64: + return LongType.getInstance(); + case FLOAT: + return FloatType.getInstance(); + case DOUBLE: + return DoubleType.getInstance(); + case BOOLEAN: + return BooleanType.getInstance(); + case TEXT: + return BinaryType.getInstance(); + case UNKNOWN: + return UnknownType.getInstance(); + default: + throw new UnsupportedOperationException( + String.format("Invalid TypeEnum for TypeFactory: %s", typeEnum)); + } + } } diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java b/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java new file mode 100644 index 00000000..818019cf --- /dev/null +++ b/tsfile/src/main/java/org/apache/tsfile/read/common/type/UnknownType.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.common.type; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.block.column.BooleanColumnBuilder; + +import java.util.Collections; +import java.util.List; + +import static org.apache.tsfile.utils.Preconditions.checkArgument; + +public class UnknownType implements Type { + public static final UnknownType UNKNOWN = new UnknownType(); + public static final String NAME = "unknown"; + + private UnknownType() {} + + @Override + public void writeBoolean(ColumnBuilder columnBuilder, boolean value) { + // Ideally, this function should never be invoked for the unknown type. + // However, some logic (e.g. AbstractMinMaxBy) relies on writing a default value before the null + // check. + checkArgument(!value); + columnBuilder.appendNull(); + } + + @Override + public boolean getBoolean(Column column, int position) { + // Ideally, this function should never be invoked for the unknown type. + // However, some logic relies on having a default value before the null check. + checkArgument(column.isNull(position)); + return false; + } + + @Override + public ColumnBuilder createColumnBuilder(int expectedEntries) { + return new BooleanColumnBuilder(null, expectedEntries); + } + + @Override + public TypeEnum getTypeEnum() { + return TypeEnum.UNKNOWN; + } + + @Override + public String getDisplayName() { + return NAME; + } + + @Override + public boolean isComparable() { + return true; + } + + @Override + public boolean isOrderable() { + return true; + } + + @Override + public List<Type> getTypeParameters() { + return Collections.emptyList(); + } + + public static UnknownType getInstance() { + return UNKNOWN; + } +} diff --git a/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java b/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java index 1fa9b4e0..f29fc7ed 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java +++ b/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/FilterFactory.java @@ -24,6 +24,8 @@ import org.apache.tsfile.read.filter.operator.And; import org.apache.tsfile.read.filter.operator.Not; import org.apache.tsfile.read.filter.operator.Or; +import java.util.List; + import static org.apache.tsfile.utils.Preconditions.checkArgument; public class FilterFactory { @@ -43,6 +45,14 @@ public class FilterFactory { return new And(left, right); } + public static Filter and(List<Filter> filterList) { + And result = new And(filterList.get(0), filterList.get(1)); + for (int i = 2, size = filterList.size(); i < size; i++) { + result = new And(result, filterList.get(i)); + } + return result; + } + public static Filter or(Filter left, Filter right) { if (left == null && right == null) { return null; @@ -54,6 +64,14 @@ public class FilterFactory { return new Or(left, right); } + public static Filter or(List<Filter> filterList) { + Or result = new Or(filterList.get(0), filterList.get(1)); + for (int i = 2, size = filterList.size(); i < size; i++) { + result = new Or(result, filterList.get(i)); + } + return result; + } + public static Not not(Filter filter) { checkArgument(filter != null, "filter cannot be null"); return new Not(filter); diff --git a/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java b/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java similarity index 73% copy from tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java copy to tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java index 5a4489de..aa91e228 100644 --- a/tsfile/src/main/java/org/apache/tsfile/read/common/type/TypeEnum.java +++ b/tsfile/src/main/java/org/apache/tsfile/utils/WriteUtils.java @@ -17,18 +17,20 @@ * under the License. */ -package org.apache.tsfile.read.common.type; +package org.apache.tsfile.utils; -public enum TypeEnum { - INT32, +public class WriteUtils { - INT64, - - FLOAT, - - DOUBLE, - - BOOLEAN, - - BINARY + public static int compareStrings(String a, String b) { + if (a == null && b == null) { + return 0; + } + if (a == null) { + return -1; + } + if (b == null) { + return 1; + } + return a.compareTo(b); + } }
