This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch config_infer_types in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit fd2f69961e3d086c68340f1d12cc801bb45eb58a Author: qiaojialin <[email protected]> AuthorDate: Tue May 26 11:14:56 2020 +0800 add infer type config --- .../resources/conf/iotdb-engine.properties | 12 ++++++ .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 43 ++++++++++++++++++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 ++++ .../apache/iotdb/db/utils/TypeInferenceUtils.java | 41 +++++++++++++++++---- .../iotdb/db/utils/TypeInferenceUtilsTest.java | 13 +++++-- 5 files changed, 104 insertions(+), 12 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 1fcb7fa..4b863d8 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -390,6 +390,18 @@ enable_auto_create_schema=true # we will set root.sg0 as the storage group if storage group level is 1 default_storage_group_level=1 +# ALL data types: BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT + +# when receive boolean string "true" or "false", we register the time series as BOOLEAN type +boolean_string_infer_type=BOOLEAN + +# when receive an integer string "67", we register the time series as INT32 type +integer_string_infer_type=FLOAT + +# when receive a floating number string "6.7", we register the time series as FLOAT type +floating_string_infer_type=FLOAT + + # BOOLEAN encoding when creating schema automatically is enabled default_boolean_encoding=RLE diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 29257db..7f4460b 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.metadata.MManager; import org.apache.iotdb.db.service.TSServiceImpl; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.fileSystem.FSType; import org.slf4j.Logger; @@ -362,6 +363,21 @@ public class IoTDBConfig { private boolean enableAutoCreateSchema = true; /** + * when receive boolean string "true" or "false", we register the time series as BOOLEAN type + */ + private TSDataType booleanStringInferType = TSDataType.BOOLEAN; + + /** + * when receive an integer string "67", we register the time series as INT32 type + */ + private TSDataType integerStringInferType = TSDataType.FLOAT; + + /** + * when receive a floating number string "6.7", we register the time series as FLOAT type + */ + private TSDataType floatingStringInferType = TSDataType.FLOAT; + + /** * Storage group level when creating schema automatically is enabled */ private int defaultStorageGroupLevel = 1; @@ -1251,6 +1267,33 @@ public class IoTDBConfig { this.enableAutoCreateSchema = enableAutoCreateSchema; } + public TSDataType getBooleanStringInferType() { + return booleanStringInferType; + } + + public void setBooleanStringInferType( + TSDataType booleanStringInferType) { + this.booleanStringInferType = booleanStringInferType; + } + + public TSDataType getIntegerStringInferType() { + return integerStringInferType; + } + + public void setIntegerStringInferType( + TSDataType integerStringInferType) { + this.integerStringInferType = integerStringInferType; + } + + public TSDataType getFloatingStringInferType() { + return floatingStringInferType; + } + + public void setFloatingStringInferType( + TSDataType floatingNumberStringInferType) { + this.floatingStringInferType = floatingNumberStringInferType; + } + public int getDefaultStorageGroupLevel() { return defaultStorageGroupLevel; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index ab08ead..1bbb9f4 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.utils.FilePathUtils; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.fileSystem.FSType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -475,6 +476,12 @@ public class IoTDBDescriptor { conf.setAutoCreateSchemaEnabled( Boolean.parseBoolean(properties.getProperty("enable_auto_create_schema", Boolean.toString(conf.isAutoCreateSchemaEnabled()).trim()))); + conf.setBooleanStringInferType(TSDataType.valueOf(properties.getProperty("boolean_string_infer_type", + conf.getBooleanStringInferType().toString()))); + conf.setIntegerStringInferType(TSDataType.valueOf(properties.getProperty("integer_string_infer_type", + conf.getIntegerStringInferType().toString()))); + conf.setFloatingStringInferType(TSDataType.valueOf(properties.getProperty("floating_string_infer_type", + conf.getFloatingStringInferType().toString()))); conf.setDefaultStorageGroupLevel( Integer.parseInt(properties.getProperty("default_storage_group_level", Integer.toString(conf.getDefaultStorageGroupLevel())))); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java index 63c79a3..9e2bba1 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java @@ -19,11 +19,19 @@ package org.apache.iotdb.db.utils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.qp.constant.SQLConstant; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.utils.Binary; public class TypeInferenceUtils { + private static TSDataType booleanStringInferType = IoTDBDescriptor.getInstance().getConfig().getBooleanStringInferType(); + + private static TSDataType integerStringInferType = IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType(); + + private static TSDataType floatingStringInferType = IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType(); + private TypeInferenceUtils() { } @@ -46,17 +54,34 @@ public class TypeInferenceUtils { * Get predicted DataType of the given value */ public static TSDataType getPredictedDataType(Object value) { - if (value instanceof Boolean || (value instanceof String && isBoolean((String) value))) { - return TSDataType.BOOLEAN; - } else if (value instanceof Number || (value instanceof String && isNumber((String) value))) { - String v = String.valueOf(value); - if (!v.contains(".")) { - return TSDataType.INT64; + + if (value instanceof String) { + String strValue = (String) value; + if (isBoolean(strValue)) { + return booleanStringInferType; + } else if (isNumber(strValue)){ + if (!strValue.contains(".")) { + return integerStringInferType; + } else { + return floatingStringInferType; + } } else { - return TSDataType.DOUBLE; + return TSDataType.TEXT; } - } else { + } else if (value instanceof Boolean) { + return TSDataType.BOOLEAN; + } else if (value instanceof Integer) { + return TSDataType.INT32; + } else if (value instanceof Long) { + return TSDataType.INT64; + } else if (value instanceof Float) { + return TSDataType.FLOAT; + } else if (value instanceof Double) { + return TSDataType.DOUBLE; + } else if (value instanceof Binary) { return TSDataType.TEXT; } + + return TSDataType.TEXT; } } diff --git a/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java b/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java index 9f46666..69b4e5c 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/TypeInferenceUtilsTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.utils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.junit.Test; @@ -36,18 +37,22 @@ public class TypeInferenceUtilsTest { false, true, false, false, false}; for (int i = 0; i < values.length; i++) { - System.out.println(values[i]); assertEquals(TypeInferenceUtils.isNumber(values[i]), results[i]); } } @Test public void getPredictedDataTypeTest() { - Object[] values = {123, "abc", 123.123, true}; - TSDataType[] encodings = {TSDataType.INT64, TSDataType.TEXT, TSDataType.DOUBLE, TSDataType.BOOLEAN}; + Object[] values = {123, "abc", 123.123d, true, 123.1f, "123", "12.2", "true"}; + TSDataType[] encodings = {TSDataType.INT32, TSDataType.TEXT, TSDataType.DOUBLE, + TSDataType.BOOLEAN, TSDataType.FLOAT, + IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType(), + IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType(), + IoTDBDescriptor.getInstance().getConfig().getBooleanStringInferType() + }; for (int i = 0; i < values.length; i++) { - assertEquals(TypeInferenceUtils.getPredictedDataType(values[i]), encodings[i]); + assertEquals(encodings[i], TypeInferenceUtils.getPredictedDataType(values[i])); } } }
