This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch NewTsFile in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a9bed897faad28991ce219efdc30fd9d058ffa66 Merge: 0b153b2 f6cd1f9 Author: JackieTien97 <[email protected]> AuthorDate: Wed Dec 2 16:08:38 2020 +0800 resolve conflicts antlr/pom.xml | 4 + .../iotdb/db/qp/{strategy => sql}/SqlBase.g4 | 50 +- client-py/compile.bat | 29 - client-py/compile.sh | 28 - client-py/pom.xml | 4 +- client-py/pypi/README.md | 34 +- client-py/readme.md | 46 +- client-py/src/SessionExample.py | 10 +- client-py/src/{ => iotdb}/Session.py | 28 +- client-py/src/{utils => iotdb}/__init__.py | 0 client-py/src/{ => iotdb}/utils/Field.py | 5 +- client-py/src/{ => iotdb}/utils/IoTDBConstants.py | 0 client-py/src/{ => iotdb}/utils/IoTDBRpcDataSet.py | 13 +- client-py/src/{ => iotdb}/utils/RowRecord.py | 8 +- client-py/src/{ => iotdb}/utils/SessionDataSet.py | 16 +- client-py/src/{ => iotdb}/utils/Tablet.py | 7 +- client-py/src/{ => iotdb/utils}/__init__.py | 0 docker/ReadMe.md | 10 +- docs/Development/ContributeGuide.md | 6 +- docs/UserGuide/Concept/SDT.md | 45 + docs/UserGuide/Operation Manual/SQL Reference.md | 8 + docs/UserGuide/Server/Config Manual.md | 27 - .../System Tools/Monitor and Log Tools.md | 321 +-- docs/UserGuide/System Tools/Watermark Tool.md | 8 +- docs/zh/Development/ContributeGuide.md | 2 +- docs/zh/UserGuide/Concept/Data Type.md | 2 +- docs/zh/UserGuide/Concept/SDT.md | 42 + docs/zh/UserGuide/Get Started/Publication.md | 2 +- .../zh/UserGuide/Operation Manual/SQL Reference.md | 8 + docs/zh/UserGuide/Server/Config Manual.md | 29 - .../System Tools/Monitor and Log Tools.md | 305 +-- docs/zh/UserGuide/System Tools/Watermark Tool.md | 8 +- .../main/java/org/apache/iotdb/kafka/Constant.java | 2 + example/pom.xml | 1 + {antlr => example/pulsar}/pom.xml | 36 +- .../java/org/apache/iotdb/pulsar/Constant.java | 19 +- .../org/apache/iotdb/pulsar/PulsarConsumer.java | 108 + .../apache/iotdb/pulsar/PulsarConsumerThread.java | 70 + .../org/apache/iotdb/pulsar/PulsarProducer.java | 82 + .../main/java/org/apache/iotdb/SessionExample.java | 14 + .../controller/DatabaseConnectController.java | 4 +- .../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 29 +- .../web/grafana/dao/impl/BasicDaoImplTest.java | 55 + pom.xml | 2 +- server/file-changelists/TsFileMods-changelist.md | 28 + .../resources/conf/iotdb-engine.properties | 19 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 47 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 25 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 26 +- .../db/engine/compaction/TsFileManagement.java | 21 + .../level/LevelCompactionTsFileManagement.java | 42 +- .../no/NoCompactionTsFileManagement.java | 22 +- .../engine/compaction/utils/CompactionUtils.java | 2 +- .../apache/iotdb/db/engine/flush/FlushManager.java | 14 + .../iotdb/db/engine/flush/FlushManagerMBean.java | 8 +- .../db/engine/querycontext/ReadOnlyMemChunk.java | 16 +- .../engine/storagegroup/StorageGroupProcessor.java | 49 +- .../db/engine/storagegroup/TsFileProcessor.java | 2 + .../org/apache/iotdb/db/metadata/MManager.java | 7 +- .../org/apache/iotdb/db/metadata/PartialPath.java | 4 + .../org/apache/iotdb/db/metrics/sink/Sink.java | 6 +- .../org/apache/iotdb/db/metrics/source/Source.java | 2 +- .../apache/iotdb/db/monitor/MonitorConstants.java | 94 +- .../org/apache/iotdb/db/monitor/StatMonitor.java | 448 ++-- .../StatMonitorMBean.java} | 29 +- .../iotdb/db/monitor/collector/FileSize.java | 162 -- .../monitor/collector/MemTableWriteTimeCost.java | 85 - .../main/java/org/apache/iotdb/db/qp/Planner.java | 8 +- .../apache/iotdb/db/qp/constant/SQLConstant.java | 2 +- .../apache/iotdb/db/qp/executor/PlanExecutor.java | 22 +- .../db/qp/logical/crud/QueryIndexOperator.java | 53 - .../iotdb/db/qp/logical/crud/QueryOperator.java | 21 + .../db/qp/physical/sys/CreateTimeSeriesPlan.java | 6 +- .../IoTDBSqlVisitor.java} | 2629 ++++++++++---------- .../iotdb/db/qp/strategy/LogicalGenerator.java | 1950 +-------------- .../apache/iotdb/db/qp/strategy/ParseDriver.java | 69 - .../iotdb/db/qp/strategy/PhysicalGenerator.java | 14 +- ...gicalGeneratorError.java => SQLParseError.java} | 4 +- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +- .../java/org/apache/iotdb/db/service/IoTDB.java | 11 +- .../java/org/apache/iotdb/db/service/Monitor.java | 150 -- .../org/apache/iotdb/db/service/TSServiceImpl.java | 147 +- .../iotdb/db/integration/IoTDBCompactionIT.java | 99 + .../iotdb/db/integration/IoTDBRestartIT.java | 61 +- .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 422 +++- .../iotdb/db/monitor/collector/FileSizeTest.java | 93 - .../iotdb/db/qp/plan/IndexLogicalPlanTest.java | 32 +- .../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 110 +- .../sql/DatetimeQueryDataSetUtilsTest.java | 2 +- .../iotdb/db/qp/strategy/LogicalGeneratorTest.java | 26 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 7 +- site/src/main/.vuepress/config.js | 2 + thrift/src/main/thrift/cluster.thrift | 1 + thrift/src/main/thrift/rpc.thrift | 1 + thrift/src/main/thrift/sync.thrift | 1 + .../iotdb/tsfile/compress/IUnCompressor.java | 2 +- .../iotdb/tsfile/encoding/encoder/SDTEncoder.java | 448 ++++ .../tsfile/encoding/encoder/TSEncodingBuilder.java | 14 +- .../iotdb/tsfile/read/TsFileSequenceReader.java | 18 +- .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 124 +- .../iotdb/tsfile/encoding/SDTEncoderTest.java | 176 ++ 101 files changed, 4082 insertions(+), 5331 deletions(-) diff --cc server/src/main/java/org/apache/iotdb/db/service/IoTDB.java index 60d7095,51be619..ea94f47 --- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java +++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java @@@ -136,8 -127,10 +127,10 @@@ public class IoTDB implements IoTDBMBea } } + // Warn: registMonitor() method should be called after systemDataRecovery() + registerManager.register(StatMonitor.getInstance()); registerManager.register(SyncServerManager.getInstance()); - registerManager.register(UpgradeSevice.getINSTANCE()); +// registerManager.register(UpgradeSevice.getINSTANCE()); registerManager.register(MergeManager.getINSTANCE()); registerManager.register(CompactionMergeTaskPoolManager.getInstance()); diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 95c4c9d,5977155..94caab4 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@@ -77,13 -78,16 +78,22 @@@ public class ChunkWriterImpl implement */ private Statistics<?> statistics; + private boolean isSdtEncoding; + + private SDTEncoder sdtEncoder; + + /** + * do not re-execute SDT compression when merging chunks + */ + private boolean isMerging; + /** + * first page info + */ + private int sizeWithoutStatistic; + private Statistics<?> firstPageStatistics; + + /** * @param schema schema of this measurement */ public ChunkWriterImpl(MeasurementSchema schema) { @@@ -101,6 -105,60 +111,61 @@@ this.statistics = Statistics.getStatsByType(measurementSchema.getType()); this.pageWriter = new PageWriter(measurementSchema); ++ + this.pageWriter.setTimeEncoder(measurementSchema.getTimeEncoder()); + this.pageWriter.setValueEncoder(measurementSchema.getValueEncoder()); + + //check if the measurement schema uses SDT + checkSdtEncoding(); + } + + public ChunkWriterImpl(MeasurementSchema schema, boolean isMerging) { + this(schema); + this.isMerging = isMerging; + } + + private void checkSdtEncoding() { + if (measurementSchema.getProps() != null && !isMerging) { + if (measurementSchema.getProps().getOrDefault("loss", "").equals("sdt")) { + isSdtEncoding = true; + sdtEncoder = new SDTEncoder(); + } + + if (isSdtEncoding && measurementSchema.getProps().containsKey("compdev")) { + try { + sdtEncoder.setCompDeviation(Double.parseDouble(measurementSchema.getProps().get("compdev"))); + } catch (NumberFormatException e) { + logger.error("meet error when formatting SDT compression deviation"); + } + if (sdtEncoder.getCompDeviation() < 0) { + logger + .error("SDT compression deviation cannot be negative. SDT encoding is turned off."); + isSdtEncoding = false; + } + } + + if (isSdtEncoding && measurementSchema.getProps().containsKey("compmin")) { + try { + sdtEncoder.setCompMin(Double.parseDouble(measurementSchema.getProps().get("compmin"))); + } catch (NumberFormatException e) { + logger.error("meet error when formatting SDT compression minimum"); + } + } + + if (isSdtEncoding && measurementSchema.getProps().containsKey("compmax")) { + try { + sdtEncoder.setCompMax(Double.parseDouble(measurementSchema.getProps().get("compmax"))); + } catch (NumberFormatException e) { + logger.error("meet error when formatting SDT compression maximum"); + } + } + + if (isSdtEncoding && sdtEncoder.getCompMax() <= sdtEncoder.getCompMin()) { + logger + .error("SDT compression maximum needs to be greater than compression minimum. SDT encoding is turned off"); + isSdtEncoding = false; + } + } } @Override @@@ -342,5 -418,23 +444,11 @@@ writer.endCurrentChunk(); } - /** - * estimate max page memory size. - * - * @return the max possible allocated size currently - */ - private long estimateMaxPageMemSize() { - // return the sum of size of buffer and page max size - return (long) (pageBuffer.size() + - PageHeader.calculatePageHeaderSizeWithoutStatistics() + - pageWriter.getStatistics().getSerializedSize()); - } - + public void setIsMerging(boolean isMerging) { + this.isMerging = isMerging; + } + public boolean isMerging() { + return isMerging; + } }
