This is an automated email from the ASF dual-hosted git repository. xxyu pushed a change to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
from 8de5c7a712 Add package info in front end new 1b9d519bbb KYLIN-5704 Avoid exceptions on iterating filters and support pruning segments with cast-in expressions on dimension columns. new 5dbf6bcd87 Refactor email notification function new 8aca676867 KYLIN-5706 fix shell command injection new 7f143a8bef KYLIN-5707 update calcite to 1.116.0-kylin-4.x-r028 new 443adfc162 KYLIN-5708 add LOG_TIME_PATTERN_WITH_TRACE_ID in getJobLogPattern new cf5fc3b900 KYLIN-5706 [FOLLOW UP] fix bug new d39787c91c KYLIN-5709 Support modifying column comment attribute new 220d272532 KYLIN-5710 Remove audit log from the diagnostic package by default new 4fb83dd593 KYLIN-5711 fix query limit get metadata url bug when node type is query new 6697caabea KYLIN-5712 upper case request params in open api create model & comments… new 3400c96600 KYLIN-5713 support REGEXP_EXTRACT function new becc65b6c9 KYLIN-5714 job event log diag dump for model or project config new 1d42d60206 KYLIN-5715 Using isolated env with fixed locale for cmd diff new 5c814ecb22 License Limitation related new 9583e2bdaf remove unused java file new ab9b3dab31 KYLIN-5717 constant project expression eliminate error of to_date/to_timestamp function new 13366b6fb4 KYLIN-5718 V3 Dictionary Automatic Merge new 58956007a6 fix discarded failed jobs do not receive email notifications, etc. new ab4007b215 KYLIN-5719 add v3dict delta table check new db19826dd5 KYLIN-5720 Fix bug that all node can not get sessionid from http requert which redirect by query node caused by commit 'prevent early initialization of SessionRepository beans' from spring community new 6855e40faf KYLIN-5721 make data_trunc support date string args and give tests to second,miniute,hour,extract funcs for date string args new bc00ef2ab5 KYLIN-5719 [FOLLOW UP] v3dct catch DeltaConcurrentModificationException new 75ce83ad66 KYLIN-5722 Optimize KylinConfigExt new 4753eb92cd KYLIN-5723 optimize KylinConfigExt new 053a3293f3 KYLIN-5724 Support local debug & glue hive client cache new 6f5030377d KYLIN-5725 fix open the email notification configuration, to operate job and query failed new b93d19181e KYLIN-5721 [FOLLOW UP] add two date funcs to support date string args new 90834ef3bc KYLIN-5726 Remove unnecessary execution goal for deploy plugin new 67ea6928e8 KYLIN-5721 [FOLLOW UP] fix calcite jar no to support timestampadd/diff's date string args new 73e0104fc7 fix async query error new f676b28346 KYLIN-5727 update spark version to 3.2.0-kylin-4.6.11.0 new d3b512af90 fix test case error new ff1bc03b44 fix testcase error new 652345ac72 optimize import new e3eaf05590 fix testcase data version info The 35 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: build/bin/diag.sh | 6 +- build/bin/upgrade.sh | 5 +- pom.xml | 13 +- .../src/main/resources/config/init.properties | 8 +- .../kylin/rest/controller/NAdminController.java | 20 +- .../kylin/rest/controller/NBasicController.java | 26 +- .../rest/controller/NBasicControllerTest.java | 84 +++-- .../apache/kylin/rest/config/AppInitializer.java | 10 + .../apache/kylin/rest/interceptor/KEFilter.java | 6 +- .../rest/request/JobNotificationConfigRequest.java | 8 +- .../kylin/rest/response/ProjectConfigResponse.java | 32 +- .../apache/kylin/rest/service/ProjectService.java | 54 +-- .../java/org/apache/kylin/rest/util/HttpUtil.java | 4 +- .../org/apache/kylin/common/KylinConfigBase.java | 46 ++- .../org/apache/kylin/common/KylinConfigExt.java | 23 +- .../apache/kylin/common/constant/Constants.java | 1 + .../constant/NonCustomProjectLevelConfig.java | 13 +- .../kylin/common/exception/ServerErrorCode.java | 1 + .../common/extension/KylinInfoExtension.java} | 37 +- .../kylin/common/mail/MailNotificationCreator.java | 70 ++++ .../kylin/common/mail/MailNotificationType.java | 97 +++++ .../org/apache/kylin/common/mail/MailNotifier.java | 57 +++ .../kylin/common/{util => mail}/MailService.java | 68 ++-- .../org/apache/kylin/common/msg/CnMessage.java | 10 + .../java/org/apache/kylin/common/msg/Message.java | 8 + .../apache/kylin/common/util/CompositeMapView.java | 9 +- .../org/apache/kylin/common/util/MailHelper.java | 100 ------ .../kylin/common/util/MailTemplateProvider.java | 86 ----- .../src/main/resources/kylin-defaults0.properties | 3 + .../resources/kylin_errorcode_conf_en.properties | 1 + .../resources/kylin_errorcode_conf_zh.properties | 1 + .../resources/mail_templates/JOB_DISCARDED.ftl} | 105 +++--- .../main/resources/mail_templates/JOB_ERROR.ftl} | 149 ++++---- .../resources/mail_templates/JOB_FINISHED.ftl} | 123 +++---- .../mail_templates/JOB_LOAD_EMPTY_DATA.ftl} | 112 +++--- .../OVER_LICENSE_CAPACITY_THRESHOLD.ftl} | 127 +------ .../apache/kylin/common/KylinConfigBaseTest.java | 12 +- .../common/extension/KylinInfoExtensionTest.java} | 24 +- .../apache/kylin/common/util/MailServiceTest.java | 74 ++-- .../kylin/job/constant/ExecutableConstants.java | 7 - .../apache/kylin/job/constant/JobIssueEnum.java | 33 -- .../kylin/job/execution/AbstractExecutable.java | 209 ++--------- .../kylin/job/execution/DefaultExecutable.java | 26 +- .../job/execution/EmailNotificationContent.java | 194 ---------- .../kylin/job/execution/ExecutableParams.java | 30 +- .../kylin/job/execution/NExecutableManager.java | 16 +- .../org/apache/kylin/job/mail/JobMailUtil.java | 126 +++++++ .../kylin/job/util/MailNotificationUtil.java | 104 ------ .../main/resources/mail_templates/JOB_ERROR.ftl | 390 --------------------- .../main/resources/mail_templates/JOB_SUCCEED.ftl | 273 --------------- .../kylin/job/execution/ErrorTestExecutable.java | 5 +- .../kylin/job/execution/JobMailUtilTest.java | 130 +++++++ .../kylin/job/execution/JobStatusChangedTest.java | 42 ++- .../job/execution/NExecutableManagerTest.java | 125 ++----- .../kylin/job/execution/SucceedTestExecutable.java | 5 +- .../metadata/model/schema/ReloadTableContext.java | 6 +- .../metadata/sourceusage/SourceUsageManager.java | 34 +- .../sourceusage/mail/SourceUsageMailUtil.java | 73 ++++ .../kylin/metadata/state/JdbcShareStateStore.java | 2 +- .../sourceusage/mail/SourceUsageMailUtilTest.java | 58 +++ .../src/main/resources/config/init.properties | 8 +- .../org/apache/kylin/rest/service/JobService.java | 22 +- .../apache/kylin/rest/service/JobServiceTest.java | 29 +- .../apache/kylin/rest/service/TableService.java | 39 ++- .../test_case_data/localmeta/kylin.properties | 23 +- .../org/apache/kylin/newten/NFilePruningTest.java | 19 +- .../sql_computedcolumn_DateUDF/query05.sql | 39 +++ .../{query00.sql => query07.sql} | 16 +- .../kylin/rest/controller/NProjectController.java | 9 +- .../rest/controller/open/OpenModelController.java | 2 + .../rest/controller/NProjectControllerTest.java | 3 +- .../controller/open/OpenModelControllerTest.java | 1 + .../apache/kylin/rest/request/ModelRequest.java | 38 +- .../kylin/rest/request/ModelRequestTest.java | 215 ++++++++++++ .../kylin/rest/service/ModelServiceTest.java | 6 + .../kylin/rest/service/ProjectServiceTest.java | 29 +- .../kylin/rest/service/TableReloadServiceTest.java | 45 ++- .../nmodel_test_expected.json | 2 +- .../src/main/resources/config/init.properties | 8 +- .../org/apache/kylin/query/util/PushDownUtil.java | 6 + .../apache/kylin/query/engine/AsyncQueryJob.java | 17 +- .../org/apache/kylin/query/udf/SparkStringUDF.java | 5 + .../org/apache/kylin/query/udf/SparkTimeUDF.java | 18 + .../kylin/query/udf/dateUdf/DateTruncUDF.java | 5 + .../kylin/query/udf/formatUdf/ToDateUDF.java | 55 --- .../kylin/query/udf/formatUdf/ToTimestampUDF.java | 69 ---- .../kylin/query/engine/AsyncQueryJobTest.java | 46 ++- .../org/apache/kylin/query/udf/FormatUDFTest.java | 27 -- .../apache/kylin/rest/MultiTenantModeFilter.java | 2 +- .../src/main/resources/config/init.properties | 8 +- .../kylin/engine/spark/job/NSparkExecutable.java | 4 +- .../spark/builder/v3dict/DictionaryBuilder.scala | 135 +++++-- .../v3dict/PreCountDistinctTransformer.scala | 2 +- .../engine/spark/NLocalWithSparkSessionTest.java | 2 + .../builder/v3dict/GlobalDictionarySuite.scala | 144 +++++++- .../kylin/query/runtime/ExpressionConverter.scala | 46 +-- .../sql/catalyst/expressions/KapExpresssions.scala | 8 +- .../apache/spark/sql/udf/TimestampAddTest.scala | 78 ++++- .../apache/spark/sql/udf/TimestampDiffTest.scala | 64 +++- .../sql/execution/datasource/FilePruner.scala | 61 ++-- .../spark/sql/common/SharedSparkSession.scala | 2 + .../java/org/apache/kylin/tool/DiagClientTool.java | 2 +- .../org/apache/kylin/tool/JobDiagInfoTool.java | 28 +- .../java/org/apache/kylin/tool/KylinLogTool.java | 19 +- .../kylin/tool/StreamingJobDiagInfoTool.java | 2 +- .../org/apache/kylin/tool/JobDiagInfoToolTest.java | 33 ++ .../org/apache/kylin/tool/KylinLogToolTest.java | 15 +- 107 files changed, 2422 insertions(+), 2555 deletions(-) copy src/{query/src/main/java/org/apache/kylin/query/util/PushDownQueryRequestLimits.java => core-common/src/main/java/org/apache/kylin/common/extension/KylinInfoExtension.java} (51%) create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/mail/MailNotificationCreator.java create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/mail/MailNotificationType.java create mode 100644 src/core-common/src/main/java/org/apache/kylin/common/mail/MailNotifier.java rename src/core-common/src/main/java/org/apache/kylin/common/{util => mail}/MailService.java (61%) delete mode 100644 src/core-common/src/main/java/org/apache/kylin/common/util/MailHelper.java delete mode 100644 src/core-common/src/main/java/org/apache/kylin/common/util/MailTemplateProvider.java rename src/{core-job/src/main/resources/mail_templates/LOAD_EMPTY_DATA.ftl => core-common/src/main/resources/mail_templates/JOB_DISCARDED.ftl} (73%) rename src/{core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl => core-common/src/main/resources/mail_templates/JOB_ERROR.ftl} (81%) rename src/{core-job/src/main/resources/mail_templates/OVER_CAPACITY_THRESHOLD.ftl => core-common/src/main/resources/mail_templates/JOB_FINISHED.ftl} (70%) rename src/{core-job/src/main/resources/mail_templates/SOURCE_RECORDS_CHANGE.ftl => core-common/src/main/resources/mail_templates/JOB_LOAD_EMPTY_DATA.ftl} (72%) rename src/{core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl => core-common/src/main/resources/mail_templates/OVER_LICENSE_CAPACITY_THRESHOLD.ftl} (52%) copy src/{core-job/src/test/java/org/apache/kylin/job/snapshot/SnapshotUtilsTest.java => core-common/src/test/java/org/apache/kylin/common/extension/KylinInfoExtensionTest.java} (65%) delete mode 100644 src/core-job/src/main/java/org/apache/kylin/job/constant/JobIssueEnum.java delete mode 100644 src/core-job/src/main/java/org/apache/kylin/job/execution/EmailNotificationContent.java create mode 100644 src/core-job/src/main/java/org/apache/kylin/job/mail/JobMailUtil.java delete mode 100644 src/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java delete mode 100644 src/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl delete mode 100644 src/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl create mode 100644 src/core-job/src/test/java/org/apache/kylin/job/execution/JobMailUtilTest.java create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/metadata/sourceusage/mail/SourceUsageMailUtil.java create mode 100644 src/core-metadata/src/test/java/org/apache/kylin/metadata/sourceusage/mail/SourceUsageMailUtilTest.java create mode 100644 src/kylin-it/src/test/resources/query/sql_computedcolumn/sql_computedcolumn_DateUDF/query05.sql copy src/kylin-it/src/test/resources/query/sql_function/sql_function_DateUDF/{query00.sql => query07.sql} (65%) create mode 100644 src/modeling-service/src/test/java/org/apache/kylin/rest/request/ModelRequestTest.java delete mode 100644 src/query/src/main/java/org/apache/kylin/query/udf/formatUdf/ToDateUDF.java delete mode 100644 src/query/src/main/java/org/apache/kylin/query/udf/formatUdf/ToTimestampUDF.java