(kylin) 32/35: KYLIN-5841 Fix vulnerability, Upgrade org.apache.avro:avro to version 1.11.3
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7d24ce642c1dc96f2a476408fd546d971963d5e0
Author: huangsheng
AuthorDate: Sun Oct 8 15:29:39 2023 +0800
KYLIN-5841 Fix vulnerability, Upgrade org.apache.avro:avro to version 1.11.3
---
pom.xml | 7 +++
1 file changed, 7 insertions(+)
diff --git a/pom.xml b/pom.xml
index 04a01dc8a8..6b37bca758 100644
--- a/pom.xml
+++ b/pom.xml
@@ -401,6 +401,7 @@
9.23
1.2.1
1.1.10.4
+1.11.3
@@ -3003,6 +3004,12 @@
netty
${netty3.version}
+
+org.apache.avro
+avro
+${avro.version}
+provided
+
(kylin) 20/35: KYLIN-5816 Add zk lock for dict v3
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2159699d8e54588d879e0ebdbb1f2ed11380bb23
Author: Yaguang Jia
AuthorDate: Thu Sep 21 16:51:14 2023 +0800
KYLIN-5816 Add zk lock for dict v3
---
.../engine/spark/builder/DFDictionaryBuilder.scala | 17 +-
.../kylin/engine/spark/builder/ZKHelper.scala | 42 +
.../spark/builder/v3dict/DictionaryBuilder.scala | 69 ++
3 files changed, 75 insertions(+), 53 deletions(-)
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
index ae95344cd0..af8399ca3a 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
@@ -48,28 +48,13 @@ class DFDictionaryBuilder(
changeAQEConfig(true)
}
- private val YARN_CLUSTER: String = "cluster"
private val AQE = "spark.sql.adaptive.enabled";
private val originalAQE = ss.conf.get(AQE)
- private def tryZKJaasConfiguration(): Unit = {
-val config = KylinConfig.getInstanceFromEnv
-if (YARN_CLUSTER.equals(config.getDeployMode)) {
- val kapConfig = KapConfig.wrap(config)
- if (KapConfig.FI_PLATFORM.equals(kapConfig.getKerberosPlatform) ||
KapConfig.TDH_PLATFORM.equals(kapConfig.getKerberosPlatform)) {
-val sparkConf = ss.sparkContext.getConf
-val principal = sparkConf.get("spark.kerberos.principal")
-val keytab = sparkConf.get("spark.kerberos.keytab")
-logInfo(s"ZKJaasConfiguration principal: $principal, keyTab: $keytab")
-javax.security.auth.login.Configuration.setConfiguration(new
ZKJaasConfiguration(principal, keytab))
- }
-}
- }
-
@throws[IOException]
private[builder] def safeBuild(ref: TblColRef): Unit = {
val sourceColumn = ref.getIdentity
-tryZKJaasConfiguration()
+ZKHelper.tryZKJaasConfiguration(ss)
val lock: Lock = KylinConfig.getInstanceFromEnv.getDistributedLockFactory
.getLockForCurrentThread(getLockPath(sourceColumn))
lock.lock()
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala
new file mode 100644
index 00..1fd321f64e
--- /dev/null
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/ZKHelper.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.spark.builder
+
+import org.apache.kylin.common.{KapConfig, KylinConfig}
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.SparkSession
+
+object ZKHelper extends Logging{
+ private val YARN_CLUSTER: String = "cluster"
+
+ def tryZKJaasConfiguration(ss: SparkSession): Unit = {
+val config = KylinConfig.getInstanceFromEnv
+if (YARN_CLUSTER.equals(config.getDeployMode)) {
+ val kapConfig = KapConfig.wrap(config)
+ if (KapConfig.FI_PLATFORM.equals(kapConfig.getKerberosPlatform) ||
KapConfig.TDH_PLATFORM.equals(kapConfig.getKerberosPlatform)) {
+val sparkConf = ss.sparkContext.getConf
+val principal = sparkConf.get("spark.kerberos.principal")
+val keytab = sparkConf.get("spark.kerberos.keytab")
+logInfo(s"ZKJaasConfiguration principal: $principal, keyTab: $keytab")
+javax.security.auth.login.Configuration.setConfiguration(new
ZKJaasConfiguration(principal, keytab))
+ }
+}
+ }
+
+}
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/Dictio
(kylin) 35/35: KYLIN-5846 upgrade spark version to 3.2.0-kylin-4.6.16.0
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 3f9b9c83bedbce17be0dcac5af427c636353621a Author: huangsheng AuthorDate: Mon Oct 30 13:30:54 2023 +0800 KYLIN-5846 upgrade spark version to 3.2.0-kylin-4.6.16.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b37bca758..889fa87c96 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 2.0.2 -3.2.0-kylin-4.6.14.0 +3.2.0-kylin-4.6.16.0 0.9.2-kylin-r4
(kylin) 29/35: KYLIN-5838 Replace join expression with the name of computedColumn
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit eacb62b8aa9c23a627ec5803d18b245420222f4c
Author: Pengfei Zhan
AuthorDate: Tue Sep 26 21:23:55 2023 +0800
KYLIN-5838 Replace join expression with the name of computedColumn
---
.../kylin/query/util/ConvertToComputedColumn.java | 20 +++
.../apache/kylin/query/util/CCOnRealModelTest.java | 68 ++
2 files changed, 52 insertions(+), 36 deletions(-)
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index 1c4fa5a287..8e4e7631b7 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -34,11 +34,13 @@ import javax.annotation.Nullable;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlAsOperator;
+import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlDynamicParam;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlJoin;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
@@ -100,12 +102,30 @@ public class ConvertToComputedColumn implements
IQueryTransformer {
List inputNodes = new LinkedList<>();
inputNodes.addAll(collectCandidateInputNodes(select.getSelectList(),
select.getGroup()));
inputNodes.addAll(collectCandidateInputNodes(select.getOrderList(),
select.getGroup()));
+if (select.getFrom() instanceof SqlJoin) {
+SqlJoin join = (SqlJoin) select.getFrom();
+collectJoinNodes(inputNodes, join);
+}
inputNodes.addAll(collectCandidateInputNode(select.getHaving(),
select.getGroup()));
inputNodes.addAll(getInputTreeNodes(select.getWhere()));
inputNodes.addAll(getInputTreeNodes(select.getGroup()));
return inputNodes;
}
+private static void collectJoinNodes(List inputNodes, SqlJoin
join) {
+if (join.getLeft() instanceof SqlJoin) {
+collectJoinNodes(inputNodes, (SqlJoin) join.getLeft());
+}
+SqlNode condition = join.getCondition();
+if (condition.getKind() == SqlKind.EQUALS) {
+SqlBasicCall call = (SqlBasicCall) condition;
+inputNodes.addAll(call.getOperandList());
+}
+if (join.getRight() instanceof SqlJoin) {
+collectJoinNodes(inputNodes, (SqlJoin) join.getRight());
+}
+}
+
private static List collectInputNodes(SqlOrderBy sqlOrderBy) {
// if order list is not empty and query is a select
// then collect order list with checking on group keys
diff --git
a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
index 164810ad51..a299fbee9e 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/CCOnRealModelTest.java
@@ -17,32 +17,20 @@
*/
package org.apache.kylin.query.util;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
/**
* test against real models
*/
-public class CCOnRealModelTest extends NLocalFileMetadataTestCase {
-private ConvertToComputedColumn converter;
-
-@Before
-public void setup() throws Exception {
-
this.createTestMetadata("../core-metadata/src/test/resources/ut_meta/ccjointest");
-converter = new ConvertToComputedColumn();
-}
-
-@After
-public void after() throws Exception {
-super.cleanupTestMetadata();
-}
+@MetadataInfo(overlay =
"../core-metadata/src/test/resources/ut_meta/ccjointest")
+class CCOnRealModelTest {
+private final ConvertToComputedColumn converter = new
ConvertToComputedColumn();
@Test
-public void testConvertSingleTableCC() {
+void testConvertSingleTableCC() {
{
String originSql = "select count(*), sum (price * item_count) from
test_kylin_fact f left join test_order o on f.ORDER_ID = o.ORDER_ID"
+ " left join test_account a on o.buyer_id = a.account
(kylin) 24/35: KYLIN-5830 Push columns of sortRel to subOlapContexts
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 191d08c272b48d04441783290459291f70c2e9e6
Author: Pengfei Zhan
AuthorDate: Wed Sep 20 10:20:56 2023 +0800
KYLIN-5830 Push columns of sortRel to subOlapContexts
---
.../query/routing/RealizationChooserTest.java | 24 +-
.../org/apache/kylin/util/OlapContextTestUtil.java | 13
.../org/apache/kylin/query/relnode/KapSortRel.java | 19 +++--
.../apache/kylin/query/relnode/KapTableScan.java | 10 +
4 files changed, 59 insertions(+), 7 deletions(-)
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
index 2cea9cf51b..267be6b1d8 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/routing/RealizationChooserTest.java
@@ -21,12 +21,14 @@ package org.apache.kylin.query.routing;
import java.util.List;
import java.util.Map;
+import org.apache.calcite.rel.RelNode;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.query.relnode.KapSortRel;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.util.OlapContextTestUtil;
import org.junit.After;
@@ -97,6 +99,25 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
}
}
+@Test
+public void testPushSortRelToSubOlapContexts() throws SqlParseException {
+overwriteSystemProp("kylin.query.print-logical-plan", "true");
+String project = "joins_graph_left_or_inner";
+String sql = "select a.NAME from TEST_BANK_INCOME a inner join
TEST_BANK_LOCATION b on a.COUNTRY = b.COUNTRY\n"
++ "order by a.INCOME nulls last";
+RelNode relNode =
OlapContextTestUtil.cutOlapContextsAndReturnRelNode(project, sql);
+KapSortRel sortRel = null;
+while (relNode != null) {
+if (relNode instanceof KapSortRel) {
+sortRel = (KapSortRel) relNode;
+break;
+}
+relNode = relNode.getInput(0);
+}
+Assert.assertNotNull(sortRel);
+Assert.assertTrue(sortRel.isNeedPushToSubCtx());
+}
+
@Test
public void testCanMatchModelInnerQueryLeft() throws SqlParseException {
// model: TEST_BANK_INCOME inner join TEST_BANK_LOCATION
@@ -127,7 +148,8 @@ public class RealizationChooserTest extends
NLocalWithSparkSessionTest {
String sql = "select a.NAME from TEST_BANK_INCOME a left join
TEST_BANK_LOCATION b \n"
+ " on a.COUNTRY = b.COUNTRY where " + filter;
OLAPContext olapContext =
OlapContextTestUtil.getOlapContexts(getProject(), sql).get(0);
-Map sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(), olapContext);
+Map sqlAlias2ModelNameMap =
OlapContextTestUtil.matchJoins(dataflow.getModel(),
+olapContext);
Assert.assertTrue(sqlAlias2ModelNameMap.isEmpty());
}
}
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
index 094fa8c3c5..e7e2a88b5c 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/OlapContextTestUtil.java
@@ -56,6 +56,19 @@ public class OlapContextTestUtil {
return getOlapContexts();
}
+public static RelNode cutOlapContextsAndReturnRelNode(String project,
String sql) throws SqlParseException {
+QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv());
+RelNode rel = null;
+try {
+rel = queryExec.parseAndOptimize(sql);
+QueryContextCutter.selectRealization(project, rel, false);
+} catch (NoRealizationFoundException |
NoStreamingRealizationFoundException e) {
+// When NoRealizationFoundException occurs, do nothing
+// because we only need to obtain OlapContexts.
+}
+return rel;
+}
+
public static List getOlapContexts(String project, String
sql, boolean reCutBanned,
Consumer consumer) throws
SqlParseException {
QueryExec queryExec = new QueryExec(project,
KylinConfig.getInstanceFromEnv()
(kylin) 21/35: KYLIN-5817 Query scanRows and bytes are incorrect
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 00352e8c2d53307b4cf3f847fc6bbf2a869c4520
Author: haocheni
AuthorDate: Fri Sep 22 10:29:48 2023 +0800
KYLIN-5817 Query scanRows and bytes are incorrect
---
.../src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala | 4 ++--
.../org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
index 4beda86353..68fc7b9b19 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/hive/QueryMetricUtils.scala
@@ -63,8 +63,8 @@ object QueryMetricUtils extends Logging {
child => {
if (child.isInstanceOf[SparkPlan]) {
val result = collectAdaptiveSparkPlanExecMetrics(child, scanRow,
scanBytes)
- newScanRow = result._1
- newScanBytes = result._2
+ newScanRow += result._1
+ newScanBytes += result._2
} else {
logTrace("Not sparkPlan in collectAdaptiveSparkPlanExecMetrics,
child: " + child.getClass.getName)
}
diff --git
a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
index ede02eef60..f0eda5b817 100644
---
a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
+++
b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/execution/SparkQueryMetricUtilsSuite.scala
@@ -217,8 +217,8 @@ class SparkQueryMetricUtilsSuite extends QueryTest with
SharedSparkSession {
assert(0 == collectScanMetrics._1.get(0))
assert(0 == collectScanMetrics._2.get(0))
val collectScanMetrics2 =
QueryMetricUtils.collectAdaptiveSparkPlanExecMetrics(dataWritingCommandExec, 1,
1)
-assert(1 == collectScanMetrics2._1)
-assert(1 == collectScanMetrics2._2)
+assert(2 == collectScanMetrics2._1)
+assert(2 == collectScanMetrics2._2)
}
(kylin) 22/35: KYLIN-5828 Concurrently dict v2 jobs lead to abnormal encoding result
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c61dc4189968f29f8e56e98f72a16f100e9d6e2b
Author: huangsheng
AuthorDate: Fri Sep 22 16:13:50 2023 +0800
KYLIN-5828 Concurrently dict v2 jobs lead to abnormal encoding result
---
.../engine/spark/builder/CreateFlatTable.scala | 15 +--
.../engine/spark/builder/DFDictionaryBuilder.scala | 16 +--
.../engine/spark/builder/DFTableEncoder.scala | 13 ++-
.../spark/builder/DictionaryBuilderHelper.java | 5 +-
.../job/stage/build/FlatTableAndDictBase.scala | 18 ++--
.../engine/spark/dict/NGlobalDictionaryV2Test.java | 77 +--
.../kylin/engine/spark/builder/TestDFChooser.scala | 5 +-
.../engine/spark/builder/TestGlobalDictBuild.scala | 58 +--
.../v3dict/GlobalDictionaryUpdateSuite.scala | 6 +-
.../scala/org/apache/spark/sql/KapFunctions.scala | 4 +-
.../sql/catalyst/expressions/KapExpresssions.scala | 24 +++--
.../org/apache/spark/sql/udf/DictEncodeImpl.scala | 9 +-
.../org/apache/spark/dict/NBucketDictionary.java | 62 +---
.../spark/dict/NGlobalDictBuilderAssist.scala | 5 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java| 10 +-
.../org/apache/spark/dict/NGlobalDictS3Store.java | 16 +--
.../org/apache/spark/dict/NGlobalDictStore.java| 4 +-
.../org/apache/spark/dict/NGlobalDictionaryV2.java | 107 +
18 files changed, 326 insertions(+), 128 deletions(-)
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
index b659b87969..654c5b9f6f 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/CreateFlatTable.scala
@@ -31,6 +31,7 @@ import org.apache.kylin.guava30.shaded.common.collect.Sets
import org.apache.kylin.metadata.cube.cuboid.NSpanningTree
import org.apache.kylin.metadata.cube.model.{NCubeJoinedFlatTableDesc,
NDataSegment}
import org.apache.kylin.metadata.model._
+import org.apache.spark.dict.NGlobalDictionaryV2.NO_VERSION_SPECIFIED
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.{Dataset, Row, SparkSession}
@@ -115,12 +116,14 @@ class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
dictCols: Set[TblColRef],
encodeCols: Set[TblColRef]): Dataset[Row] = {
val ccDataset = withColumn(ds, ccCols)
+var buildVersion = System.currentTimeMillis()
if (seg.isDictReady) {
logInfo(s"Skip already built dict, segment: ${seg.getId} of dataflow:
${seg.getDataflow.getId}")
+ buildVersion = NO_VERSION_SPECIFIED
} else {
- buildDict(ccDataset, dictCols)
+ buildDict(ccDataset, dictCols, buildVersion)
}
-encodeColumn(ccDataset, encodeCols)
+encodeColumn(ccDataset, encodeCols, buildVersion)
}
private def withColumn(ds: Dataset[Row], withCols: Set[TblColRef]):
Dataset[Row] = {
@@ -131,21 +134,21 @@ class CreateFlatTable(val flatTable: IJoinedFlatTableDesc,
withDs
}
- private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef]): Unit = {
+ private def buildDict(ds: Dataset[Row], dictCols: Set[TblColRef],
buildVersion: Long): Unit = {
val matchedCols = if (seg.getIndexPlan.isSkipEncodeIntegerFamilyEnabled) {
filterOutIntegerFamilyType(ds, dictCols)
} else {
selectColumnsInTable(ds, dictCols)
}
val builder = new DFDictionaryBuilder(ds, seg, ss,
Sets.newHashSet(matchedCols.asJavaCollection))
-builder.buildDictSet()
+builder.buildDictSet(buildVersion)
}
- private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef]):
Dataset[Row] = {
+ private def encodeColumn(ds: Dataset[Row], encodeCols: Set[TblColRef],
buildVersion: Long): Dataset[Row] = {
val matchedCols = selectColumnsInTable(ds, encodeCols)
var encodeDs = ds
if (matchedCols.nonEmpty) {
- encodeDs = DFTableEncoder.encodeTable(ds, seg, matchedCols.asJava)
+ encodeDs = DFTableEncoder.encodeTable(ds, seg, matchedCols.asJava,
buildVersion)
}
encodeDs
}
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
index af8399ca3a..d930fd178b 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/DFDictionaryBuilder.scala
@@ -43,8 +43,
(kylin) 23/35: KYLIN-5829 Add a tool for generating sql from dumped metadata
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f9255a515d38edafd099c81712c0b6ff8e7038d7
Author: Pengfei Zhan
AuthorDate: Mon Jun 19 11:41:51 2023 +0800
KYLIN-5829 Add a tool for generating sql from dumped metadata
---
build/bin/create-table.sh | 1 -
build/conf/kylin-tools-diag-log4j.xml | 6 ++
.../kylin/tool}/util/CreateTableFromJson.java | 31 +-
.../kylin/tool/util/CreateTableFromJsonTest.java | 68 ++
4 files changed, 89 insertions(+), 17 deletions(-)
diff --git a/build/bin/create-table.sh b/build/bin/create-table.sh
index 5fbc49541d..a21e39036e 100755
--- a/build/bin/create-table.sh
+++ b/build/bin/create-table.sh
@@ -16,7 +16,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
if [ -z $KYLIN_HOME ];then
export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
fi
diff --git a/build/conf/kylin-tools-diag-log4j.xml
b/build/conf/kylin-tools-diag-log4j.xml
index 728e50d2ca..227e71cdd1 100644
--- a/build/conf/kylin-tools-diag-log4j.xml
+++ b/build/conf/kylin-tools-diag-log4j.xml
@@ -28,6 +28,9 @@
+
+
+
@@ -38,5 +41,8 @@
+
+
+
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
b/src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
similarity index 81%
rename from
src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
rename to
src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
index 2ee7efef22..67148ad608 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
@@ -16,10 +16,13 @@
* limitations under the License.
*/
-package org.apache.kylin.rest.util;
+package org.apache.kylin.tool.util;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -29,20 +32,16 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* A tool to generate database and tables from metadata backups of customer
*/
+@Slf4j
public class CreateTableFromJson {
private static final String QUOTE = "`";
-private static final Map TYPE_MAP = Maps.newHashMap();
-
-private static final Logger logger =
LoggerFactory.getLogger(CreateTableFromJson.class);
+private static final Map TYPE_MAP = new HashMap<>();
static {
TYPE_MAP.put("integer", "int");
@@ -54,23 +53,23 @@ public class CreateTableFromJson {
String pathDir = args[0];
Map> map = createDbAndTables(pathDir);
-map.forEach((k, v) -> {
-logger.info(k);
-v.forEach(logger::info);
+map.forEach((db, tables) -> {
+for (String s : db.split("\n")) {
+log.info(s);
+}
+tables.forEach(log::info);
});
-
-logger.info("\n\n\n\n\n");
}
// the path is /{metadata_backup_path}/{project_name}/table/
private static Map> createDbAndTables(String pathDir)
throws IOException {
-Map> map = Maps.newHashMap();
+Map> map = new LinkedHashMap<>();
File file = new File(pathDir).getAbsoluteFile();
File[] files = file.listFiles();
for (File f : Objects.requireNonNull(files)) {
final TableDesc tableDesc = JsonUtil.readValue(f, TableDesc.class);
-List columnNameTypeList = Lists.newArrayList();
+List columnNameTypeList = new ArrayList<>();
for (ColumnDesc column : tableDesc.getColumns()) {
String name = column.getName();
String type = convert(column.getDatatype());
@@ -79,7 +78,7 @@ public class CreateTableFromJson {
String databaseSql = String.format(Locale.ROOT, "create database
%s;%nuse %s;",
quote(tableDesc.getDatabase()),
quote(tableDesc.getDatabase()));
-map.putIfAbsent(databaseSql, Lists.newArrayList());
+map.putIfAbsent(databaseSql, new ArrayList<>());
(kylin) 13/35: KYLIN-5807 Fix query execute as user without data query acl
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a212350a518cde5fa1fe0357a71b806f5533e2c3
Author: Zhimin Wu <596361...@qq.com>
AuthorDate: Fri Sep 1 18:26:22 2023 +0800
KYLIN-5807 Fix query execute as user without data query acl
Co-authored-by: Feng Zhu
---
.../java/org/apache/kylin/rest/service/AccessService.java | 14 ++
.../java/org/apache/kylin/rest/service/QueryService.java | 13 +
.../org/apache/kylin/rest/service/QueryServiceTest.java| 2 ++
3 files changed, 25 insertions(+), 4 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 54dcf242c8..031282378f 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -756,13 +756,19 @@ public class AccessService extends BasicService {
return sidWithPermissions;
}
-@SneakyThrows(IOException.class)
public Set getUserNormalExtPermissions(String project) {
-String projectUuid =
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
-.getUuid();
Authentication authentication =
SecurityContextHolder.getContext().getAuthentication();
if (Objects.nonNull(authentication)) {
-val userName = authentication.getName();
+return getUserNormalExtPermissionsByUserInProject(project,
authentication.getName());
+}
+return new HashSet<>();
+}
+
+@SneakyThrows(IOException.class)
+public Set getUserNormalExtPermissionsByUserInProject(String
project, String userName) {
+String projectUuid =
NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
+.getUuid();
+if (Objects.nonNull(userName)) {
if (userAclService.canAdminUserQuery(userName)) {
return Collections.singleton(AclConstants.DATA_QUERY);
}
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 60837454a0..7b10e1eb67 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -81,6 +81,7 @@ import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.constants.AclConstants;
import org.apache.kylin.engine.spark.filter.BloomFilterSkipCollector;
import org.apache.kylin.engine.spark.filter.ParquetPageFilterCollector;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
@@ -567,6 +568,18 @@ public class QueryService extends BasicService implements
CacheSignatureQuerySup
if (!grantedProjects.contains(sqlRequest.getProject())) {
throw new KylinException(ACCESS_DENIED, "Access is denied.");
}
+
+val isDataPermissionDefaultEnabled =
KylinConfig.getInstanceFromEnv().isDataPermissionDefaultEnabled();
+if (isDataPermissionDefaultEnabled) {
+try {
+Set extPermissions =
accessService.getUserNormalExtPermissionsByUserInProject(sqlRequest.getProject(),
executeUser);
+if (!extPermissions.contains(AclConstants.DATA_QUERY)) {
+throw new KylinException(ACCESS_DENIED, "Access is
denied.");
+}
+} catch (Exception e) {
+throw new KylinException(ACCESS_DENIED, e);
+}
+}
}
private void checkSqlRequest(SQLRequest sqlRequest) {
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 200fbb960f..89fead0d99 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -230,6 +230,8 @@ public class QueryServiceTest extends
NLocalFileMetadataTestCase {
ReflectionTestUtils.setField(queryService, "aclTCRService",
aclTCRService);
ReflectionTestUtils.setField(accessService, "userService",
userService);
ReflectionTestUtils.setField(accessService, "aclService", aclService);
+ReflectionTestUtils.setField(accessService, "userAclService",
userAclService);
+
(kylin) 01/35: KYLIN-5792 Support ldap memberid mode
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d6285db6ab757a13ce83d119001a8ef37944522e
Author: Mingming Ge <7mmi...@gmail.com>
AuthorDate: Tue Aug 22 17:05:27 2023 +0800
KYLIN-5792 Support ldap memberid mode
---
.../java/org/apache/kylin/tool/util/LdapUtils.java | 12 -
.../org/apache/kylin/tool/util/LdapUtilsTest.java | 57 ++
2 files changed, 68 insertions(+), 1 deletion(-)
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
b/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
index 79b621291a..ed7d3ecbca 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
@@ -112,8 +112,18 @@ public class LdapUtils {
left += maxValRange;
}
}
-
+ldapUserDNs = rewriteUserDnIfNeeded(ldapUserDNs);
return ldapUserDNs;
}
+public static Set rewriteUserDnIfNeeded(Set ldapUserDNs) {
+String ldapUserSearchBase =
KylinConfig.getInstanceFromEnv().getLDAPUserSearchBase();
+return ldapUserDNs.stream().map(dn -> {
+String dnName = dn;
+if (!dn.contains(ldapUserSearchBase)) {
+dnName = "uid=" + dn + "," + ldapUserSearchBase;
+}
+return dnName;
+}).collect(Collectors.toSet());
+}
}
diff --git
a/src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java
b/src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java
new file mode 100644
index 00..216d535a78
--- /dev/null
+++ b/src/tool/src/test/java/org/apache/kylin/tool/util/LdapUtilsTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.tool.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LdapUtilsTest extends NLocalFileMetadataTestCase {
+
+@Before
+public void setup() throws Exception {
+createTestMetadata();
+}
+
+@After
+public void teardown() {
+cleanupTestMetadata();
+}
+
+@Test
+public void testRewriteUserDn() {
+getTestConfig().setProperty("kylin.security.ldap.user-search-base",
"cn=Users,dc=example,dc=com");
+String ldapUserSearchBase = getTestConfig().getLDAPUserSearchBase();
+Set ldapUserDNs = new HashSet<>();
+ldapUserDNs.add("a");
+ldapUserDNs.add("b");
+ldapUserDNs = LdapUtils.rewriteUserDnIfNeeded(ldapUserDNs);
+Assert.assertTrue(ldapUserDNs.stream().allMatch(x ->
x.contains(ldapUserSearchBase)));
+
+Set ldapUserDNs2 = new HashSet<>();
+ldapUserDNs2.add("uid=a,cn=Users,dc=example,dc=com");
+ldapUserDNs2.add("uid=b,cn=Users,dc=example,dc=com");
+ldapUserDNs2 = LdapUtils.rewriteUserDnIfNeeded(ldapUserDNs2);
+Assert.assertTrue(ldapUserDNs2.stream().allMatch(x ->
x.contains(ldapUserSearchBase)));
+}
+}
(kylin) 28/35: KYLIN-5837 Support only reusing user defined computedColumns
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 148d8f61aaafbca0c207d76f0747a85ed0b11fbb
Author: Pengfei Zhan
AuthorDate: Mon Sep 11 14:43:04 2023 +0800
KYLIN-5837 Support only reusing user defined computedColumns
---
.../java/org/apache/kylin/common/KapConfig.java| 27 +++---
.../org/apache/kylin/common/KylinConfigBase.java | 25 ++---
.../org/apache/kylin/common/KapConfigTest.java | 15 ++
.../apache/kylin/common/KylinConfigBaseTest.java | 8 +++
.../kylin/query/util/ComputedColumnRewriter.java | 17 +++---
.../kylin/query/util/ConvertToComputedColumn.java | 60 --
.../query/util/RestoreFromComputedColumn.java | 19 +++
7 files changed, 79 insertions(+), 92 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index c7c10c500d..c79a588c95 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -22,13 +22,14 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
-import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.annotation.Clarification;
import org.apache.kylin.common.util.EncryptUtil;
import org.apache.kylin.common.util.FileUtils;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
@Clarification(priority = Clarification.Priority.MAJOR, msg = "Enterprise")
public class KapConfig {
@@ -233,21 +234,6 @@ public class KapConfig {
return config.getOptional("kylin.smart.conf." + conf, null);
}
-/**
- * Query
- */
-public boolean isImplicitComputedColumnConvertEnabled() {
-return
Boolean.parseBoolean(config.getOptional("kylin.query.implicit-computed-column-convert",
"true"));
-}
-
-public boolean isAggComputedColumnRewriteEnabled() {
-return
Boolean.parseBoolean(config.getOptional("kylin.query.agg-computed-column-rewrite",
"true"));
-}
-
-public int getComputedColumnMaxRecursionTimes() {
-return
Integer.parseInt(config.getOptional("kylin.query.computed-column-max-recursion-times",
"10"));
-}
-
public boolean isJdbcEscapeEnabled() {
return
Boolean.parseBoolean(config.getOptional("kylin.query.jdbc-escape-enabled",
"true"));
}
@@ -266,7 +252,8 @@ public class KapConfig {
}
public Boolean isConstraintPropagationEnabled() {
-return
Boolean.parseBoolean(config.getOptional("kylin.query.engine.spark-constraint-propagation-enabled",
FALSE));
+return Boolean
+
.parseBoolean(config.getOptional("kylin.query.engine.spark-constraint-propagation-enabled",
FALSE));
}
/**
@@ -716,11 +703,13 @@ public class KapConfig {
}
public boolean isApplyLimitInfoToSourceScanRowsEnabled() {
-return
Boolean.parseBoolean(config.getOptional("kylin.query.apply-limit-info-to-source-scan-rows-enabled",
FALSE));
+return Boolean
+
.parseBoolean(config.getOptional("kylin.query.apply-limit-info-to-source-scan-rows-enabled",
FALSE));
}
public boolean isAutoAdjustBigQueryRowsThresholdEnabled() {
-return
Boolean.parseBoolean(config.getOptional("kylin.query.auto-adjust-big-query-rows-threshold-enabled",
FALSE));
+return Boolean
+
.parseBoolean(config.getOptional("kylin.query.auto-adjust-big-query-rows-threshold-enabled",
FALSE));
}
public long getBigQuerySecond() {
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a2518358a9..3042b23c58 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -793,6 +793,18 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.metadata.only-reuse-user-defined-computed-column",
FALSE));
}
+public boolean isConvertExpressionToCcEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.query.implicit-computed-column-convert",
TRUE));
+}
+
+public boolean isAggComputedColumnRewriteEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.query.agg-computed-column-rewrite",
TRUE));
+}
+
+public int getConvertCcMaxIterations() {
+return
Integer.parseInt(getOptional("kylin.quer
(kylin) 26/35: KYLIN-5833 Use TableIndex answer select * only works for query
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 412eff5c73908508e3967412a2dbffd76d171b30
Author: Pengfei Zhan
AuthorDate: Tue Aug 29 16:24:25 2023 +0800
KYLIN-5833 Use TableIndex answer select * only works for query
---
.../src/main/java/org/apache/kylin/common/QueryContext.java | 4
.../main/java/org/apache/kylin/query/routing/RealizationChooser.java | 4 +++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index b112591e27..b6ef63867e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -107,6 +107,10 @@ public class QueryContext implements Closeable {
@Setter
private boolean partialMatchIndex = false;
+@Getter
+@Setter
+private boolean isForModeling;
+
@Getter
@Setter
private ForceToTieredStorage forcedToTieredStorage;
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index 768edd91ff..56bee577c9 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -255,7 +255,9 @@ public class RealizationChooser {
buildDimensionsAndMetrics(context, dimensions, metrics);
buildStorageContext(context, dimensions, metrics, candidate);
buildSecondStorageEnabled(context.getSQLDigest());
-fixContextForTableIndexAnswerNonRawQuery(context);
+if (!QueryContext.current().isForModeling()) {
+fixContextForTableIndexAnswerNonRawQuery(context);
+}
}
}
(kylin) 09/35: KYLIN-5791 & KYLIN-5801 Fix computation of timestampdff & add_month
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e33ef57cce3f3865ca9b4a6f132a170257d46be2
Author: liang.huang <83992752+lhuang09287...@users.noreply.github.com>
AuthorDate: Thu Aug 31 10:46:48 2023 +0800
KYLIN-5791 & KYLIN-5801 Fix computation of timestampdff & add_month
1. Add datetype for add_month function as required
2. Support string args for Timestampdiff constant computation
---
pom.xml| 2 +-
.../scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala | 7 ++-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/pom.xml b/pom.xml
index ccbdfbae22..940eb93ff0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
0.3.0
-1.116.0-kylin-4.x-r035
+1.116.0-kylin-4.x-r036
4.x_1.10-r01
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
index 0b7116a673..b8e34366b8 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/SparderRexVisitor.scala
@@ -171,7 +171,12 @@ class SparderRexVisitor(val inputFieldNames: Seq[String],
case num: MonthNum => {
// both add_month and add_year case
val ts = k_lit(children.head).cast(TimestampType)
- return k_lit(k_add_months(k_lit(ts), num.num))
+ call.getType.getSqlTypeName match {
+case SqlTypeName.DATE =>
+ return k_lit(k_add_months(k_lit(ts), num.num)).cast(DateType)
+case _ =>
+ return k_lit(k_add_months(k_lit(ts), num.num))
+ }
}
case _ =>
}
(kylin) 05/35: KYLIN-5797 Remove nacos jar
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 82ee25fdf1a66601a3cebcab4ca86c1664da57c4 Author: Zhimin Wu <596361...@qq.com> AuthorDate: Wed Aug 30 14:00:26 2023 +0800 KYLIN-5797 Remove nacos jar --- src/server/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/pom.xml b/src/server/pom.xml index deb02b3ade..5f96ac4bc7 100644 --- a/src/server/pom.xml +++ b/src/server/pom.xml @@ -420,6 +420,8 @@ org.json4s, org.mortbay.jetty, org.scala-lang, +com.alibaba.nacos, +com.alibaba.cloud, com.google.flatbuffers, com.google.inject com.sun.jersey,
(kylin) 18/35: KYLIN-5814 Support lettuce sentinel mode
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ebd2f3d6704a566896b60f7a3c67bf7226c67348
Author: haocheni
AuthorDate: Thu Sep 14 10:00:44 2023 +0800
KYLIN-5814 Support lettuce sentinel mode
---
pom.xml| 7 +
src/common-service/pom.xml | 4 +
.../kylin/rest/cache/AbstractKylinCache.java | 78 +++
.../org/apache/kylin/rest/cache/CacheConstant.java | 29 +++
.../org/apache/kylin/rest/cache/RedisCache.java| 91 ++---
.../org/apache/kylin/rest/cache/RedisCacheV2.java | 224 +
.../apache/kylin/rest/cache/CacheConstantTest.java | 30 +++
.../apache/kylin/rest/cache/RedisCacheTest.java| 36
.../apache/kylin/rest/cache/RedisCacheV2Test.java | 195 ++
.../org/apache/kylin/common/KylinConfigBase.java | 8 +
.../kylin/common/exception/ServerErrorCode.java| 1 +
.../resources/kylin_errorcode_conf_en.properties | 1 +
.../resources/kylin_errorcode_conf_zh.properties | 1 +
.../kylin/metadata/query/QueryMetricsContext.java | 3 +-
src/query-service/pom.xml | 5 +
.../kylin/rest/service/QueryCacheManager.java | 113 +--
.../kylin/rest/service/QueryCacheManagerTest.java | 191 ++
17 files changed, 923 insertions(+), 94 deletions(-)
diff --git a/pom.xml b/pom.xml
index 88261b4d28..07224bd0f1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -188,6 +188,7 @@
2.10.9.2
2.12.3
3.8.0
+6.1.8.RELEASE
4.5.13
1.9.4
2.12.2
@@ -797,6 +798,12 @@
${redis.version}
+
+io.lettuce
+lettuce-core
+${lettuce.version}
+
+
org.apache.kylin
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index 747735c9ea..7a0d6d3403 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -75,6 +75,10 @@
redis.clients
jedis
+
+io.lettuce
+lettuce-core
+
net.sf.ehcache
ehcache
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java
b/src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java
new file mode 100644
index 00..9461a29f09
--- /dev/null
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/cache/AbstractKylinCache.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.cache;
+
+import static org.apache.kylin.rest.cache.CacheConstant.PREFIX;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+import org.apache.kylin.common.util.CompressionUtils;
+import org.apache.kylin.rest.service.CommonQueryCacheSupporter;
+import org.apache.kylin.rest.util.SerializeUtil;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public abstract class AbstractKylinCache implements KylinCache {
+protected boolean isExceptionQuery(String type) {
+return
type.equals(CommonQueryCacheSupporter.Type.EXCEPTION_QUERY_CACHE.rootCacheName);
+}
+
+protected String getTypeProjectPrefix(String type, String project) {
+return String.format(Locale.ROOT, "%s-%s", type, project);
+}
+
+protected byte[] convertKeyToByte(String type, Object key) {
+try {
+String prefixAndType = PREFIX + type;
+byte[] typeBytes = getBytesFromString(prefixAndType);
+byte[] keyBytes = SerializeUtil.serialize(key);
+byte[] trueKeyBytes = new byte[keyBytes.length + typeBytes.length];
+System.arraycopy(typeBytes, 0, trueKeyBytes, 0, typeBytes.length);
+System.arraycopy(keyBytes, 0, trueKeyBytes, typeBytes.length,
keyBytes.length);
+return trueKeyBytes;
+} catch (Exception e) {
+log.error("serialize fail!&q
(kylin) 27/35: KYLIN-5836 Fix mapr env building error
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e49c2f15a645cb33a0f04a0b894af8be24613bf5
Author: Yinghao Lin
AuthorDate: Mon Sep 11 11:52:48 2023 +0800
KYLIN-5836 Fix mapr env building error
---
.../org/apache/kylin/engine/spark/application/SparkApplication.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index 1f1d359eb6..2c2dc08bed 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -294,7 +294,7 @@ public abstract class SparkApplication implements
Application {
monitorSparkMaster();
-
HadoopUtil.setCurrentConfiguration(HadoopUtil.getHadoopConfFromSparkEngine());
+
HadoopUtil.setCurrentConfiguration(HadoopUtil.getCurrentConfiguration());
exchangeSparkConf(buildEnv.sparkConf());
(kylin) 25/35: KYLIN-5832 Support queryDetect api
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d7f37675e2de44478da78d5ae7b4b70fd043b13f
Author: haocheni
AuthorDate: Fri Sep 22 19:21:38 2023 +0800
KYLIN-5832 Support queryDetect api
---
.../java/org/apache/kylin/common/QueryContext.java | 1 +
.../kylin/rest/controller/NQueryController.java| 9 +
.../kylin/rest/request/QueryDetectRequest.java | 33 +++
.../kylin/rest/response/QueryDetectResponse.java | 156
.../apache/kylin/rest/service/QueryService.java| 37 ++-
.../kylin/rest/service/QueryServiceTest.java | 278 -
.../kylin/query/engine/QueryRoutingEngine.java | 7 +
.../engine/exec/sparder/SparderQueryPlanExec.java | 6 +
.../kylin/query/engine/QueryRoutingEngineTest.java | 34 +++
9 files changed, 557 insertions(+), 4 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 592df87f68..b112591e27 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -389,6 +389,7 @@ public class QueryContext implements Closeable {
private boolean isRefused;
private boolean includeHeader;
private boolean isVacant;
+private boolean isQueryDetect;
}
@Getter
diff --git
a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index 6d69ea475f..1f56073647 100644
---
a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++
b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -73,12 +73,14 @@ import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLFormatRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.request.SyncFileSegmentsRequest;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.QueryHistoryFiltersResponse;
import org.apache.kylin.rest.response.QueryStatisticsResponse;
import org.apache.kylin.rest.response.SQLResponse;
@@ -662,6 +664,13 @@ public class NQueryController extends NBasicController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
queryService.format(request.getSqls()), "");
}
+@ApiOperation(value = "queryDetect", tags = { "QE" })
+@PostMapping("/detection")
+public EnvelopeResponse queryDetect(@RequestBody
QueryDetectRequest request) {
+checkProjectName(request.getProject());
+return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
queryService.queryDetect(request), "");
+}
+
private void checkQueryName(String queryName) {
if (!queryNamePattern.matcher(queryName).matches()) {
throw new KylinException(INVALID_NAME,
MsgPicker.getMsg().getInvalidQueryName());
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
b/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
new file mode 100644
index 00..9ee03f4b3f
--- /dev/null
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
(kylin) 33/35: KYLIN-5844 & KYLIN-5843 Column ACL works for PushDown
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e804eee81eff19424ea61c6899c226ba3d5fcf59
Author: Pengfei Zhan
AuthorDate: Sun Oct 1 17:19:17 2023 +0800
KYLIN-5844 & KYLIN-5843 Column ACL works for PushDown
1. Column ACL works for PushDown
2. Flat-table sql should consider acl
---
.../apache/kylin/metadata/model/ColumnDesc.java| 5 +-
.../apache/kylin/rest/util/AclPermissionUtil.java | 2 +-
.../kylin/rest/response/ExecutableResponse.java| 4 +-
.../routing/DataflowCapabilityCheckerTest.java | 3 +-
.../kylin/query/rules/AggPushdownRuleTest.java | 4 +-
.../kylin/query/rules/CalciteRuleTestBase.java | 12 +-
.../rest/response/LayoutRecDetailResponse.java | 2 +-
.../apache/kylin/rest/service/ModelService.java| 10 +-
.../rest/service/params/BasicSegmentParams.java| 3 +-
.../security/HackSelectStarWithColumnACL.java | 259 +++--
.../org/apache/kylin/query/util/PushDownUtil.java | 32 +-
.../security/HackSelectStarWithColumnACLTest.java | 405 ++---
.../apache/kylin/query/util/PushDownUtilTest.java | 359 ++
.../merger/AfterMergeOrRefreshResourceMerger.java | 3 +-
.../spark/source/NSparkMetadataExplorer.java | 3 +-
.../engine/spark/job/NSparkMergingJobTest.java | 10 +-
16 files changed, 705 insertions(+), 411 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index c2efaccd4e..1754104ae4 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -23,6 +23,7 @@ import java.io.Serializable;
import org.apache.calcite.avatica.util.Quoting;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.StringHelper;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -30,7 +31,6 @@ import
com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSetter;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import lombok.Getter;
import lombok.Setter;
@@ -231,8 +231,7 @@ public class ColumnDesc implements Serializable {
// for test mainly
public static ColumnDesc mockup(TableDesc table, int oneBasedColumnIndex,
String name, String datatype) {
ColumnDesc desc = new ColumnDesc();
-String id = "" + oneBasedColumnIndex;
-desc.setId(id);
+desc.setId(String.valueOf(oneBasedColumnIndex));
desc.setName(name);
desc.setDatatype(datatype);
desc.init(table);
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
b/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
index 36360a0f12..82aaca58dc 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
@@ -235,7 +235,7 @@ public class AclPermissionUtil {
}
public static QueryContext.AclInfo createAclInfo(String project,
Set groups) {
-return new QueryContext.AclInfo(getCurrentUsername(), groups,
isAdminInProject(project, groups));
+return new QueryContext.AclInfo(getCurrentUsername(), groups,
hasProjectAdminPermission(project, groups));
}
public static boolean hasExtPermission(Permission permission) {
diff --git
a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
index 70144b83a2..6736c2bb74 100644
---
a/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
+++
b/src/data-loading-service/src/main/java/org/apache/kylin/rest/response/ExecutableResponse.java
@@ -27,6 +27,8 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.engine.spark.job.NSparkSnapshotJob;
import org.apache.kylin.engine.spark.job.NTableSamplingJob;
import org.apache.kylin.job.SecondStorageCleanJobUtil;
@@ -43,11 +45,9 @@ import org.apache.kylin.metadata.model.NTableMetadataMa
(kylin) 19/35: KYLIN-5815 Failed to add primaryKey of table when upgrading concurrently
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 768a5ececcd855b0f796881d42b36b17088c61d5
Author: Jiawei Li <1019037...@qq.com>
AuthorDate: Thu Sep 21 14:20:24 2023 +0800
KYLIN-5815 Failed to add primaryKey of table when upgrading concurrently
---
.../common/persistence/metadata/JdbcEpochStore.java | 20
.../common/persistence/metadata/jdbc/JdbcUtil.java | 10 ++
.../persistence/metadata/jdbc/JdbcUtilTest.java | 17 +
3 files changed, 39 insertions(+), 8 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
index 97b50d2969..2d510801b6 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/JdbcEpochStore.java
@@ -41,6 +41,7 @@ import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.Singletons;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
@@ -122,17 +123,20 @@ public class JdbcEpochStore extends EpochStore {
}
return dataSource.getConnection();
}
-
+
@Override
public void createIfNotExist() throws Exception {
if (isTableExists(getConnection(jdbcTemplate), table)) {
-if (!isPrimaryKeyExists(getConnection(jdbcTemplate), table)) {
-withTransaction(transactionManager, () -> {
-jdbcTemplate.execute(getAddPrimarykeySql(table));
-return 1;
-});
-log.info("Succeed to add table primary key: {}", table);
-}
+JdbcUtil.retry(() -> {
+if (!isPrimaryKeyExists(getConnection(jdbcTemplate), table)) {
+withTransaction(transactionManager, () -> {
+jdbcTemplate.execute(getAddPrimarykeySql(table));
+return 1;
+});
+log.info("Succeed to add table primary key: {}", table);
+}
+return 1;
+});
return;
}
String fileName = "metadata-jdbc-default.properties";
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
index 51b61c348a..265777c1a3 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
@@ -260,4 +260,14 @@ public class JdbcUtil {
val dataSource = BasicDataSourceFactory.createDataSource(props);
return new JdbcTemplate(dataSource);
}
+
+public static T retry(Callback func) throws Exception {
+try {
+return func.handle();
+} catch (Exception e) {
+log.error("occur exception, retry", e);
+return func.handle();
+}
+}
+
}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
index ab0f7926c9..a3d05f453d 100644
---
a/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
+++
b/src/core-common/src/test/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtilTest.java
@@ -20,6 +20,8 @@ package org.apache.kylin.common.persistence.metadata.jdbc;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.util.HashSet;
+import java.util.Set;
import org.junit.Assert;
import org.junit.Before;
@@ -51,4 +53,19 @@ public class JdbcUtilTest {
null);
Assert.assertFalse(JdbcUtil.isColumnExists(connection, table,
"not_exists"));
}
+
+@Test
+public void testRetry() throws Exception {
+Set set = new HashSet();
+int result = JdbcUtil.retry(() -> {
+boolean shouldThrowException = set.isEmpty();
+set.add(true);
+if (shouldThrowException) {
+throw new Exception("test");
+}
+return 1;
+});
+Assert.assertEquals(1, result);
+
+}
}
(kylin) 07/35: KYLIN-5799 Fix round, bround, lpad, rpad function
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit deb209f5319bc6d2dc0fe727d65202e3b834f55c
Author: Zhimin Wu <596361...@qq.com>
AuthorDate: Wed Aug 30 18:36:05 2023 +0800
KYLIN-5799 Fix round, bround, lpad, rpad function
---
.../org/apache/kylin/query/udf/SparkMathUDF.java | 7 +-
.../org/apache/kylin/query/udf/SparkStringUDF.java | 15 +++--
.../kylin/query/runtime/ExpressionConverter.scala | 25 --
.../kylin/query/runtime/SparderRexVisitor.scala| 1 -
4 files changed, 23 insertions(+), 25 deletions(-)
diff --git
a/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
b/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
index ae20f7d1a3..38db80697d 100644
--- a/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
+++ b/src/query/src/main/java/org/apache/kylin/query/udf/SparkMathUDF.java
@@ -24,7 +24,12 @@ import
org.apache.kylin.common.exception.CalciteNotSupportException;
public class SparkMathUDF implements NotConstant {
-public Object BROUND(@Parameter(name = "str1") Double exp1,
@Parameter(name = "str2") Integer exp2)
+public Object BROUND(@Parameter(name = "str1") Object exp1,
@Parameter(name = "str2") Object exp2)
+throws CalciteNotSupportException {
+throw new CalciteNotSupportException();
+}
+
+public Object BROUND(@Parameter(name = "str1") Object exp1)
throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
diff --git
a/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
b/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
index 867ffbefe1..93c36e6af3 100644
--- a/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
+++ b/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
@@ -64,21 +64,32 @@ public class SparkStringUDF implements NotConstant {
throw new CalciteNotSupportException();
}
-public String LPAD(@Parameter(name = "str1") String exp1, @Parameter(name
= "num2") Integer exp2,
+public String LPAD(@Parameter(name = "str1") String exp1, @Parameter(name
= "num2") Object exp2,
@Parameter(name = "str3") String exp3) throws
CalciteNotSupportException {
throw new CalciteNotSupportException();
}
+public String LPAD(@Parameter(name = "str1") String exp1, @Parameter(name
= "num2") Object exp2)
+throws CalciteNotSupportException {
+throw new CalciteNotSupportException();
+}
+
public String REPLACE(@Parameter(name = "str1") String exp1,
@Parameter(name = "str2") String exp2,
@Parameter(name = "str3") String exp3) throws
CalciteNotSupportException {
throw new CalciteNotSupportException();
}
-public String RPAD(@Parameter(name = "str1") String exp1, @Parameter(name
= "num2") Integer exp2,
+public String RPAD(@Parameter(name = "str1") String exp1, @Parameter(name
= "num2") Object exp2,
@Parameter(name = "str3") String exp3) throws
CalciteNotSupportException {
throw new CalciteNotSupportException();
}
+public String RPAD(@Parameter(name = "str1") String exp1, @Parameter(name
= "num2") Object exp2)
+throws CalciteNotSupportException {
+throw new CalciteNotSupportException();
+}
+
+
public String RTRIM(@Parameter(name = "str1") String exp1) throws
CalciteNotSupportException {
throw new CalciteNotSupportException();
}
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
index 04afba3c94..53dc5b25a5 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
@@ -45,19 +45,20 @@ object ExpressionConverter {
"explode"
)
- private val ternaryParameterFunc = mutable.HashSet("replace",
"substring_index", "lpad", "rpad", "conv", "regexp_extract")
+ private val ternaryParameterFunc = mutable.HashSet("replace",
"substring_index", "conv", "regexp_extract")
private val binaryParameterFunc =
mutable.HashSet("decode", "encode", "find_in_set", "levenshtein", "sha2",
"trunc", "add_
(kylin) 04/35: KYLIN-5796 Support databinder auto grow collections limit
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e824503ecaeca7f657de11bf12113ab5fd1eadb3
Author: haocheni
AuthorDate: Tue Aug 29 14:59:55 2023 +0800
KYLIN-5796 Support databinder auto grow collections limit
---
.../kylin/rest/controller/NBasicController.java| 14
.../kylin/rest/aspect/InsensitiveNameAspect.java | 3 ++
.../org/apache/kylin/common/KylinConfigBase.java | 15 +
.../kylin/common/exception/ServerErrorCode.java| 1 +
.../org/apache/kylin/common/msg/CnMessage.java | 10 ++
.../java/org/apache/kylin/common/msg/Message.java | 9 +
.../resources/kylin_errorcode_conf_en.properties | 1 +
.../resources/kylin_errorcode_conf_zh.properties | 1 +
src/metadata-server/pom.xml| 5 +++
.../controller/NProjectControllerKylinTest.java| 17 --
.../rest/controller/NProjectControllerV2Test.java | 17 --
.../rest/controller/NUserGroupControllerTest.java | 38 ++
.../controller/NUserGroupControllerV2Test.java | 22 +
.../rest/controller/OpenAclTCRControllerTest.java | 15 +++--
.../controller/open/OpenAclTCRControllerTest.java | 8 +++--
.../open/OpenUserGroupControllerTest.java | 14
.../rest/controller/NQueryMetaControllerTest.java | 2 ++
.../rest/controller/SparkSourceControllerTest.java | 12 ---
18 files changed, 127 insertions(+), 77 deletions(-)
diff --git
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index e88ce19d72..61854b7436 100644
---
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -24,6 +24,7 @@ import static
org.apache.kylin.common.exception.ServerErrorCode.EMPTY_PROJECT_NA
import static
org.apache.kylin.common.exception.ServerErrorCode.FAILED_CONNECT_CATALOG;
import static
org.apache.kylin.common.exception.ServerErrorCode.FAILED_DOWNLOAD_FILE;
import static
org.apache.kylin.common.exception.ServerErrorCode.INCORRECT_PROJECT_MODE;
+import static
org.apache.kylin.common.exception.ServerErrorCode.INVALID_DATA_BINDER_AUTO_GROW_COLLECTION_LIMIT;
import static
org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER;
import static
org.apache.kylin.common.exception.ServerErrorCode.LOW_LEVEL_LICENSE;
import static
org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION;
@@ -110,6 +111,7 @@ import org.apache.kylin.rest.util.PagingUtil;
import org.apache.kylin.util.DataRangeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.InvalidPropertyException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.HttpEntity;
@@ -128,7 +130,9 @@ import
org.springframework.transaction.CannotCreateTransactionException;
import org.springframework.util.StreamUtils;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.MissingServletRequestParameterException;
+import org.springframework.web.bind.WebDataBinder;
import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.InitBinder;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.client.RequestCallback;
@@ -184,6 +188,10 @@ public class NBasicController {
if (cause instanceof CannotCreateTransactionException) {
kylinException = new KylinException(FAILED_CONNECT_CATALOG,
msg.getConnectDatabaseError(), false);
}
+if (cause instanceof InvalidPropertyException && cause.getCause()
instanceof IndexOutOfBoundsException) {
+kylinException = new
KylinException(INVALID_DATA_BINDER_AUTO_GROW_COLLECTION_LIMIT,
+msg.getInvalidDataBinderAutoGrowCollectionLimit(),
false);
+}
if (cause instanceof KylinException) {
kylinException = (KylinException) cause;
}
@@ -679,4 +687,10 @@ public class NBasicController {
}
}
}
+
+@InitBinder
+private void initBinder(WebDataBinder binder) {
+int autoGrowCollectionLimit =
KylinConfig.getInstanceFromEnv().getDataBinderAutoGrowCollectionLimit();
+binder.setAutoGrowCollectionLimit(autoGrowCollectionLimit);
+}
}
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/aspect/InsensitiveNameAspect.java
b/src/common-service/src/main/java/org/apac
(kylin) 34/35: KYLIN-5845 Avoid model initialization for multi-thread
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0bbb74ee794beb17a8a501f5df5dfcc421bfd2a1
Author: Pengfei Zhan
AuthorDate: Tue Oct 31 11:11:03 2023 +0800
KYLIN-5845 Avoid model initialization for multi-thread
---
.../main/java/org/apache/kylin/common/util/AddressUtil.java | 4
.../java/org/apache/kylin/common/util/AddressUtilTest.java | 1 +
.../org/apache/kylin/metadata/model/ColExcludedChecker.java | 13 +++--
.../java/org/apache/kylin/metadata/model/NDataModel.java| 4
.../org/apache/kylin/metadata/model/NDataModelManager.java | 4
.../java/org/apache/kylin/tool/MetadataToolTestFixture.java | 2 +-
6 files changed, 13 insertions(+), 15 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
index 71cd033143..bbb80a332f 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/util/AddressUtil.java
@@ -107,4 +107,8 @@ public class AddressUtil {
public static String concatInstanceName() {
return AddressUtil.getLocalHostExactAddress() + ":" +
KylinConfig.getInstanceFromEnv().getServerPort();
}
+
+public static void clearLocalIpAddressCache() {
+localIpAddressCache = null;
+}
}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
index 5277bec0c6..0112d7b2ba 100644
---
a/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/AddressUtilTest.java
@@ -72,6 +72,7 @@ public class AddressUtilTest {
val old = getTestConfig().getServerIpAddress();
val mockIp = "192.168.1.101";
getTestConfig().setProperty("kylin.env.ip-address", mockIp);
+AddressUtil.clearLocalIpAddressCache();
val servIp = AddressUtil.getLocalHostExactAddress();
Assert.assertEquals(servIp, mockIp);
if (!StringUtils.isEmpty(old)) {
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
index 3f63975bc4..ab2ae1e877 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
@@ -29,12 +29,11 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
-
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -84,11 +83,13 @@ public class ColExcludedChecker {
collectExcludedComputedColumns(config, project, model);
}
-private void collectExcludedComputedColumns(KylinConfig config, String
project, NDataModel model) {
-if (model == null || model.isBroken()) {
+private void collectExcludedComputedColumns(KylinConfig config, String
project, NDataModel originModel) {
+if (originModel == null || originModel.isBroken()) {
return;
}
-if (!model.isInitAlready()) {
+NDataModel model = originModel;
+if (originModel.getAllTables().isEmpty() ||
originModel.getEffectiveCols() == null) {
+model = NDataModelManager.getInstance(config,
project).copyForWrite(originModel);
model.init(config, project, Lists.newArrayList());
}
model.getAllTables().stream().filter(Objects::nonNull) //
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index 183b5ebc59..3e981ade03 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -286,9 +286,6 @@ public class NDataModel extends RootPersistentEntity {
// mark this model as used for model save checking
private boolean saveCheck = false;
-// mark this model has invoked init() function
-private boolean init
(kylin) 31/35: KYLIN-5840 Fix vulnerability, Upgrade org.xerial.snappy:snappy-java to version 1.1.10.4
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 1c05436a01d41e490c986ee32c0ca1d07c6e90e0 Author: huangsheng AuthorDate: Sat Oct 7 16:46:03 2023 +0800 KYLIN-5840 Fix vulnerability, Upgrade org.xerial.snappy:snappy-java to version 1.1.10.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 07224bd0f1..04a01dc8a8 100644 --- a/pom.xml +++ b/pom.xml @@ -400,7 +400,7 @@ 2.13.4.2 9.23 1.2.1 -1.1.10.1 +1.1.10.4
(kylin) 16/35: KYLIN-5810 Add the spark.session.isDefined check in queryExec
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit dcfa26067ff19352e064801d191f27c98fe1e6d2
Author: Yinghao Lin <39019287+yhca...@users.noreply.github.com>
AuthorDate: Wed Sep 6 14:04:06 2023 +0800
KYLIN-5810 Add the spark.session.isDefined check in queryExec
---
.../src/main/java/org/apache/kylin/query/engine/QueryExec.java | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index d9015fb340..442eea7faf 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -78,6 +78,7 @@ import org.apache.kylin.query.util.QueryContextCutter;
import org.apache.kylin.query.util.QueryInterruptChecker;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.RelAggPushDownUtil;
+import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,7 +274,7 @@ public class QueryExec {
postOptRules.addAll(HepUtils.AggPushDownRules);
}
-if(kylinConfig.isScalarSubqueryJoinEnabled()) {
+if (kylinConfig.isScalarSubqueryJoinEnabled()) {
postOptRules.addAll(HepUtils.ScalarSubqueryJoinRules);
}
@@ -598,14 +599,16 @@ public class QueryExec {
}
private String processAcceptCacheTimeInSql(String sql) {
-if (kylinConfig.isKylinLocalCacheEnabled() &&
kylinConfig.isKylinFileStatusCacheEnabled()) {
+if (kylinConfig.isKylinLocalCacheEnabled() &&
kylinConfig.isKylinFileStatusCacheEnabled()
+&& SparkSession.getDefaultSession().isDefined()) {
return KylinCacheFileSystem.processAcceptCacheTimeInSql(sql);
}
return sql;
}
private void clearAcceptCacheTimeLocally() {
-if (kylinConfig.isKylinLocalCacheEnabled() &&
kylinConfig.isKylinFileStatusCacheEnabled()) {
+if (kylinConfig.isKylinLocalCacheEnabled() &&
kylinConfig.isKylinFileStatusCacheEnabled()
+&& SparkSession.getDefaultSession().isDefined()) {
KylinCacheFileSystem.clearAcceptCacheTimeLocally();
}
}
(kylin) 30/35: KYLIN-5839 Block agg-push-down for non-equal-join
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit be22ca0de7e512df5dc443b99ed6df63635f9dfe
Author: Pengfei Zhan
AuthorDate: Mon Oct 2 17:48:57 2023 +0800
KYLIN-5839 Block agg-push-down for non-equal-join
KYLIN-5382 only deals with the equal-join conditions and gives the
corresponding test cases.
KYLIN-5831 fixed the problem where non-equiv-join conditions were
incorrectly converted to equivalent conditions. Due to the lack of test
cases, a regression bug occurs in ContextUtil.isJoinFromSameContext.
The related non-equiv-join sql is as follows.
If you want a corresponding equal-join sql, just remove the condition
content `or (t2.fcol_10 is null and t5.fcol_17 is null)`.
```sql
select t2.fcol_7 fcol_7, count(distinct t2.fcol_6) fcol_6
from (select t1.company_code fcol_6, t1.type_name fcol_7,
case when t1.created_date = t0.fcol_1 then 'TRUE'
else 'FALSE' end fcol_10
from ( select company_code, created_date, type_name
from "DEFAULT"."TEST_AGG_PUSH_DOWN" ) t1
join ( select company_code, max(created_date) fcol_1
from "DEFAULT"."TEST_AGG_PUSH_DOWN"
group by company_code
) t0 on t1.company_code = t0.company_code
) t2 join ( select 'TRUE' fcol_17 ) t5
on t2.fcol_10 = t5.fcol_17 or (t2.fcol_10 is null and t5.fcol_17 is
null)
group by t2.fcol_7
```
---
.../apache/kylin/query/relnode/ContextUtil.java| 24 ++
.../kylin/query/relnode/KapNonEquiJoinRel.java | 3 +++
2 files changed, 19 insertions(+), 8 deletions(-)
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
index d08d0d073d..4a063cf82f 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
@@ -36,15 +36,14 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexSlot;
import org.apache.calcite.sql.fun.SqlCountAggFunction;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.util.RexUtils;
import org.apache.kylin.util.CalciteSystemProperty;
import org.slf4j.Logger;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
public class ContextUtil {
private ContextUtil() {
}
@@ -203,12 +202,21 @@ public class ContextUtil {
return false;
}
if (potentialSubRel instanceof KapProjectRel) {
-((KapJoinRel) joinRel).leftKeys.forEach(leftKey -> {
-RexNode leftCol = ((KapProjectRel)
potentialSubRel).getProjects().get(leftKey);
-if (leftCol instanceof RexCall) {
-indexOfInputCols.add(leftKey);
+if (joinRel instanceof KapJoinRel) {
+((KapJoinRel) joinRel).leftKeys.forEach(leftKey -> {
+RexNode leftCol = ((KapProjectRel)
potentialSubRel).getProjects().get(leftKey);
+if (leftCol instanceof RexCall) {
+indexOfInputCols.add(leftKey);
+}
+});
+} else {
+// non-equiv-join rel: treat the left and right subtrees as a
context,
+// and refuse to push agg down.
+KapNonEquiJoinRel nonEquivJoinRel = (KapNonEquiJoinRel)
joinRel;
+if (!nonEquivJoinRel.isScd2Rel()) {
+return false;
}
-});
+}
}
return derivedFromSameContext(indexOfInputCols, potentialSubRel,
subContext, hasCountConstant);
}
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
index d3cfda0c2c..0b9840cc1b 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapNonEquiJoinRel.java
@@ -63,6 +63,8 @@ import org.apache.kylin.query.util.ICutContextStrategy;
import org.apache.kylin.query.util.RexToTblColRefTranslator;
import org.apache.kylin.query.util.RexUtils;
+import lombok.Getter;
+
public class KapNonEquiJoinRel extends EnumerableThetaJoin implements KapRel {
private OLAPC
(kylin) 15/35: KYLIN-5809 Add TableIndex suggestion control in open api
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 630a3aa7cef2e2c36653d2d92915b01c201d1ae6
Author: lixiang <447399...@qq.com>
AuthorDate: Sat Sep 2 11:24:20 2023 +0800
KYLIN-5809 Add TableIndex suggestion control in open api
---
.../java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java| 2 ++
1 file changed, 2 insertions(+)
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
index 8ab786b427..7b8959e374 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/LayoutRecDetailResponse.java
@@ -43,6 +43,8 @@ public class LayoutRecDetailResponse implements Serializable {
private List measures = Lists.newArrayList();
@JsonProperty("computed_columns")
private List computedColumns = Lists.newArrayList();
+@JsonProperty("discarded")
+private boolean discarded;
@Getter
@Setter
(kylin) 08/35: KYLIN-5800 & KYLIN-5812 Support scalar subquery join aggregate pushdown
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit dc344ce2fff2b2ca1ece1955b98c4e4130fecbd3
Author: Yu Gan
AuthorDate: Fri Aug 25 16:01:23 2023 +0800
KYLIN-5800 & KYLIN-5812 Support scalar subquery join aggregate pushdown
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +
.../query/rules/ScalarSubqueryJoinRuleTest.java| 112 +++
.../query/rules/ScalarSubqueryJoinRuleTest.xml | 778
.../query/sql_scalar_subquery/query01.sql | 26 +
.../query/sql_scalar_subquery/query02.sql | 30 +
.../query/sql_scalar_subquery/query03.sql | 32 +
.../query/sql_scalar_subquery/query04.sql | 28 +
.../query/sql_scalar_subquery/query05.sql | 37 +
.../query/sql_scalar_subquery/query06.sql | 35 +
.../query/sql_scalar_subquery/query07.sql | 33 +
.../query/sql_scalar_subquery/query08.sql | 36 +
.../query/sql_scalar_subquery/query09.sql | 36 +
.../query/sql_scalar_subquery/query10.sql | 33 +
.../query/sql_scalar_subquery/query11.sql | 32 +
.../query/sql_scalar_subquery/query12.sql | 41 ++
.../query/sql_scalar_subquery/query13.sql | 36 +
.../query/sql_scalar_subquery/query14.sql | 32 +
.../query/sql_scalar_subquery/query15.sql | 27 +
.../query/sql_scalar_subquery/query16.sql | 33 +
.../query/sql_scalar_subquery/query17.sql | 31 +
.../query/sql_scalar_subquery/query18.sql | 65 ++
.../query/sql_scalar_subquery/query19.sql | 26 +
.../apache/kylin/query/relnode/KapValuesRel.java | 21 +-
.../apache/kylin/query/relnode/OLAPValuesRel.java | 13 +-
.../kyligence/kap/query/optrule/KAPValuesRule.java | 11 +-
.../kap/query/optrule/ScalarSubqueryJoinRule.java | 799 +
.../org/apache/kylin/query/engine/QueryExec.java | 4 +
.../java/org/apache/kylin/query/util/HepUtils.java | 20 +-
28 files changed, 2393 insertions(+), 18 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8b9a0308f7..64b4a775d4 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2532,6 +2532,10 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.query.calcite.aggregate-pushdown-enabled",
FALSE));
}
+public boolean isScalarSubqueryJoinEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.query.scalar-subquery-join-enabled",
FALSE));
+}
+
public int getCalciteBindableCacheSize() {
return
Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.maxSize",
"10"));
}
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java
new file mode 100644
index 00..0c8c75ac89
--- /dev/null
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/ScalarSubqueryJoinRuleTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.rules;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
+import org.apache.calcite.test.DiffRepository;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import io.kyligence.kap.query.optrule.AggregateProjectReduceRule;
+import io.kyligence.kap.query.optrule.KapAggregateR
(kylin) 14/35: KYLIN-5808 Optimize performance when saving model/adding CC
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c4aad6e78b24c36cad6ce18a2d00dfd86527be0d
Author: Yaguang Jia
AuthorDate: Sat Sep 2 09:16:38 2023 +0800
KYLIN-5808 Optimize performance when saving model/adding CC
---
.../engine/spark/utils/ComputedColumnEvalUtil.java | 20 +++-
.../job/stage/build/FlatTableAndDictBase.scala | 57
.../spark/smarter/IndexDependencyParser.scala | 60 +++---
3 files changed, 106 insertions(+), 31 deletions(-)
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
index 603dbd7a5a..f7e9b7936e 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/utils/ComputedColumnEvalUtil.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.engine.spark.job.NSparkCubingUtil;
import org.apache.kylin.engine.spark.smarter.IndexDependencyParser;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.model.BadModelException;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
@@ -33,13 +34,9 @@ import
org.apache.kylin.metadata.model.exception.IllegalCCExpressionException;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
-import org.apache.spark.sql.SparderEnv;
-import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.util.SparderTypeUtil;
import org.springframework.util.CollectionUtils;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -62,7 +59,7 @@ public class ComputedColumnEvalUtil {
public static void evalDataTypeOfCCInAuto(List
computedColumns, NDataModel nDataModel,
int start, int end) {
try {
-evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(),
nDataModel, start, end);
+evalDataTypeOfCC(computedColumns, nDataModel, start, end);
} catch (Exception e) {
if (end - start > 1) { //numbers of CC > 1
evalDataTypeOfCCInAuto(computedColumns, nDataModel, start,
start + (end - start) / 2);
@@ -80,7 +77,7 @@ public class ComputedColumnEvalUtil {
return;
}
try {
-evalDataTypeOfCC(computedColumns, SparderEnv.getSparkSession(),
nDataModel, 0, computedColumns.size());
+evalDataTypeOfCC(computedColumns, nDataModel, 0,
computedColumns.size());
} catch (Exception e) {
evalDataTypeOfCCInManual(computedColumns, nDataModel, 0,
computedColumns.size());
}
@@ -90,7 +87,7 @@ public class ComputedColumnEvalUtil {
int start, int end) {
for (int i = start; i < end; i++) {
try {
-evalDataTypeOfCC(computedColumns,
SparderEnv.getSparkSession(), nDataModel, i, i + 1);
+evalDataTypeOfCC(computedColumns, nDataModel, i, i + 1);
} catch (Exception e) {
Preconditions.checkNotNull(computedColumns.get(i));
throw new
IllegalCCExpressionException(QueryErrorCode.CC_EXPRESSION_ILLEGAL,
@@ -101,15 +98,14 @@ public class ComputedColumnEvalUtil {
}
}
-private static void evalDataTypeOfCC(List
computedColumns, SparkSession ss,
-NDataModel nDataModel, int start, int end) {
+private static void evalDataTypeOfCC(List
computedColumns, NDataModel nDataModel, int start,
+int end) {
IndexDependencyParser parser = new IndexDependencyParser(nDataModel);
-Dataset originDf = parser.generateFullFlatTableDF(ss, nDataModel);
-originDf.persist();
+Dataset df = parser.getFullFlatTableDataFrame(nDataModel);
String[] ccExprArray = computedColumns.subList(start, end).stream() //
.map(ComputedColumnDesc::getInnerExpression) //
.map(NSparkCubingUtil::convertFromDotWithBackTick).toArray(String[]::new);
-Dataset ds = originDf.selectExpr(ccExprArray);
+Dataset ds = df.selectExpr(ccExprArray);
for (int i = start; i < end; i++) {
String dataType =
SparderTypeUtil.convertSparkTypeToSqlType(ds.schema().fields()[i -
start].dataType());
computedColumns.get(i).setDatatype(dataType);
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job
(kylin) 10/35: KYLIN-5802 Models with the same name exist when restore metadata
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9b8050e00af23094104bf35439a5fd36470aa23a
Author: Hang Jia <754332...@qq.com>
AuthorDate: Thu Aug 31 11:13:06 2023 +0800
KYLIN-5802 Models with the same name exist when restore metadata
---
.../kylin/common/exception/code/ErrorCodeTool.java | 3 +-
.../resources/kylin_error_msg_conf_cn.properties | 1 +
.../resources/kylin_error_msg_conf_en.properties | 1 +
.../main/resources/kylin_errorcode_conf.properties | 1 +
.../apache/kylin/helper/MetadataToolHelper.java| 118 +
.../org/apache/kylin/tool/MetadataToolTest.java| 57 ++
6 files changed, 180 insertions(+), 1 deletion(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
index 61cf7d13cd..6a7a46bee4 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeTool.java
@@ -24,7 +24,8 @@ public enum ErrorCodeTool implements ErrorCodeProducer {
"KE-050040203"), PARAMETER_TIMESTAMP_COMPARE("KE-050040204"),
// path & file
-PATH_NOT_EXISTS("KE-050041201"), FILE_ALREADY_EXISTS("KE-050041202");
+PATH_NOT_EXISTS("KE-050041201"), FILE_ALREADY_EXISTS("KE-050041202"),
+MODEL_DUPLICATE_UUID_FAILED("KE-050041203");
private final ErrorCode errorCode;
private final ErrorMsg errorMsg;
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 010c2b10c3..2e19879659 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -226,6 +226,7 @@ KE-050040204=参数 “-endTime” <= 参数 “-startTime” 。
## 500412XX
KE-050041201=该路径不存在：%s。
KE-050041202=该路径已存在：%s。
+KE-050041203=请修改模型名称后再恢复:[project]:models: %s
# Common
## KE-060100201
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 75f05b384b..f9d66504ca 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -226,6 +226,7 @@ KE-050040204=Parameter "-endTime" <= Parameter "-startTime".
## 500412XX
KE-050041201=The path does not exist: %s.
KE-050041202=The path already exists: %s.
+KE-050041203=Please modify the model name and then restore:[project]:models: %s
# Common
## KE-060100201
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index f8ed420188..f8dd010445 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -237,6 +237,7 @@ KE-050040204
## 500412XX
KE-050041201
KE-050041202
+KE-050041203
# Common
KE-060100201
diff --git
a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index f000e766a6..f3d920d918 100644
--- a/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
+++ b/src/tool/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -19,6 +19,7 @@
package org.apache.kylin.helper;
import static
org.apache.kylin.common.exception.code.ErrorCodeTool.FILE_ALREADY_EXISTS;
+import static
org.apache.kylin.common.exception.code.ErrorCodeTool.MODEL_DUPLICATE_UUID_FAILED;
import java.io.File;
import java.io.IOException;
@@ -28,9 +29,12 @@ import java.nio.file.Paths;
import java.time.Clock;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.NavigableSet;
import java.util.Objects;
import java.util.Set;
@@ -58,8 +62,10 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.guava30.shaded.common.io.ByteSource;
+import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.a
(kylin) 06/35: KYLIN-5798 Add user-table access cache for table-loading
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4d3418c237d4a839a85573712d0111d81086a477
Author: Yaguang Jia
AuthorDate: Wed Aug 30 14:11:32 2023 +0800
KYLIN-5798 Add user-table access cache for table-loading
---
.../org/apache/kylin/common/KylinConfigBase.java | 13 +++
.../apache/kylin/common/KylinConfigBaseTest.java | 22 ++
.../apache/kylin/rest/service/SparkDDLTest.java| 26 ++
.../spark/source/NSparkMetadataExplorer.java | 22 +++---
.../spark/source/NSparkMetadataExplorerTest.java | 14 +++-
5 files changed, 79 insertions(+), 18 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3de55bf84e..8b9a0308f7 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2800,6 +2800,19 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.source.hive.table-access-filter-enabled",
FALSE));
}
+public boolean getTableAccessCacheEnable() {
+return
Boolean.parseBoolean(getOptional("kylin.source.hive.table-access-cache-enabled",
TRUE));
+}
+
+public long getTableAccessCacheSize() {
+return
Long.parseLong(getOptional("kylin.source.hive.table-access-cache-size",
ONE_HUNDRED_THOUSAND));
+}
+
+public long getTableAccessCacheTTL() {
+return
TimeUtil.timeStringAs(getOptional("kylin.source.hive.table-access-cache-ttl",
"7d"),
+TimeUnit.MINUTES);
+}
+
public String[] getHiveDatabases() {
return getOptionalStringArray("kylin.source.hive.databases", new
String[0]);
}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 6be4411004..59e4a3fc9b 100644
---
a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++
b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -36,6 +36,7 @@
package org.apache.kylin.common;
+import static org.apache.kylin.common.KylinConfigBase.FALSE;
import static org.apache.kylin.common.KylinConfigBase.PATH_DELIMITER;
import static
org.apache.kylin.common.KylinConfigBase.WRITING_CLUSTER_WORKING_DIR;
import static
org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_ENABLE_KEY;
@@ -1499,6 +1500,27 @@ class KylinConfigBaseTest {
config.setProperty("kylin.metadata.audit-log.max-size", "300");
assertEquals(300, config.getMetadataAuditLogMaxSize());
}
+
+@Test
+void testGetTableAccessCache() {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+
+assertTrue(config.getTableAccessCacheEnable());
+config.setProperty("kylin.source.hive.table-access-cache-enabled",
FALSE);
+assertFalse(config.getTableAccessCacheEnable());
+
+assertEquals(10, config.getTableAccessCacheSize());
+config.setProperty("kylin.source.hive.table-access-cache-size",
"20");
+assertEquals(20, config.getTableAccessCacheSize());
+
+assertEquals(10080, config.getTableAccessCacheTTL());
+config.setProperty("kylin.source.hive.table-access-cache-ttl", "1m");
+assertEquals(1, config.getTableAccessCacheTTL());
+config.setProperty("kylin.source.hive.table-access-cache-ttl", "1h");
+assertEquals(60, config.getTableAccessCacheTTL());
+config.setProperty("kylin.source.hive.table-access-cache-ttl", "1d");
+assertEquals(1440, config.getTableAccessCacheTTL());
+}
}
class EnvironmentUpdateUtils {
diff --git
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
index cf23d159aa..dc293e2b0c 100644
---
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
+++
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/SparkDDLTest.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
(kylin) 03/35: KYLIN-5795 Remove the check of `hadoop.tmp.dir`
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 91383f54d5c70bf6ce0f3f641dcd186de193029b
Author: Yaguang Jia
AuthorDate: Mon Aug 28 15:41:05 2023 +0800
KYLIN-5795 Remove the check of `hadoop.tmp.dir`
---
.../src/main/java/org/apache/kylin/common/util/HadoopUtil.java| 4
1 file changed, 4 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
index 8e4250fc0f..0e3ac41ae5 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
@@ -98,10 +98,6 @@ public class HadoopUtil {
}
public static Configuration healSickConfig(Configuration conf) {
-// https://issues.apache.org/jira/browse/KYLIN-953
-if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) {
-conf.set("hadoop.tmp.dir", "/tmp");
-}
// https://issues.apache.org/jira/browse/KYLIN-3064
conf.set("yarn.timeline-service.enabled", "false");
(kylin) branch kylin5 updated (1d1ecbec63 -> 3f9b9c83be)
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from 1d1ecbec63 KYLIN-5823,fix text recognition bug
new d6285db6ab KYLIN-5792 Support ldap memberid mode
new 07354967d4 KYLIN-5793 Fix Chinese gibberish when Spring ${} attribute
injection
new 91383f54d5 KYLIN-5795 Remove the check of `hadoop.tmp.dir`
new e824503eca KYLIN-5796 Support databinder auto grow collections limit
new 82ee25fdf1 KYLIN-5797 Remove nacos jar
new 4d3418c237 KYLIN-5798 Add user-table access cache for table-loading
new deb209f531 KYLIN-5799 Fix round, bround, lpad, rpad function
new dc344ce2ff KYLIN-5800 & KYLIN-5812 Support scalar subquery join
aggregate pushdown
new e33ef57cce KYLIN-5791 & KYLIN-5801 Fix computation of timestampdff &
add_month
new 9b8050e00a KYLIN-5802 Models with the same name exist when restore
metadata
new bfff6d002c KYLIN-5803 Add file segments and smart cache
new 085d707216 KYLIN-5805 Refactor project lock
new a212350a51 KYLIN-5807 Fix query execute as user without data query acl
new c4aad6e78b KYLIN-5808 Optimize performance when saving model/adding CC
new 630a3aa7ce KYLIN-5809 Add TableIndex suggestion control in open api
new dcfa26067f KYLIN-5810 Add the spark.session.isDefined check in
queryExec
new 5092e7a239 KYLIN-5811 Enhancement of SCD2
new ebd2f3d670 KYLIN-5814 Support lettuce sentinel mode
new 768a5ececc KYLIN-5815 Failed to add primaryKey of table when upgrading
concurrently
new 2159699d8e KYLIN-5816 Add zk lock for dict v3
new 00352e8c2d KYLIN-5817 Query scanRows and bytes are incorrect
new c61dc41899 KYLIN-5828 Concurrently dict v2 jobs lead to abnormal
encoding result
new f9255a515d KYLIN-5829 Add a tool for generating sql from dumped
metadata
new 191d08c272 KYLIN-5830 Push columns of sortRel to subOlapContexts
new d7f37675e2 KYLIN-5832 Support queryDetect api
new 412eff5c73 KYLIN-5833 Use TableIndex answer select * only works for
query
new e49c2f15a6 KYLIN-5836 Fix mapr env building error
new 148d8f61aa KYLIN-5837 Support only reusing user defined computedColumns
new eacb62b8aa KYLIN-5838 Replace join expression with the name of
computedColumn
new be22ca0de7 KYLIN-5839 Block agg-push-down for non-equal-join
new 1c05436a01 KYLIN-5840 Fix vulnerability, Upgrade
org.xerial.snappy:snappy-java to version 1.1.10.4
new 7d24ce642c KYLIN-5841 Fix vulnerability, Upgrade org.apache.avro:avro
to version 1.11.3
new e804eee81e KYLIN-5844 & KYLIN-5843 Column ACL works for PushDown
new 0bbb74ee79 KYLIN-5845 Avoid model initialization for multi-thread
new 3f9b9c83be KYLIN-5846 upgrade spark version to 3.2.0-kylin-4.6.16.0
The 35 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build/bin/create-table.sh | 32 +
build/conf/kylin-tools-diag-log4j.xml | 6 +
build/conf/kylin-tools-log4j.xml | 6 +
.../kap/secondstorage/metadata/Manager.java| 23 +-
.../kap/secondstorage/metadata/NodeGroup.java | 5 +
.../kap/secondstorage/metadata/TablePlan.java | 5 +
pom.xml| 48 +-
.../kylin/rest/controller/NBasicController.java| 14 +
.../kylin/rest/controller/NSystemController.java | 33 +
src/common-service/pom.xml | 4 +
.../apache/kylin/rest/KylinPrepareEnvListener.java | 3 +-
.../kylin/rest/aspect/InsensitiveNameAspect.java | 3 +
.../kylin/rest/cache/AbstractKylinCache.java | 78 ++
.../apache/kylin/rest/cache/CacheConstant.java}| 12 +-
.../org/apache/kylin/rest/cache/RedisCache.java| 91 +--
.../org/apache/kylin/rest/cache/RedisCacheV2.java | 224 ++
.../apache/kylin/rest/config/AppInitializer.java | 21 +
.../config/initialize/EpochChangedListener.java| 16 +-
.../config/initialize/ModelBrokenListener.java | 12 +-
.../rest/request/SyncFileSegmentsRequest.java} | 23 +-
.../rest/response/SyncFileSegmentsResponse.java} | 29 +-
.../apache/kylin/rest/service/AccessService.java | 14 +-
.../kylin/rest/service/KylinUserService.java | 8 +-
.../apache/kylin/rest/service/ProjectService.java | 5 +-
.../apache/kylin/rest/service/SystemService.java | 44 +-
.../service/task/QueryHistoryTaskScheduler.java| 21 +-
.../task/RecommendationTopNUpdateScheduler.java| 16 +-
.../kylin/rest/cache/CacheConstantTest.java} | 13 +-
.../apache/kylin/rest/cache/RedisCacheTest.java} | 20 +-
.../apache/kylin/rest/cache/RedisCacheV2Test.java | 195 +
.../tas
(kylin) 02/35: KYLIN-5793 Fix Chinese gibberish when Spring ${} attribute injection
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 07354967d424f3ba76d87ea562948f8b5f69a3a5
Author: Liang.Hua <36814772+jacob...@users.noreply.github.com>
AuthorDate: Mon Aug 28 14:56:11 2023 +0800
KYLIN-5793 Fix Chinese gibberish when Spring ${} attribute injection
---
.../org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java| 1 +
1 file changed, 1 insertion(+)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
index 3f30693497..1f715fb8bb 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/rest/security/PasswordPlaceholderConfigurer.java
@@ -62,6 +62,7 @@ public class PasswordPlaceholderConfigurer extends
PropertyPlaceholderConfigurer
}
InputStream is = IOUtils.toInputStream(propString,
Charset.defaultCharset());
resources[0] = new InputStreamResource(is);
+this.setFileEncoding(Charset.defaultCharset().toString());
this.setLocations(resources);
}
(kylin) branch kylin5 updated: KYLIN-5823,fix text recognition bug
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 1d1ecbec63 KYLIN-5823,fix text recognition bug
1d1ecbec63 is described below
commit 1d1ecbec63e0b61251cff87ad06082fa84ac8289
Author: huangchunyan
AuthorDate: Thu Apr 11 11:45:33 2024 +0800
KYLIN-5823,fix text recognition bug
---
kystudio/src/components/common/RecognizeAggregateModal/store.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/kystudio/src/components/common/RecognizeAggregateModal/store.js
b/kystudio/src/components/common/RecognizeAggregateModal/store.js
index d2d2009ab0..18e8b15a16 100644
--- a/kystudio/src/components/common/RecognizeAggregateModal/store.js
+++ b/kystudio/src/components/common/RecognizeAggregateModal/store.js
@@ -1,4 +1,5 @@
import { AGGREGATE_TYPE } from '../../../config'
+import { objectClone } from '../../../util/index'
const types = {
SHOW_MODAL: 'SHOW_MODAL',
(kylin) branch kylin5 updated: KYLIN-5825,fix aggregate modal and table index modal ui and fix chrome 50 layout bug
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 6adffadde7 KYLIN-5825,fix aggregate modal and table index modal ui and
fix chrome 50 layout bug
6adffadde7 is described below
commit 6adffadde703a0ee7975301eff81766fb9fcbf6e
Author: huangchunyan
AuthorDate: Fri Apr 5 17:45:25 2024 +0800
KYLIN-5825,fix aggregate modal and table index modal ui and fix chrome 50
layout bug
---
.../StudioModel/ModelList/AggregateModal/index.vue | 333 +
.../ModelList/AggregateModal/locales.js| 21 +-
.../ModelList/DataFeatures/dataFeatures.vue| 2 +-
.../StudioModel/TableIndexEdit/tableindex_edit.vue | 32 +-
4 files changed, 162 insertions(+), 226 deletions(-)
diff --git
a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue
b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue
index 129b426541..ef4b60475e 100644
---
a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue
+++
b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue
@@ -71,13 +71,16 @@
-*
{{$t('indexTimeRange')}}
-
+*
- {{$t('kylinLang.common.HYBRID')}}
+
+{{$t('kylinLang.common.HYBRID')}}
+
+
+
{{$t('kylinLang.common.BATCH')}}
@@ -98,7 +101,7 @@
:disabled="!(model.model_type === 'HYBRID' &&
!form.aggregateArray[aggregateIdx].index_range) || (!indexUpdateEnabled &&
['HYBRID', 'STREAMING'].includes(aggregate.index_range))">
{{$t('clearAll')}}
@@ -122,7 +125,7 @@
plain
size="mini"
icon="el-ksd-icon-edit_22"
-class="add-all-item"
+class="add-all-item ksd-ml-8"
type="primary"
:disabled="(model.model_type === 'HYBRID' &&
!form.aggregateArray[aggregateIdx].index_range) || (!indexUpdateEnabled &&
['HYBRID', 'STREAMING'].includes(aggregate.index_range))"
@click="handleEditIncludes(aggregateIdx,
aggregate.id)">{{$t('edit')}}
@@ -152,8 +155,8 @@
-
-
+
+
+{{$t('clearAll')}}
+
- {{$t('clearAll')}}
- {{$t('edit')}}
@@ -458,7 +471,7 @@
{{$t('kylinLang.common.cancel')}}
-{{$t('kylinLang.common.save')}}
+{{$t('kylinLang.common.save')}}
{{$t('saveAndBuild')}}
@@ -470,7 +483,7 @@
-
-{{$t('editIncludeDimensionTip')}}
-
-
-
-
-
-
-
-
-
- {{$t('th_name')}}
-
-
- {{$t('th_column')}}
-
-
- {{$t('th_dataType')}}
- {{$t('cardinality')}}
- {{$t('th_info')}}
- {{$t('th_order')}}
-
-
-
-
-{{item.name}}
-{{item.column}}
-{{item.type}}
-
- NULL
- {{ item.cardinality }}
-
-{{item.comment}}
-
-
-
-
-
-
-
-
-
-
+
+
+
+{{$t('excludeTableCheckbox')}}
+
+
+{{ $t('excludeTableCheckb
(kylin) branch kylin5 updated: KYLIN-5822,support history table adds left-closed and right-closed time interval
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 16f8da9071 KYLIN-5822,support history table adds left-closed and
right-closed time interval
16f8da9071 is described below
commit 16f8da90718a2104729fb2a0c8d3cf61b1f8617f
Author: huangchunyan
AuthorDate: Sat Apr 6 16:31:37 2024 +0800
KYLIN-5822,support history table adds left-closed and right-closed time
interval
---
.../components/setting/SettingAdvanced/locales.js | 6 +-
.../ModelList/Components/ModelTitleDescription.vue | 118 ++---
.../studio/StudioModel/ModelList/locales.js| 2 +-
.../studio/StudioModel/TableJoinModal/index.vue| 75 ++---
.../studio/StudioModel/TableJoinModal/locales.js | 2 +-
5 files changed, 95 insertions(+), 108 deletions(-)
diff --git a/kystudio/src/components/setting/SettingAdvanced/locales.js
b/kystudio/src/components/setting/SettingAdvanced/locales.js
index 121517fd63..1c38df5169 100644
--- a/kystudio/src/components/setting/SettingAdvanced/locales.js
+++ b/kystudio/src/components/setting/SettingAdvanced/locales.js
@@ -51,9 +51,9 @@ export default {
confirmOpenTip: 'Do you want to continue?',
SCD2Settings: 'Support History table',
nonEqualJoin: 'Show non-equal join conditions for History table',
-noEqualDecription: 'With this switch ON, you may use the history table for
slowly changing dimension (as was): Non-equal join conditions (≥, <) could be
used for modeling, building and queries. ',
+noEqualDecription: 'With this switch ON, you may use the history table for
slowly changing dimension (as was): Non-equal join conditions could be used for
modeling, building and queries. ',
confirmOpen: 'Turn On',
-closeSCDTip: 'With this switch OFF, non-equal join conditions (≥, <)
couldn\'t be used when editing model. The following models will go offline
automatically as they include non-equal join conditions:',
+closeSCDTip: 'With this switch OFF, non-equal join conditions couldn\'t be
used when editing model. The following models will go offline automatically as
they include non-equal join conditions:',
closeSCDTip1: 'To make these models online, please delete the non-equal
join conditions, or turn this switch ON. Do you want to continue?',
confirmClose: 'Turn Off',
mulPartitionSettings: 'Multilevel Partitioning',
@@ -63,7 +63,7 @@ export default {
openMulPartitionTip: 'Please note that this feature is still in BETA
phase. Potential risks or known limitations might exist. Check ',
openMulPartitionTip1: ' for details.',
closeMulPartitionSetting: 'Turn Off Multilevel Partitioning',
-closeMulPartitionTip: 'With this switch OFF, multilevel partitioning
couldn’t be used. The following models would go offline automatically as they
used multilevel partitioning:',
+closeMulPartitionTip: 'With this switch OFF, multilevel partitioning
couldn\'t be used. The following models would go offline automatically as they
used multilevel partitioning:',
closeMulPartitionTip1: 'To make these models online, please delete all
subpartitions, or turn this switch ON. Do you want to continue?',
snapshotTitle: 'Snapshot Management',
snapshotManagment: 'Support Snapshot Management',
diff --git
a/kystudio/src/components/studio/StudioModel/ModelList/Components/ModelTitleDescription.vue
b/kystudio/src/components/studio/StudioModel/ModelList/Components/ModelTitleDescription.vue
index b66af1b080..71390cda59 100644
---
a/kystudio/src/components/studio/StudioModel/ModelList/Components/ModelTitleDescription.vue
+++
b/kystudio/src/components/studio/StudioModel/ModelList/Components/ModelTitleDescription.vue
@@ -1,64 +1,64 @@
-
-
-
-
- {{modelData.status}}
- {{$t('emptyIndexTips')}}
-
-{{modelData.model_type === 'HYBRID' ?
$t('modelSegmentHoleTips1') : $t('modelSegmentHoleTips')}}{{$t('seeDetail')}}
+
+
+
+
+{{modelData.status}}
+{{$t('emptyIndexTips')}}
+
+ {{modelData.model_type === 'HYBRID' ?
$t('modelSegmentHoleTips1') : $t('modelSegmentHoleTips')}}{{$t('seeDetail')}}
+
+
+ {{$t('modelSegmentHoleTips2')}}
+
+
+ {{$t('modelMetadataChangedTips')}}{{$t('seeDetail')}}
+
+
+ {{$t('SCD2ModalOfflineTip')}}
+
+
+ {{$t('noSegmentOnlineTip')}}
+
+
+ {{$t('multilParTip')}}
+
+
+
+
+ {{modelData.alias || modelData.name}}
+
+{{$t('kylinLang.model.ownerGrid')}}{{modelData.owner}}
+{{$t('description')}}{{modelData.description || '-'}}
-
-{{$t('modelSegmentHol
(kylin) branch kylin5 updated: KYLIN-5826,fix table index select all issue and fix load user or usergroup list filter issue
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 458bddf09f KYLIN-5826,fix table index select all issue and fix load
user or usergroup list filter issue
458bddf09f is described below
commit 458bddf09f85989bcac1874d3beea6d3791d3129
Author: huangchunyan
AuthorDate: Sat Apr 6 15:07:20 2024 +0800
KYLIN-5826,fix table index select all issue and fix load user or usergroup
list filter issue
---
kystudio/src/components/admin/Group/index.vue | 6 +-
kystudio/src/components/admin/User/index.vue | 10 +-
.../common/CustomTransferData/TransferData.vue | 335 +++--
3 files changed, 177 insertions(+), 174 deletions(-)
diff --git a/kystudio/src/components/admin/Group/index.vue
b/kystudio/src/components/admin/Group/index.vue
index 36cc253bf3..72b49f8d9e 100644
--- a/kystudio/src/components/admin/Group/index.vue
+++ b/kystudio/src/components/admin/Group/index.vue
@@ -135,7 +135,7 @@ export default class SecurityGroup extends Vue {
inputFilter (value) {
this.pagination.page_offset = 0
this.filterName = value
-this.loadGroupUsers(value)
+this.loadGroupUsers()
}
get emptyText () {
@@ -163,11 +163,11 @@ export default class SecurityGroup extends Vue {
}
}
- loadGroupUsers (filterGroupName) {
+ loadGroupUsers () {
this.isLoadingUserGroups = true
this.loadGroupUsersList({
...this.pagination,
- user_group_name: filterGroupName
+ user_group_name: this.filterName
}).then(() => {
this.isLoadingUserGroups = false
}, (res) => {
diff --git a/kystudio/src/components/admin/User/index.vue
b/kystudio/src/components/admin/User/index.vue
index eb323de881..83a89cf888 100644
--- a/kystudio/src/components/admin/User/index.vue
+++ b/kystudio/src/components/admin/User/index.vue
@@ -211,7 +211,7 @@ export default class SecurityUser extends Vue {
inputFilter (value) {
this.pagination.page_offset = 0
this.filterName = value
-this.loadUsers(value)
+this.loadUsers()
}
get emptyText () {
@@ -221,16 +221,16 @@ export default class SecurityUser extends Vue {
handleCurrentChange (pager, pageSize) {
this.pagination.page_offset = pager
this.pagination.page_size = pageSize
-this.loadUsers(this.filterName)
+this.loadUsers()
}
- async loadUsers (name) {
+ async loadUsers () {
this.isLoadingUsers = true
try {
const parameter = {
...this.pagination,
// project: this.currentSelectedProject, // 处理资源组时，发现这个接口不用传 project 参数
-name: name || '',
+name: this.filterName,
group_uuid: this.currentGroup && this.currentGroup.uuid
}
const res = !this.currentGroup
@@ -254,7 +254,7 @@ export default class SecurityUser extends Vue {
async editUser (editType, userDetail) {
if (!this.isTestingSecurityProfile) return
const isSubmit = await this.callUserEditModal({ editType, userDetail })
-isSubmit && this.loadUsers(this.filterName)
+isSubmit && this.loadUsers()
}
async editUserDataPermission (row) {
diff --git a/kystudio/src/components/common/CustomTransferData/TransferData.vue
b/kystudio/src/components/common/CustomTransferData/TransferData.vue
index 848658f500..2305331ba8 100644
--- a/kystudio/src/components/common/CustomTransferData/TransferData.vue
+++ b/kystudio/src/components/common/CustomTransferData/TransferData.vue
@@ -1,186 +1,186 @@
-
-
-
-
-
-
-
-
-
-
-
- {{$t('kylinLang.common.selectAll')}}
-
-
-
-{{$t('column')}}{{columnSort ? `: ${$t(columnSort)}` :
''}}
-
-
-{{$t('type')}}{{typeOptions.length ? `:
${typeOptions.length}` : ''}}
-
-
-
-
-{{$t('column')}}{{columnSort ? `: ${$t(columnSort)}` :
''}}
-
- {{$t('ascending')}}
- {{$t('descending')}}
-
-
-
-{{$t('type')}}{{typeOptions.length ? `:
${typeOptions.length}` : ''}}
-
-
-{{item}}
-
-
-
-
-
- |
-
-{{
$t('textRecognition') }}
-
-
-
-
-
-
-
-
-
-
-
(kylin) branch kylin5 updated: KYLIN-5824,refine handel error for export models
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 9e8ca1a953 KYLIN-5824,refine handel error for export models
9e8ca1a953 is described below
commit 9e8ca1a9536c9ecf0eafca048c8733e3668f2c76
Author: huangchunyan
AuthorDate: Fri Apr 5 17:31:28 2024 +0800
KYLIN-5824,refine handel error for export models
---
.../common/ModelsExportModal/ModelsExportModal.vue | 2 +-
.../ModelList/AggregateModal/locales.js| 4 ++--
kystudio/src/util/business.js | 2 +-
kystudio/src/util/domHelper.js | 28 ++
4 files changed, 28 insertions(+), 8 deletions(-)
diff --git
a/kystudio/src/components/common/ModelsExportModal/ModelsExportModal.vue
b/kystudio/src/components/common/ModelsExportModal/ModelsExportModal.vue
index 3b3b701b39..49f8931d8d 100644
--- a/kystudio/src/components/common/ModelsExportModal/ModelsExportModal.vue
+++ b/kystudio/src/components/common/ModelsExportModal/ModelsExportModal.vue
@@ -270,7 +270,7 @@ export default class ModelsExportModal extends Vue {
this.handleClose(true)
this.$message.success(this.$t('exportSuccess'))
} catch (e) {
- this.$message.error(this.$t('exportFailed'))
+ // this.$message.error(this.$t('exportFailed'))
}
this.isSubmiting = false
}
diff --git
a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js
b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js
index 272a94d5ff..56905e9e0b 100644
---
a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js
+++
b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js
@@ -113,11 +113,11 @@ export default {
excludeTableCheckbox: 'Display columns excluded from recommendations',
excludeTableCheckboxTip: 'If an excluded column is added to indexes, this
column will store "historical truth"(SCD Type2, As Was).',
excludedTableIconTip: 'Excluded from recommendations',
-indexTimeRange: 'Index’s Data Range',
+indexTimeRange: 'Real-time Index Types',
manyToManyAntiTableTip: 'For the tables excluded from recommendations, if
the join relationship of a table is One-to-Many or Many-to-Many, dimensions
from this table can\'t be used in indexes. ',
indexTimeRangeTips: 'The data range that the indexes will be built in.
With “Batch and Streaming“ selected, there will be generated batch indexes and
streaming indexes with same content respectively. ',
refuseAddIndexTip: 'Can\'t add streaming indexes. Please stop the
streaming job and then delete all the streaming segments.',
-disableAddDim: 'Select index\'s data range',
+disableAddDim: 'Please select the real-time index type',
textRecognition: 'Text Recognition'
}
}
diff --git a/kystudio/src/util/business.js b/kystudio/src/util/business.js
index 5aa4d387df..d170361201 100644
--- a/kystudio/src/util/business.js
+++ b/kystudio/src/util/business.js
@@ -30,7 +30,7 @@ export function handleError (res, errorcallback) {
if (window.kylinVm.$store.state.config.platform === 'iframe') {
window.kylinVm.$store.state.config.errorMsgBox.msg = res.message ||
window.kylinVm.$t('kylinLang.common.notConnectServerIframe')
} else {
- window.kylinVm.$store.state.config.errorMsgBox.msg = res.message ||
window.kylinVm.$t('kylinLang.common.notConnectServer')
+ window.kylinVm.$store.state.config.errorMsgBox.msg = res.message ||
(responseData && responseData.msg) ||
window.kylinVm.$t('kylinLang.common.notConnectServer')
}
window.kylinVm.$store.state.config.errorMsgBox.detail = responseData &&
responseData.stacktrace || res.stack || JSON.stringify(res)
} else {
diff --git a/kystudio/src/util/domHelper.js b/kystudio/src/util/domHelper.js
index ddec46a24e..40b8146435 100644
--- a/kystudio/src/util/domHelper.js
+++ b/kystudio/src/util/domHelper.js
@@ -1,5 +1,7 @@
import Vue from 'vue'
import ElementUI from 'kyligence-kylin-ui'
+import $ from 'jquery'
+import { handleError } from './business'
export const download = {
post (url, data) {
@@ -32,11 +34,29 @@ export const download = {
}
document.body.appendChild($form)
-$form.submit()
-
-setTimeout(() => {
- document.body.removeChild($form)
+return $.ajax({
+ headers: {
+'Accept-Language': localStorage.getItem('kystudio_lang') === 'en' ?
'en' : 'cn'
+ },
+ url: $($form).attr('action'), // 获取表单的提交地址
+ method: $($form).attr('method'), // 获取表单的提交方法
+ data: $($form).serialize(), // 序列化表单数据
+ success: (res) => {
+// 当服务器响应成功时的处理逻辑
+$form.submit()
+setTimeout(() => {
+ document.body.removeChild($form)
+})
+ }
(kylin) branch kylin5 updated: KYLIN-5823,improve text recognition dialog of add batch dimensions
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 93fee73901 KYLIN-5823,improve text recognition dialog of add batch
dimensions
93fee73901 is described below
commit 93fee7390113733c4b0c367734b953c0cc271b16
Author: huangchunyan
AuthorDate: Fri Apr 5 17:16:33 2024 +0800
KYLIN-5823,improve text recognition dialog of add batch dimensions
---
.../RecognizeAggregateModal.vue| 208 -
.../common/RecognizeAggregateModal/handler.js | 3 +-
.../common/RecognizeAggregateModal/locales.js | 10 +-
.../common/RecognizeAggregateModal/store.js| 24 ++-
.../studio/StudioModel/DimensionsModal/index.vue | 71 +--
.../studio/StudioModel/DimensionsModal/locales.js | 4 +-
6 files changed, 203 insertions(+), 117 deletions(-)
diff --git
a/kystudio/src/components/common/RecognizeAggregateModal/RecognizeAggregateModal.vue
b/kystudio/src/components/common/RecognizeAggregateModal/RecognizeAggregateModal.vue
index 7ca329cbbf..4ca6f00b91 100644
---
a/kystudio/src/components/common/RecognizeAggregateModal/RecognizeAggregateModal.vue
+++
b/kystudio/src/components/common/RecognizeAggregateModal/RecognizeAggregateModal.vue
@@ -1,83 +1,83 @@
-
-
-
-
-
- {{$tc('errorCount', errorCount, { count:
errorCount })}}
-
-{{$tc('repeatCount', repeatCount, {
count: repeatCount })}}
-
-
-
-
-
-
-
-
-
- {{$t('dexecute')}}{{$t('acceleratorKey')}}
-
+
+
+
+
+
+{{$tc('errorCount', errorCount, { count:
errorCount })}}
+
+ {{$tc('repeatCount', repeatCount, { count:
repeatCount })}}
-
-
-
-
- {{$tc('selectedDimensionCount', selectedDimensionCount, { count:
selectedDimensionCount })}}
-
-|{{$t('usedDimensionCount', {count:
usedDimensionCount})}}
-
-
-
-
- {{$t('dimensionName')}}
- {{$t('dataType')}}
-
-
-
-
-
- {{ item.label }}
- {{ item.dataType }}
-
-
-
-
-
-
-
-{{$t('recognizeFailed')}}
+
+
-
+
+
+
+
+{{$t('dexecute')}}{{$t('acceleratorKey')}}
+
+
-
-
- {{$t('kylinLang.common.cancel')}}
-
-
- {{$t('kylinLang.common.save')}}
-
+
+
+
+{{$tc('selectedDimensionCount', selectedDimensionCount, { count:
selectedDimensionCount })}}
+
+ |{{$t('usedDimensionCount', {count:
usedDimensionCount})}}
+
+
+
+
+{{$t('dimensionName')}}
+{{$t('dataType')}}
+
+
+
+
+
+{{ item.label }}
+{{ item.dataType }}
+
+
+
+
+
+
+
+ {{$t('recognizeFailed')}}
+
+
-
-
+
+
+
+{{$t('kylinLang.common.cancel')}}
+
+
+{{$t('kylinLang.common.save')}}
+
+
+
+
-
+<script>
import Vue from 'vue'
import AceEditor from 'vue2-ace-editor'
import { Component, Watch } from 'vue-property-decorator'
@@ -92,6 +92,7 @@
vuex.registerModule(['modals', 'RecognizeAggregateModal'], store)
const TABLE_INDEX = 'TABLE_INDEX'
+ const DIMENSION = 'DIMENSION'
@Component({
components: {
@@ -105,6 +106,7 @@
type: state => state.type,
status: state => state.status,
form: state => state.form,
+usedColumns: state => state.usedColumns,
errors: state => state.errors,
errorLines: state => state.errorLines,
errorInEditor: state => state.errorInEditor,
@@ -119,7 +121,7 @@
'hierarchyItems',
'joints',
'jointItems',
-'tableIndexCols'
+'allColumns'
])
},
methods: {
@@ -181,16 +183,17 @@
}
get isIndeterminate () {
const { selectedDimensionCount, isSelectAll } = this
- return selectedDimensionCount && !isSelectAll
+ return selectedDimensionCount > 0 && !isSelectAll
}
isColu
(kylin) branch kylin5 updated (e734e6e31b -> 0d742c639c)
This is an automated email from the ASF dual-hosted git repository. liyang pushed a change to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git from e734e6e31b KYLIN-5819,optimize the warning message on the dialog for editing computed column add 0d742c639c KYLIN-5821,fix model edit page pfk flag bug and enhance model canvas window position No new revisions were added by this update. Summary of changes: .../studio/StudioModel/ModelEdit/config.js | 2 +- .../studio/StudioModel/ModelEdit/model.js | 32 -- 2 files changed, 24 insertions(+), 10 deletions(-)
(kylin) branch kylin5 updated: KYLIN-5819,optimize the warning message on the dialog for editing computed column
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new e734e6e31b KYLIN-5819,optimize the warning message on the dialog for
editing computed column
e734e6e31b is described below
commit e734e6e31bd035c59c945ecfa99b136d2f6380b4
Author: huangchunyan
AuthorDate: Fri Apr 5 15:25:32 2024 +0800
KYLIN-5819,optimize the warning message on the dialog for editing computed
column
---
kystudio/src/components/common/GlobalDialog/dialog/detail_dialog.vue | 2 +-
kystudio/src/components/studio/StudioModel/AddCCModal/locales.js | 2 +-
.../components/studio/StudioModel/ModelList/ModelSaveConfig/index.vue | 4 ++--
.../studio/StudioModel/ModelList/ModelSaveConfig/locales.js | 3 ++-
4 files changed, 6 insertions(+), 5 deletions(-)
diff --git
a/kystudio/src/components/common/GlobalDialog/dialog/detail_dialog.vue
b/kystudio/src/components/common/GlobalDialog/dialog/detail_dialog.vue
index 1307c2f737..ec1f04335c 100644
--- a/kystudio/src/components/common/GlobalDialog/dialog/detail_dialog.vue
+++ b/kystudio/src/components/common/GlobalDialog/dialog/detail_dialog.vue
@@ -91,7 +91,7 @@
{{cancelT}}
{{cancelT}}
-{{submitSubText}}
+{{submitSubText}}
{{submitT}}
diff --git a/kystudio/src/components/studio/StudioModel/AddCCModal/locales.js
b/kystudio/src/components/studio/StudioModel/AddCCModal/locales.js
index e437f7077b..87988ac338 100644
--- a/kystudio/src/components/studio/StudioModel/AddCCModal/locales.js
+++ b/kystudio/src/components/studio/StudioModel/AddCCModal/locales.js
@@ -1,5 +1,5 @@
export default {
en: {
-editCCTip: 'Please be aware that the modified expression would be
effective until all the related indexes have been built.'
+editCCTip: 'Modify with caution, it may cause query performance
degradation! If the edited computed column is used as a dimension or metric,
the system will delete the relevant index and generate new index. The query
performance will not be restored until the build index job is done.'
}
}
diff --git
a/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/index.vue
b/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/index.vue
index 13839e9a1c..d8ded0cc7b 100644
---
a/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/index.vue
+++
b/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/index.vue
@@ -785,13 +785,13 @@ export default class ModelPartitionModal extends Vue {
try {
const res = await this.callGlobalDetailDialog({
msg: this.$t('editCCBuildTip'),
-title: this.$t('kylinLang.common.tip'),
+title: this.$t('editCCBuildTipTitle'),
dialogType: 'warning',
showDetailBtn: false,
isSubSubmit: true,
+isHideSubmit: true,
wid: '600px',
submitSubText: this.$t('kylinLang.common.save'),
-submitText: this.$t('saveAndLoad'),
needConcelReject: true
})
this.handleClose(true, res.isOnlySave)
diff --git
a/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/locales.js
b/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/locales.js
index 441fb30e54..1bd439fbde 100644
---
a/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/locales.js
+++
b/kystudio/src/components/studio/StudioModel/ModelList/ModelSaveConfig/locales.js
@@ -38,7 +38,8 @@ export default {
incrementalTips: 'It will load data incrementally based on the selected
partition column, which is more resource-efficient.',
fullLoadTips: 'The system will load all data',
changeBuildTypeTips: 'With partition setting changed, all segments and
data would be deleted. The model couldn\'t serve queries. Meanwhile, the
related ongoing jobs for building index would be discarded.',
-editCCBuildTip: 'The modified expression of computed column would be
effective until all the related indexes have been built. Do you want to save
and build index now? ',
+editCCBuildTipTitle: 'Will delete and generate new indexes, continue to
save model?',
+editCCBuildTip: 'Query performance will degrade until the new index are
built! Modification of a computed column as a dimension or metric in the
current model will cause the system to delete the relevant index and generate
new index. The query performance will not be restored until the build index job
is done.',
saveAndBuild: 'Save and Build',
purgeSegmentDataTips: 'Model definition has changed. Once saving the
model, all data in the segments will be deleted. As a result, this model CAN\'T
be used to serve queries. We
(kylin) 12/18: KYLIN-5787 Use t-digest as spark percentile_approx function
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit dd16079c42e4f6eb446945ec42980891934a20c6
Author: Zhiting Guo <35057824+fre...@users.noreply.github.com>
AuthorDate: Fri Aug 11 13:15:12 2023 +0800
KYLIN-5787 Use t-digest as spark percentile_approx function
---
.../org/apache/kylin/common/KylinConfigBase.java | 6
.../measure/percentile/PercentileCounter.java | 1 +
.../java/org/apache/kylin/util/ExecAndComp.java| 42 +-
.../kylin/query/engine/AsyncQueryApplication.java | 5 +++
.../query/engine/AsyncQueryApplicationTest.java| 3 ++
.../kylin/query/runtime/plan/AggregatePlan.scala | 11 --
.../scala/org/apache/spark/sql/KapFunctions.scala | 8 +++--
.../scala/org/apache/spark/sql/SparderEnv.scala| 3 ++
.../org/apache/spark/sql/udf/UdfManager.scala | 9 ++---
.../org/apache/spark/sql/udaf/Percentile.scala | 5 +++
10 files changed, 67 insertions(+), 26 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ff7bd4c7c3..5bfe8c262a 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1914,6 +1914,12 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(this.getOptional("kylin.query.replace-dynamic-params-enabled",
FALSE));
}
+
+public String getPercentileApproxAlgorithm() {
+// Valid values: t-digest
+return this.getOptional("kylin.query.percentile-approx-algorithm", "");
+}
+
public boolean isCollectUnionInOrder() {
return
Boolean.parseBoolean(this.getOptional("kylin.query.collect-union-in-order",
TRUE));
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
b/src/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
index 823b47ba38..11851d48c6 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
@@ -25,6 +25,7 @@ import com.tdunning.math.stats.AVLTreeDigest;
import com.tdunning.math.stats.TDigest;
public class PercentileCounter implements Serializable {
+public static final int DEFAULT_PERCENTILE_ACCURACY = 100;
private static final double INVALID_QUANTILE_RATIO = -1;
double compression;
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
b/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
index 34126ab7f4..d089e9e34a 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
@@ -135,27 +135,37 @@ public class ExecAndComp {
@SneakyThrows
public static QueryResult queryWithSpark(String prj, String originSql,
String joinType, String sqlPath) {
+return queryWithSpark(prj, originSql, joinType, sqlPath, true);
+}
+
+@SneakyThrows
+public static QueryResult queryWithSpark(String prj, String originSql,
String joinType, String sqlPath,
+boolean withCache) {
int index = sqlPath.lastIndexOf('/');
String resultFilePath = "";
String schemaFilePath = "";
-if (index > 0) {
-resultFilePath = sqlPath.substring(0, index) + "/result-" +
joinType + sqlPath.substring(index) + ".json";
-schemaFilePath = sqlPath.substring(0, index) + "/result-" +
joinType + sqlPath.substring(index) + ".schema";
-}
+if (withCache) {
+if (index > 0) {
+resultFilePath = sqlPath.substring(0, index) + "/result-" +
joinType + sqlPath.substring(index)
++ ".json";
+schemaFilePath = sqlPath.substring(0, index) + "/result-" +
joinType + sqlPath.substring(index)
++ ".schema";
+}
-// query with cache
-try {
-if (index > 0 && Files.exists(Paths.get(resultFilePath)) &&
Files.exists(Paths.get(schemaFilePath))) {
-StructType schema = StructType.fromDDL(new
String(Files.readAllBytes(Paths.get(schemaFilePath;
-List structs = Arrays.stream(schema.fields())
-
.map(SparderTypeUtil::convertSparkFieldToJavaField).collect(Collectors.toList());
-Dataset ds =
SparderEnv.getSparkSession().read().schema(schema).json(resultFilePath);
(kylin) 11/18: KYLIN-5785 Support implicit query of split function
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d7d80985987a344c0a49909cd8d8584829015082
Author: Pengfei Zhan
AuthorDate: Wed Aug 9 19:33:45 2023 +0800
KYLIN-5785 Support implicit query of split function
1. support split('aaa-bbb-ccc', '-')[0] implicit query
2. support simplify not(true/false) to false/true
---
pom.xml | 2 +-
.../org/apache/kylin/query/engine/QueryRoutingEngine.java| 12
.../main/java/org/apache/kylin/query/udf/SparkStringUDF.java | 6 --
.../org/apache/kylin/query/runtime/ExpressionConverter.scala | 2 ++
4 files changed, 7 insertions(+), 15 deletions(-)
diff --git a/pom.xml b/pom.xml
index 2138ca85f8..71763ea249 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
0.3.0
-1.116.0-kylin-4.x-r034
+1.116.0-kylin-4.x-r035
4.x_1.10-r01
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index 081edc0d0a..0e3f86b399 100644
---
a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++
b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -40,7 +40,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryTrace;
import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.common.exception.CalciteNotSupportException;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.NewQueryRefuseException;
import org.apache.kylin.common.exception.TargetSegmentNotFoundException;
@@ -141,8 +140,6 @@ public class QueryRoutingEngine {
return handleTransactionException(queryParams, e);
} catch (SQLException e) {
return handleSqlException(queryParams, e);
-} catch (AssertionError e) {
-return handleAssertionError(queryParams, e);
} finally {
QueryResultMasks.remove();
}
@@ -183,15 +180,6 @@ public class QueryRoutingEngine {
throw e;
}
-private QueryResult handleAssertionError(QueryParams queryParams,
AssertionError e) throws SQLException {
-// for example: split('abc', 'b') will jump into this AssertionError
-if (e.getMessage().equals("OTHER")) {
-SQLException ex = new SQLException(e.getMessage(), new
CalciteNotSupportException());
-return pushDownQuery(ex, queryParams);
-}
-throw e;
-}
-
public boolean checkIfRetryQuery(Throwable cause) {
if (TargetSegmentNotFoundException.causedBySegmentNotFound(cause)
&& QueryContext.current().getMetrics().getRetryTimes() == 0) {
diff --git
a/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
b/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
index 5ce28e9851..867ffbefe1 100644
--- a/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
+++ b/src/query/src/main/java/org/apache/kylin/query/udf/SparkStringUDF.java
@@ -18,6 +18,8 @@
package org.apache.kylin.query.udf;
+import java.util.List;
+
import org.apache.calcite.linq4j.function.Parameter;
import org.apache.calcite.sql.type.NotConstant;
import org.apache.kylin.common.exception.CalciteNotSupportException;
@@ -90,12 +92,12 @@ public class SparkStringUDF implements NotConstant {
throw new CalciteNotSupportException();
}
-public String[] SPLIT(@Parameter(name = "str") Object str, @Parameter(name
= "regex") Object regex)
+public List SPLIT(@Parameter(name = "str") Object str,
@Parameter(name = "regex") Object regex)
throws CalciteNotSupportException {
throw new CalciteNotSupportException();
}
-public String[] SPLIT(@Parameter(name = "str") Object str, @Parameter(name
= "regex") Object regex,
+public List SPLIT(@Parameter(name = "str") Object str,
@Parameter(name = "regex") Object regex,
@Parameter(name = "limit") Object limit) throws
CalciteNotSupportException {
throw new CalciteNotSupportException();
}
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
index d08a37d450..04afba3c94 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/ExpressionConverter.scala
@@ -42
(kylin) 16/18: KYLIN-5790 Security of kafka-clients
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2695c5da83d9ccfb2a3177b9947e6eda4e6c1715
Author: sibingzhang <74443791+sibingzh...@users.noreply.github.com>
AuthorDate: Fri Aug 25 15:18:08 2023 +0800
KYLIN-5790 Security of kafka-clients
Co-authored-by: sibing.zhang
---
.../java/org/apache/kylin/rest/KafkaCondition.java | 32 ++
.../apache/kylin/rest/service/KafkaService.java| 8 --
.../apache/kylin/rest/service/TableService.java| 2 +-
.../kylin/rest/controller/KafkaController.java | 2 +-
4 files changed, 39 insertions(+), 5 deletions(-)
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/KafkaCondition.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/KafkaCondition.java
new file mode 100644
index 00..c4388b8cd4
--- /dev/null
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/KafkaCondition.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest;
+
+import org.apache.kylin.common.KylinConfig;
+import org.springframework.context.annotation.Condition;
+import org.springframework.context.annotation.ConditionContext;
+import org.springframework.core.type.AnnotatedTypeMetadata;
+
+public class KafkaCondition implements Condition {
+@Override
+public boolean matches(ConditionContext context, AnnotatedTypeMetadata
metadata) {
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+return config.streamingEnabled();
+}
+}
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/KafkaService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/KafkaService.java
index 2fec7df8cc..496f2f90ad 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/KafkaService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/KafkaService.java
@@ -40,26 +40,28 @@ import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.loader.ParserClassLoaderState;
import org.apache.kylin.metadata.jar.JarInfoManager;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.streaming.DataParserInfo;
import org.apache.kylin.metadata.streaming.DataParserManager;
import org.apache.kylin.metadata.streaming.KafkaConfig;
+import org.apache.kylin.rest.KafkaCondition;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.sample.KafkaSourceHandler;
import org.apache.kylin.sample.StreamingSourceHandler;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Conditional;
import org.springframework.stereotype.Component;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component("kafkaService")
+@Conditional(KafkaCondition.class)
public class KafkaService extends BasicService {
@Autowired
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
index b6498f1a6f..b2ae72706b 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -217,7 +217,7 @@ public class TableService extends BasicService {
@Autowired
private AccessService accessService;
-@Autowired
+@Autowired(required = false)
@Qualifier("kafkaService")
private KafkaService kafkaService;
diff --git
a/src/metadata-server/
(kylin) 06/18: KYLIN-5781 Adjust the query result order of sql with union
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit cb2a11d503f4d857b236ea62152d606d5c8d5fe7
Author: Zhiting Guo <35057824+fre...@users.noreply.github.com>
AuthorDate: Fri Aug 11 11:29:19 2023 +0800
KYLIN-5781 Adjust the query result order of sql with union
Co-authored-by: Zhiting Guo
---
.../main/java/org/apache/kylin/common/KylinConfigBase.java| 4
.../org/apache/kylin/query/runtime/CalciteToSparkPlaner.scala | 11 +++
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f48c808cdc..eddb15bffa 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1914,6 +1914,10 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(this.getOptional("kylin.query.replace-dynamic-params-enabled",
FALSE));
}
+public boolean isCollectUnionInOrder() {
+return
Boolean.parseBoolean(this.getOptional("kylin.query.collect-union-in-order",
TRUE));
+}
+
//
// Cache
//
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/CalciteToSparkPlaner.scala
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/CalciteToSparkPlaner.scala
index bab69339b4..76e9940270 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/CalciteToSparkPlaner.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/CalciteToSparkPlaner.scala
@@ -19,13 +19,13 @@
package org.apache.kylin.query.runtime
import java.util
+import java.util.Collections
-import org.apache.kylin.guava30.shaded.common.collect.Lists
-import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.calcite.DataContext
import org.apache.calcite.rel.{RelNode, RelVisitor}
import org.apache.kylin.common.KylinConfig
import org.apache.kylin.engine.spark.utils.LogEx
+import org.apache.kylin.guava30.shaded.common.collect.Lists
import org.apache.kylin.query.relnode.{KapAggregateRel, KapFilterRel,
KapJoinRel, KapLimitRel, KapMinusRel, KapModelViewRel, KapNonEquiJoinRel,
KapProjectRel, KapRel, KapSortRel, KapTableScan, KapUnionRel, KapValuesRel,
KapWindowRel}
import org.apache.kylin.query.runtime.plan.{AggregatePlan, FilterPlan,
LimitPlan, ProjectPlan, SortPlan, TableScanPlan, ValuesPlan, WindowPlan}
import org.apache.kylin.query.util.KapRelUtil
@@ -84,8 +84,11 @@ class CalciteToSparkPlaner(dataContext: DataContext) extends
RelVisitor with Log
case rel: KapNonEquiJoinRel => convertNonEquiJoinRel(rel)
case rel: KapUnionRel =>
val size = setOpStack.pop()
-val java = Range(0, stack.size() - size).map(a => stack.pop()).asJava
-logTime("union") { plan.UnionPlan.union(Lists.newArrayList(java), rel,
dataContext) }
+var unionBlocks = Range(0, stack.size() - size).map(a => stack.pop())
+if (KylinConfig.getInstanceFromEnv.isCollectUnionInOrder) {
+ unionBlocks = unionBlocks.reverse
+}
+logTime("union") { plan.UnionPlan.union(unionBlocks.asJava, rel,
dataContext) }
case rel: KapMinusRel =>
val size = setOpStack.pop()
logTime("minus") { plan.MinusPlan.minus(Range(0, stack.size() -
size).map(a => stack.pop()).reverse, rel, dataContext) }
(kylin) 03/18: KYLIN-5777 Change spark.yarn.submit.file.replication from 1 to 3
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a67e854a1d598eead4d39c10ae4295c70a94e74d
Author: Yaguang Jia
AuthorDate: Tue Aug 8 17:18:25 2023 +0800
KYLIN-5777 Change spark.yarn.submit.file.replication from 1 to 3
---
src/common-booter/src/main/resources/config/init.properties | 2 +-
src/core-common/src/main/resources/kylin-defaults0.properties | 2 +-
src/data-loading-booter/src/main/resources/config/init.properties | 2 +-
src/query-booter/src/main/resources/config/init.properties| 2 +-
src/server/src/main/resources/config/init.properties | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/common-booter/src/main/resources/config/init.properties
b/src/common-booter/src/main/resources/config/init.properties
index 14ad876715..f0ce35de8e 100644
--- a/src/common-booter/src/main/resources/config/init.properties
+++ b/src/common-booter/src/main/resources/config/init.properties
@@ -56,7 +56,7 @@ kylin.server.cors.allow-all=false
# JOB EXECUTION
### Spark conf overwrite for build engine
-kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
+kylin.engine.spark-conf.spark.yarn.submit.file.replication=3
kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.driver.host=${HOST_IP}
kylin.engine.spark-conf.spark.driver.memoryOverhead=256
diff --git a/src/core-common/src/main/resources/kylin-defaults0.properties
b/src/core-common/src/main/resources/kylin-defaults0.properties
index 61412133ec..b10ef59365 100644
--- a/src/core-common/src/main/resources/kylin-defaults0.properties
+++ b/src/core-common/src/main/resources/kylin-defaults0.properties
@@ -61,7 +61,7 @@ kylin.server.cors.allow-all=false
# JOB EXECUTION
### Spark conf overwrite for build engine
-kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
+kylin.engine.spark-conf.spark.yarn.submit.file.replication=3
kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.driver.memoryOverhead=256
#kylin.engine.spark-conf.spark.driver.memory=512m
diff --git a/src/data-loading-booter/src/main/resources/config/init.properties
b/src/data-loading-booter/src/main/resources/config/init.properties
index 14ad876715..f0ce35de8e 100644
--- a/src/data-loading-booter/src/main/resources/config/init.properties
+++ b/src/data-loading-booter/src/main/resources/config/init.properties
@@ -56,7 +56,7 @@ kylin.server.cors.allow-all=false
# JOB EXECUTION
### Spark conf overwrite for build engine
-kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
+kylin.engine.spark-conf.spark.yarn.submit.file.replication=3
kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.driver.host=${HOST_IP}
kylin.engine.spark-conf.spark.driver.memoryOverhead=256
diff --git a/src/query-booter/src/main/resources/config/init.properties
b/src/query-booter/src/main/resources/config/init.properties
index 14ad876715..f0ce35de8e 100644
--- a/src/query-booter/src/main/resources/config/init.properties
+++ b/src/query-booter/src/main/resources/config/init.properties
@@ -56,7 +56,7 @@ kylin.server.cors.allow-all=false
# JOB EXECUTION
### Spark conf overwrite for build engine
-kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
+kylin.engine.spark-conf.spark.yarn.submit.file.replication=3
kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.driver.host=${HOST_IP}
kylin.engine.spark-conf.spark.driver.memoryOverhead=256
diff --git a/src/server/src/main/resources/config/init.properties
b/src/server/src/main/resources/config/init.properties
index e705611a5a..ff253daebb 100644
--- a/src/server/src/main/resources/config/init.properties
+++ b/src/server/src/main/resources/config/init.properties
@@ -56,7 +56,7 @@ kylin.server.cors.allow-all=false
# JOB EXECUTION
### Spark conf overwrite for build engine
-kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
+kylin.engine.spark-conf.spark.yarn.submit.file.replication=3
kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.driver.host=${HOST_IP}
kylin.engine.spark-conf.spark.driver.memoryOverhead=256
(kylin) 01/18: KYLIN-5775 upgrade testcontainers to 1.16.2
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 8ab87c93b9d6e2208bffa71845270841313faf1e Author: liang.huang <83992752+lhuang09287...@users.noreply.github.com> AuthorDate: Thu Jul 27 16:53:19 2023 +0800 KYLIN-5775 upgrade testcontainers to 1.16.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f969b9a14..645b569fcc 100644 --- a/pom.xml +++ b/pom.xml @@ -232,7 +232,7 @@ 3.1.1 -1.15.3 +1.16.2 2.4.21 1.7.26
(kylin) 02/18: KYLIN-5776 Support parameter type is string for timestampadd function
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 5d781d1917ec261a95971086f8d8540afc8434fb Author: liang.huang <83992752+lhuang09287...@users.noreply.github.com> AuthorDate: Thu Aug 3 15:03:42 2023 +0800 KYLIN-5776 Support parameter type is string for timestampadd function --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 645b569fcc..2138ca85f8 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ 0.3.0 -1.116.0-kylin-4.x-r033 +1.116.0-kylin-4.x-r034 4.x_1.10-r01
(kylin) 13/18: KYLIN-5774 Optimize Calcite plan to convert spark logical plan
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9db3502f80265753b38b4ecbb0ee57f7871b6466
Author: Mingming Ge <7mmi...@gmail.com>
AuthorDate: Mon Aug 7 19:02:02 2023 +0800
KYLIN-5774 Optimize Calcite plan to convert spark logical plan
---
pom.xml| 4 +
.../org/apache/kylin/common/KylinConfigBase.java | 4 -
.../src/main/resources/kylin-defaults0.properties | 1 -
.../kylin/newten/EnhancedAggPushDownTest.java | 2 +-
.../kylin/newten/SpecialColumnNamesTest.java | 104 ++
.../apache/kylin/query/engine/QueryExecTest.java | 59 +---
.../_global/project/special_column_names.json | 35 +
.../c41390c5-b35d-4db3-b167-029874b85a2c.json | 13 ++
.../c41390c5-b35d-4db3-b167-029874b85a2c.json | 63
.../c41390c5-b35d-4db3-b167-029874b85a2c.json | 158 +
.../special_column_names/table/SSB.CUSTOMER.json | 68 +
.../table/SSB.P_LINEORDER.json | 118 +++
.../kylin/query/runtime/CalciteToSparkPlaner.scala | 121 +---
.../kylin/query/runtime/SparderRexVisitor.scala| 37 ++---
.../apache/kylin/query/runtime/SparkEngine.java| 8 +-
.../kylin/query/runtime/plan/AggregatePlan.scala | 87 +++-
.../kylin/query/runtime/plan/FilterPlan.scala | 25 ++--
.../apache/kylin/query/runtime/plan/JoinPlan.scala | 58
.../kylin/query/runtime/plan/LimitPlan.scala | 25 ++--
.../kylin/query/runtime/plan/MinusPlan.scala | 14 +-
.../kylin/query/runtime/plan/ProjectPlan.scala | 19 +--
.../kylin/query/runtime/plan/ResultPlan.scala | 3 +-
.../apache/kylin/query/runtime/plan/SortPlan.scala | 52 ---
.../kylin/query/runtime/plan/TableScanPlan.scala | 118 +++
.../kylin/query/runtime/plan/UnionPlan.scala | 30 ++--
.../kylin/query/runtime/plan/ValuesPlan.scala | 15 +-
.../kylin/query/runtime/plan/WindowPlan.scala | 29 ++--
.../apache/kylin/query/util/RuntimeHelper.scala| 31 ++--
.../kylin/query/util/SparderDerivedUtil.scala | 52 ---
.../apache/spark/sql/KylinDataFrameManager.scala | 19 ++-
.../org/apache/spark/sql/SparkOperation.scala | 64 ++---
.../sql/execution/utils/SchemaProcessor.scala | 36 ++---
.../spark/sql/manager/SparderLookupManager.scala | 69 -
.../kylin/query/sql/KylinDataFrameManagerTest.java | 32 +++--
.../query/runtime/plan/SegmentEmptyTest.scala | 38 ++---
.../org/apache/spark/sql/SparkInternalAgent.scala | 0
.../sql/datasource/storage/StorageStore.scala | 73 +-
.../scala/org/apache/kylin/common/JobSupport.scala | 7 +-
38 files changed, 1104 insertions(+), 587 deletions(-)
diff --git a/pom.xml b/pom.xml
index 71763ea249..dac240b005 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1509,6 +1509,10 @@
org.codehaus.jackson
jackson-mapper-asl
+
+org.apache.calcite
+calcite
+
provided
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5bfe8c262a..c1dd5379e3 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1906,10 +1906,6 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(this.getOptional("kylin.query.schema-cache-enabled",
FALSE));
}
-public boolean isDataFrameCacheEnabled() {
-return
Boolean.parseBoolean(this.getOptional("kylin.query.dataframe-cache-enabled",
TRUE));
-}
-
public boolean enableReplaceDynamicParams() {
return
Boolean.parseBoolean(this.getOptional("kylin.query.replace-dynamic-params-enabled",
FALSE));
}
diff --git a/src/core-common/src/main/resources/kylin-defaults0.properties
b/src/core-common/src/main/resources/kylin-defaults0.properties
index b10ef59365..56cd346578 100644
--- a/src/core-common/src/main/resources/kylin-defaults0.properties
+++ b/src/core-common/src/main/resources/kylin-defaults0.properties
@@ -232,7 +232,6 @@ kylin.query.join-match-optimization-enabled=false
kylin.query.convert-count-distinct-expression-enabled=false
kylin.query.memory-limit-during-collect-mb=5400
kylin.query.cartesian-partition-num-threshold-factor=100
-kylin.query.dataframe-cache-enabled=true
# spark context canary to monitor spark
kylin.canary.sqlcontext-enabled=false
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/newten/EnhancedAggPu
(kylin) 18/18: KYLIN-5778 Upgrade spark to 3.2.0-kylin-4.6.14.0
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 4ed37293122dc700483fceafdd085b8058a436bf Author: Yaguang Jia AuthorDate: Thu Aug 10 10:09:33 2023 +0800 KYLIN-5778 Upgrade spark to 3.2.0-kylin-4.6.14.0 Add parquet footer read cache --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dac240b005..ccbdfbae22 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 2.0.2 -3.2.0-kylin-4.6.13.0 +3.2.0-kylin-4.6.14.0 0.9.2-kylin-r4
(kylin) 09/18: Add some code to compatible with the enterprise version
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 40a10818a045b1ec5e909df12f7365f7679eae9a
Author: lixiang <447399...@qq.com>
AuthorDate: Fri Aug 11 11:48:41 2023 +0800
Add some code to compatible with the enterprise version
-
Co-authored-by: Zhiting Guo
---
.../common/exception/code/ErrorCodeServer.java | 3 +
.../resources/kylin_error_msg_conf_cn.properties | 3 +
.../resources/kylin_error_msg_conf_en.properties | 3 +
.../kylin_error_suggestion_conf_cn.properties | 3 +
.../kylin_error_suggestion_conf_en.properties | 3 +
.../main/resources/kylin_errorcode_conf.properties | 3 +
.../apache/kylin/metadata/model/NDataModel.java| 9 ++
.../recommendation/entity/DimensionRecItemV2.java | 17 +++-
.../kylin/rest/controller/SegmentController.java | 6 +-
.../rest/controller/SegmentControllerTest.java | 43 -
.../kylin/rest/controller/NModelController.java| 11 ++-
.../rest/controller/NModelControllerTest.java | 107 -
.../org/apache/kylin/rest/util/ModelUtils.java | 36 ++-
13 files changed, 234 insertions(+), 13 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index 26f558be38..f2562e3a9f 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -37,6 +37,8 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_SUM_LC_INVALID_DATA_TYPE("KE-010002303"),
MODEL_SUM_LC_INVALID_TIMESTAMP_TYPE("KE-010002304"),
MODEL_NAME_TOO_LONG("KE-010002305"),
+MODEL_SECOND_STORAGE_PARTITION_INVALID("KE-010002306"),
+PARTITION_SECOND_STORAGE_PARTITION_INVALID("KE-010002307"),
// 100252XX Cube
CUBE_NOT_EXIST("KE-010025201"),
@@ -64,6 +66,7 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
SEGMENT_INDEX_CONFLICT_PARAMETER("KE-01000"),
SEGMENT_INDEX_STATUS_INVALID("KE-01001"),
SEGMENT_SINGLE_JOB_THRESHOLD("KE-01002"),
+SEGMENT_SECOND_STORAGE_PARTITION_INVALID("KE-01003"),
// 100072XX table
TABLE_RELOAD_MODEL_RETRY("KE-010007204"),
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index ffb3f55714..010c2b10c3 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -37,6 +37,8 @@ KE-010002302=模型中的列名 %s 与度量名 %s 重复，无法导出 TDS。
KE-010002303=SUM_LC度量的返回类型 '%s' 不合法。返回类型必须是这其中的一个：%s。
KE-010002304=SUM_LC度量的时间类型 '%s' 不合法。
KE-010002305=模型名称最长 127 字符，请修改后重试。
+KE-010002306=无法保存模型。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
+KE-010002307=无法保存分区设置。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
## 100252XX Cube
KE-010025201=无法找到相关 Cube。
@@ -64,6 +66,7 @@ KE-010022219=当前 Segments 所包含的分区不一致，请先构建分区并
KE-01000=index_ids或index_status不能同时设置，请修改后重试。
KE-01001=索引状态错误，请输入正确的索引状态（NO_BUILD, ONLINE, BUILDING）后重试。
KE-01002=单个任务中最多包含100个Segments，请修改后重试。
+KE-01003=无法构建索引。当增量加载的模型开启分层存储时，必须将时间分区列加入维度。
## 100072XX table
KE-010007204=源表 %1$s 中列 %2$s 的数据类型发生变更。请从模型 %3$s 中删除该列，或修改该列的数据类型。
diff --git
a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index b2bcbac2bd..75f05b384b 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -37,6 +37,8 @@ KE-010002302=There are duplicated names among model column %s
and measure name %
KE-010002303=SUM_LC Measure's return type '%s' is illegal. It must be one of
%s.
KE-010002304=SUM_LC Measure's time column type '%s' is illegal.
KE-010002305=The maximum length of the model name is 127 characters, please
modify and try again.
+KE-010002306=Can't save the model. When the model uses incremental load method
and the tiered storage is ON, the time partition column must be added as a
dimension.
+KE-010002307=Can't save the model partition. When the model uses incremental
load method and the tiered storage is ON, the time partition column must be
added as a dimension.
## 100252XX Cube
KE-010025201=Can't find the cube.
@@ -64,6 +66,7 @@ KE-010022219=The partitions included in the selected segments
are not consistent
KE-01000=Index_ids or index_status can not be set at the same time, please
modify and try again.
KE-01001=The index status is
(kylin) 10/18: KYLIN-5784 Optimization for query history filters
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1367e9ccff65ece51e57c654380d526c472ed2fb
Author: lixiang <447399...@qq.com>
AuthorDate: Fri Aug 11 11:48:59 2023 +0800
KYLIN-5784 Optimization for query history filters
---
.../kylin/rest/service/QueryHistoryService.java| 27 --
.../rest/service/QueryHistoryServiceTest.java | 6 ++---
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
index 0200c2f080..0c947c8de1 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
@@ -18,7 +18,11 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.common.QueryContext.PUSHDOWN_OBJECT_STORAGE;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_NOT_EXIST;
+import static
org.apache.kylin.metadata.query.QueryHistory.EngineType.CONSTANTS;
+import static org.apache.kylin.metadata.query.QueryHistory.EngineType.HIVE;
+import static org.apache.kylin.metadata.query.QueryHistory.EngineType.RDBMS;
import static
org.apache.kylin.metadata.query.RDBMSQueryHistoryDAO.fillZeroForQueryStatistics;
import java.lang.reflect.Field;
@@ -35,6 +39,7 @@ import java.util.Map;
import java.util.TimeZone;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -269,26 +274,24 @@ public class QueryHistoryService extends BasicService
implements AsyncTaskQueryH
}
public QueryHistoryFiltersResponse
getQueryHistoryModels(QueryHistoryRequest request, int size) {
-QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao();
-List queryStatistics =
queryHistoryDAO.getQueryHistoriesModelIds(request);
val dataFlowManager =
NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(),
request.getProject());
-Stream engineStream =
queryStatistics.stream().map(QueryStatistics::getEngineType);
List models = dataFlowManager.listAllDataflows();
-Stream modelStream = models.stream()
+List modelList = models.stream()
.sorted(Comparator.comparing(NDataflow::getQueryHitCount,
Comparator.reverseOrder()))
-.map(NDataflow::getModel).map(NDataModel::getAlias);
-List engineList = filterByName(engineStream,
request.getFilterModelName());
-List modelList = filterByName(modelStream,
request.getFilterModelName());
+
.map(NDataflow::getModel).map(NDataModel::getAlias).filter(filterByName(request.getFilterModelName()))
+.collect(Collectors.toList());
+
+List engineList = Stream.of(HIVE.name(), RDBMS.name(),
CONSTANTS.name(), PUSHDOWN_OBJECT_STORAGE)
+
.filter(filterByName(request.getFilterModelName())).collect(Collectors.toList());
+
Integer count = engineList.size() + modelList.size();
return new QueryHistoryFiltersResponse(count, models.size(),
engineList,
modelList.stream().limit(size).collect(Collectors.toList()));
}
-private List filterByName(Stream stream, String name) {
-return stream
-.filter(alias -> !StringUtils.isEmpty(alias) &&
(StringUtils.isEmpty(name)
-||
alias.toLowerCase(Locale.ROOT).contains(name.toLowerCase(Locale.ROOT
-.collect(Collectors.toList());
+private Predicate filterByName(String name) {
+return alias -> !StringUtils.isEmpty(alias) &&
(StringUtils.isEmpty(name)
+||
alias.toLowerCase(Locale.ROOT).contains(name.toLowerCase(Locale.ROOT)));
}
private boolean haveSpaces(String text) {
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
index e93b35dcce..ddc67f464c 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
@@ -721,11 +721,11 @@ public class QueryHistoryServiceTest extends
NLocalFileMetadataTestCase {
Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao();
QueryHistoryFiltersResponse response =
queryHistoryService.getQueryHistoryModels(request, 10);
-assertEquals(10, (int) response.getSearchCount());
+assertEquals(12, (
(kylin) 05/18: KYLIN-5780 Dimension capacity of aggGroup does not work
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8929bb820c881e7172eddbc5f6606d04f62a0dd3
Author: Hang Jia <754332...@qq.com>
AuthorDate: Fri Aug 11 11:24:39 2023 +0800
KYLIN-5780 Dimension capacity of aggGroup does not work
---
.../kylin/rest/service/FusionIndexService.java | 10 ++
.../kylin/rest/service/IndexPlanService.java | 2 ++
.../kylin/rest/service/FusionIndexServiceTest.java | 23 ++
3 files changed, 35 insertions(+)
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
index a7f2c67d68..ee11aa8d83 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/FusionIndexService.java
@@ -117,6 +117,7 @@ public class FusionIndexService extends BasicService {
if (modelRule != null) {
newRuleBasedIndex.getAggregationGroups().addAll(modelRule.getAggregationGroups());
+newRuleBasedIndex.setGlobalDimCap(modelRule.getGlobalDimCap());
}
if (model.fusionModelStreamingPart()) {
@@ -126,11 +127,20 @@ public class FusionIndexService extends BasicService {
if (batchRule != null) {
newRuleBasedIndex.getAggregationGroups().addAll(batchRule.getAggregationGroups().stream()
.filter(agg -> agg.getIndexRange() ==
IndexEntity.Range.BATCH).collect(Collectors.toList()));
+
newRuleBasedIndex.setGlobalDimCap(lastModifiedTimeGlobalDimCap(batchRule,
modelRule));
}
}
return newRuleBasedIndex;
}
+private int lastModifiedTimeGlobalDimCap(RuleBasedIndex batchRule,
RuleBasedIndex modelRule) {
+if (modelRule == null || (modelRule.getLastModifiedTime() <
batchRule.getLastModifiedTime())) {
+return batchRule.getGlobalDimCap();
+} else {
+return modelRule.getGlobalDimCap();
+}
+}
+
@Transaction(project = 0)
public BuildIndexResponse createTableIndex(String project,
CreateTableIndexRequest request) {
NDataModel model = getManager(NDataModelManager.class,
project).getDataModelDesc(request.getModelId());
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
index 9a50ab5b75..d5acd58ca5 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/IndexPlanService.java
@@ -798,6 +798,8 @@ public class IndexPlanService extends BasicService
implements TableIndexPlanSupp
newRuleBasedIndex.setIndexUpdateEnabled(index.getIndexUpdateEnabled());
newRuleBasedIndex.setAggregationGroups(new LinkedList<>());
+newRuleBasedIndex.setGlobalDimCap(index.getGlobalDimCap());
+newRuleBasedIndex.setLastModifiedTime(index.getLastModifiedTime());
for (NAggregationGroup aggGrp : index.getAggregationGroups()) {
val aggGrpCopy = new NAggregationGroup();
diff --git
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
index 52435701a5..9f5efe8296 100644
---
a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
+++
b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/FusionIndexServiceTest.java
@@ -202,6 +202,29 @@ public class FusionIndexServiceTest extends SourceTestCase
{
Assert.assertEquals(false, rule1.getIndexUpdateEnabled());
}
+@Test
+public void testGetRuleWithGlobalDimCap() {
+val modelId = "b05034a8-c037-416b-aa26-9e6b4a41ee40";
+val fusionModelId = "334671fd-e383-4fc9-b5c2-94fce832f77a";
+
+val indePlanManager = NIndexPlanManager.getInstance(getTestConfig(),
"streaming_test");
+indePlanManager.updateIndexPlan(modelId, copy ->
copy.getRuleBasedIndex().setGlobalDimCap(1));
+indePlanManager.updateIndexPlan(fusionModelId, copy ->
copy.getRuleBasedIndex().setGlobalDimCap(2));
+
+val rule = fusionIndexService.getRule("streaming_test", modelId);
+Assert.assertEquals(1, rule.getGlobalDimCap());
+
+indePlanManager.updateIndexPlan(fusionModelId,
+copy ->
copy.getRuleBasedIndex().setLastModifiedTime(System.currentTimeMillis() -
1000));
+val rule2 = fusionIndexService.getRule("streamin
(kylin) 14/18: KYLIN-5788 Enhance global dict on flat table encoding stage logging & retry
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 775efe62b525a7f9188c2c478c44e57ed4b8dc48
Author: Yinghao Lin <39019287+yhca...@users.noreply.github.com>
AuthorDate: Sun Aug 13 10:56:53 2023 +0800
KYLIN-5788 Enhance global dict on flat table encoding stage logging & retry
---
.../org/apache/kylin/common/KylinConfigBase.java | 1 +
.../sql/catalyst/expressions/KapExpresssions.scala | 2 +-
.../org/apache/spark/sql/udf/DictEncodeImpl.scala | 6 +-
.../org/apache/spark/dict/NBucketDictionary.java | 7 +-
.../apache/spark/dict/NGlobalDictHDFSStore.java| 92 +-
.../org/apache/spark/dict/NGlobalDictMetaInfo.java | 6 +-
.../org/apache/spark/dict/NGlobalDictStore.java| 2 +-
.../org/apache/spark/dict/NGlobalDictionaryV2.java | 6 +-
8 files changed, 111 insertions(+), 11 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c1dd5379e3..9bce3f735a 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -4035,4 +4035,5 @@ public abstract class KylinConfigBase implements
Serializable {
public boolean isRoundDecimalZero() {
return
Boolean.parseBoolean(getOptional("kylin.query.round-decimal-zero", FALSE));
}
+
}
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
index 2233e201ba..74beffaae3 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
@@ -402,7 +402,7 @@ case class DictEncode(left: Expression, mid: Expression,
right: Expression) exte
| try {
| int bucketId = idx % $bucketSizeTerm;
| $globalDictTerm = new
org.apache.spark.dict.NGlobalDictionaryV2("$dictParamsTerm");
- | $bucketDictTerm =
$globalDictTerm.loadBucketDictionary(bucketId);
+ | $bucketDictTerm =
$globalDictTerm.loadBucketDictionary(bucketId, true);
| } catch (Exception e) {
| throw new RuntimeException(e);
| }
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
index 8cc6e1a825..a33f29d736 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/DictEncodeImpl.scala
@@ -17,12 +17,12 @@
*/
package org.apache.spark.sql.udf
-import java.util
-
import org.apache.spark.TaskContext
import org.apache.spark.dict.{NBucketDictionary, NGlobalDictionaryV2}
import org.apache.spark.util.TaskCompletionListener
+import java.util
+
object DictEncodeImpl {
@transient val cacheBucketDict: ThreadLocal[java.util.HashMap[String,
NBucketDictionary]] =
@@ -43,7 +43,7 @@ object DictEncodeImpl {
val encodeBucketId = partitionID % bucketSize.toInt
val globalDict = new NGlobalDictionaryV2(dictParams)
-val cachedBucketDict = globalDict.loadBucketDictionary(encodeBucketId)
+val cachedBucketDict = globalDict.loadBucketDictionary(encodeBucketId,
true)
DictEncodeImpl.cacheBucketDict.get.put(dictParams, cachedBucketDict)
TaskContext.get().addTaskCompletionListener(new TaskCompletionListener {
override def onTaskCompletion(context: TaskContext): Unit = {
diff --git
a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
index 2d2522b252..e7fb062c1d 100644
---
a/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
+++
b/src/spark-project/spark-common/src/main/scala/org/apache/spark/dict/NBucketDictionary.java
@@ -39,6 +39,11 @@ public class NBucketDictionary {
NBucketDictionary(String baseDir, String workingDir, int bucketId,
NGlobalDictMetaInfo metainfo)
throws IOException {
+this(baseDir, workingDir, bucketId, metainfo, false);
+}
+
+NBucketDictionary(String baseDir, String workingDir, int bucketId,
NGlobalDictMetaInfo metainfo, boolean isForColumnEncoding)
+throws IOException {
this.workingDir = workingDir;
this.bucketId = bucketId;
final NGlobalDictStore globalDictStore =
NGlobalDictStoreFactory.getResource
(kylin) 15/18: KYLIN-5789 Clean sparder history and spark history automatically
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit ca370d2dad5855941ebb699c1ff3d3eb343414f6 Author: sibingzhang <74443791+sibingzh...@users.noreply.github.com> AuthorDate: Sun Aug 13 19:57:05 2023 +0800 KYLIN-5789 Clean sparder history and spark history automatically --- .../src/main/resources/kylinSecurity.xml | 1 + .../kylin/rest/controller/NSystemController.java | 18 +- .../src/main/resources/kylinSecurity.xml | 1 + .../apache/kylin/rest/service/ProjectService.java | 6 +- .../apache/kylin/rest/service/SystemService.java | 9 + .../src/test/resources/kylinSecurity.xml | 1 + .../metadata/query/JdbcQueryHistoryStore.java | 19 ++- .../kylin/metadata/query/QueryHistoryDAO.java | 2 + .../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 5 + .../metadata/query/util/QueryHisStoreUtil.java | 5 + .../metadata/query/RDBMSQueryHistoryDaoTest.java | 17 +- .../src/main/resources/kylinSecurity.xml | 1 + .../apache/kylin/rest/service/ScheduleService.java | 60 +-- .../kylin/rest/service/ScheduleServiceTest.java| 25 ++- .../src/main/resources/kylinSecurity.xml | 1 + .../org/apache/kylin/rest/QueryNodeFilter.java | 2 + .../apache/kylin/helper/MetadataToolHelper.java| 7 +- .../org/apache/kylin/helper/RoutineToolHelper.java | 41 - .../java/org/apache/kylin/tool/AuditLogTool.java | 18 +- .../org/apache/kylin/tool/MaintainModeTool.java| 21 +-- .../constant/StringConstant.java} | 16 +- .../tool/garbage/CleanTaskExecutorService.java | 14 +- .../kylin/tool/garbage/ExecutableCleaner.java | 6 +- .../apache/kylin/tool/garbage/GarbageCleaner.java | 49 +- .../apache/kylin/tool/garbage/IndexCleaner.java| 6 +- .../apache/kylin/tool/garbage/MetadataCleaner.java | 47 - .../apache/kylin/tool/garbage/SnapshotCleaner.java | 6 +- .../kylin/tool/garbage/SourceUsageCleaner.java | 7 +- .../apache/kylin/tool/garbage/StorageCleaner.java | 190 ++--- .../apache/kylin/tool/routine/FastRoutineTool.java | 1 + .../org/apache/kylin/tool/routine/RoutineTool.java | 31 +++- .../kylin/tool/security/AdminUserInitCLI.java | 18 +- .../kylin/tool/security/KylinPasswordResetCLI.java | 18 +- .../org/apache/kylin/tool/StorageCleanerTest.java | 126 +- .../garbage/CleanTaskExecutorServiceTests.java | 2 +- .../kylin/tool/garbage/ExecutableCleanerTest.java | 8 +- .../kylin/tool/garbage/SnapshotCleanerTest.java| 2 +- .../kylin/tool/garbage/SourceUsageCleanerTest.java | 8 +- .../kylin/tool/security/AdminUserInitCLITest.java | 8 +- .../tool/security/KylinPasswordResetCLITest.java | 10 +- .../appstatus_application_1677899901295_8490 | 0 ..._1_application_1677899901295_8490_1690953331329 | 0 ..._2_application_1677899901295_8490_1690953331329 | 0 ..._1_application_1677899901295_5936_1690235831577 | 0 .../spark-history/application_1677899901295_0989 | 0 .../spark-history/application_1677899901295_8243 | 0 46 files changed, 619 insertions(+), 214 deletions(-) diff --git a/src/common-booter/src/main/resources/kylinSecurity.xml b/src/common-booter/src/main/resources/kylinSecurity.xml index 37dcfc2214..bf3f7aae8e 100644 --- a/src/common-booter/src/main/resources/kylinSecurity.xml +++ b/src/common-booter/src/main/resources/kylinSecurity.xml @@ -269,6 +269,7 @@ + diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java index a8cdcda4c2..b29d4a4eaf 100644 --- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java +++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java @@ -43,6 +43,7 @@ import org.apache.kylin.common.persistence.transaction.UnitOfWork; import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams; import org.apache.kylin.common.scheduler.EventBusFactory; import org.apache.kylin.common.util.AddressUtil; +import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting; import org.apache.kylin.helper.MetadataToolHelper; import org.apache.kylin.metadata.project.EnhancedUnitOfWork; import org.apache.kylin.metadata.project.ProjectInstance; @@ -79,8 +80,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; -import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting; - import io.swagger.annotations.ApiOperation; import lombok.val;
(kylin) 08/18: KYLIN-5783 Round query result of decimal zero by config
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8aaa87cbb3f6b9e247ced8ac34d72fb1365a39f1
Author: lixiang <447399...@qq.com>
AuthorDate: Fri Aug 11 11:48:18 2023 +0800
KYLIN-5783 Round query result of decimal zero by config
---
.../java/org/apache/kylin/common/KylinConfigBase.java| 4
.../org/apache/kylin/query/pushdown/SparkSqlClient.scala | 2 ++
.../apache/kylin/query/pushdown/SparderTypeUtilTest.java | 16
.../scala/org/apache/spark/sql/SparderTypeUtil.scala | 11 ++-
4 files changed, 32 insertions(+), 1 deletion(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index eddb15bffa..ff7bd4c7c3 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -4029,4 +4029,8 @@ public abstract class KylinConfigBase implements
Serializable {
public int getExpectedIndexSizeOptimized() {
return
Integer.parseInt(getOptional("kylin.index.expected-size-after-optimization",
"0"));
}
+
+public boolean isRoundDecimalZero() {
+return
Boolean.parseBoolean(getOptional("kylin.query.round-decimal-zero", FALSE));
+}
}
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
index d2ba4fca42..a015f6d2c3 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
@@ -18,6 +18,7 @@
package org.apache.kylin.query.pushdown
+import java.math.BigDecimal
import java.sql.Timestamp
import java.util
import java.util.concurrent.{Callable, Executors, TimeUnit, TimeoutException}
@@ -213,6 +214,7 @@ object SparkSqlClient {
case value: immutable.Map[Any, Any] =>
value.map(p => rawValueToString(p._1, true) + ":" +
rawValueToString(p._2, true)).mkString("{", ",", "}")
case value: Array[Byte] => new String(value)
+case value: BigDecimal => SparderTypeUtil.adjustDecimal(value)
case value: Any => value.toString
}
}
diff --git
a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/SparderTypeUtilTest.java
b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/SparderTypeUtilTest.java
index ea055a98ef..0626a96c3e 100644
---
a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/SparderTypeUtilTest.java
+++
b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/SparderTypeUtilTest.java
@@ -28,6 +28,8 @@ import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeSystem;
import org.apache.calcite.sql.type.BasicSqlType;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.TempMetadataBuilder;
import org.apache.spark.sql.util.SparderTypeUtil;
import org.junit.Assert;
import org.junit.Test;
@@ -140,4 +142,18 @@ public class SparderTypeUtilTest {
Assert.assertEquals(Long.class, convert_value.getClass());
Assert.assertEquals(0L, convert_value);
}
+
+@Test
+public void testBigDecimalRoundZero() {
+String tempMetadataDir =
TempMetadataBuilder.prepareLocalTempMetadata();
+KylinConfig.setKylinConfigForLocalTest(tempMetadataDir);
+KylinConfig config = KylinConfig.getInstanceFromEnv();
+Assert.assertEquals("0.000", convertToStringWithDecimalType(new
BigDecimal("0.000"), 29, 3));
+config.setProperty("kylin.query.round-decimal-zero", "true");
+Assert.assertEquals("0", convertToStringWithDecimalType(new
BigDecimal("0.000"), 29, 3));
+Assert.assertEquals("0", convertToStringWithDecimalType(new
BigDecimal("0."), 29, 4));
+Assert.assertEquals("0.001", convertToStringWithDecimalType(new
BigDecimal("0.001"), 29, 3));
+
+}
+
}
diff --git
a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
index 83499cda60..acce92b76a 100644
---
a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
+++
b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
@@ -252,6 +252,15 @@ object SparderTypeUtil extends Logging {
ret
(kylin) 07/18: KYLIN-5782 Duplicated join keys result empty query result
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1b63eefbc92e570572b5e7d92ff3d9130a5dec63
Author: Zhiting Guo <35057824+fre...@users.noreply.github.com>
AuthorDate: Fri Aug 11 11:29:39 2023 +0800
KYLIN-5782 Duplicated join keys result empty query result
-
Co-authored-by: Zhiting Guo
---
.../org/apache/kylin/query/relnode/KapJoinRel.java | 17
.../apache/kylin/query/relnode/OLAPJoinRel.java| 48 +-
2 files changed, 37 insertions(+), 28 deletions(-)
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapJoinRel.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapJoinRel.java
index a98ea03827..4a5145d64b 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapJoinRel.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapJoinRel.java
@@ -44,15 +44,14 @@ import org.apache.calcite.rel.type.RelDataTypeFieldImpl;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.util.ImmutableIntList;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.util.ICutContextStrategy;
import org.apache.kylin.query.util.RexUtils;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.val;
public class KapJoinRel extends OLAPJoinRel implements KapRel {
@@ -232,9 +231,10 @@ public class KapJoinRel extends OLAPJoinRel implements
KapRel {
if (context != null) {
collectCtxOlapInfoIfExist();
} else {
-Map joinColumns =
translateJoinColumn(this.getCondition());
+Map> joinColumns =
translateJoinColumn(this.getCondition());
pushDownJoinColsToSubContexts(joinColumns.entrySet().stream()
-.flatMap(e -> Stream.of(e.getKey(),
e.getValue())).collect(Collectors.toSet()));
+.flatMap(e -> Stream.concat(Stream.of(e.getKey()),
e.getValue().stream()))
+.collect(Collectors.toSet()));
}
}
@@ -251,9 +251,10 @@ public class KapJoinRel extends OLAPJoinRel implements
KapRel {
this.context.joins.add(join);
} else {
-Map joinColumnsMap =
translateJoinColumn(this.getCondition());
+Map> joinColumnsMap =
translateJoinColumn(this.getCondition());
Collection joinCols = joinColumnsMap.entrySet().stream()
-.flatMap(e -> Stream.of(e.getKey(),
e.getValue())).collect(Collectors.toSet());
+.flatMap(e -> Stream.concat(Stream.of(e.getKey()),
e.getValue().stream()))
+.collect(Collectors.toSet());
joinCols.stream().flatMap(e ->
e.getSourceColumns().stream()).filter(context::belongToContextTables)
.forEach(colRef -> {
context.getSubqueryJoinParticipants().add(colRef);
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index c62ba3e4ab..1c88f275c9 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -20,6 +20,7 @@ package org.apache.kylin.query.relnode;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -55,12 +56,11 @@ import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.util.ImmutableIntList;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.schema.OLAPTable;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
/**
*
*/
@@ -192,11 +192,14 @@ public class OLAPJoinRel extends EnumerableJoin
implements OLAPRel {
this.context.joins.add(join);
} else {
//When join contains subquery, the join-condition fields of
fact_table will add into context.
-Map joinCol =
translateJoinColumn(this.getCondition());
+Map> joinCol =
translateJoinColumn(this.getCondition());
-for (Map.Ent
(kylin) 04/18: KYLIN-5779 Without project permission，API "/access/acls" throws NPE
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9f269b527f33911dbad4395f7e8395dc9fdbec96
Author: Liang.Hua <36814772+jacob...@users.noreply.github.com>
AuthorDate: Thu Aug 10 13:46:10 2023 +0800
KYLIN-5779 Without project permission，API "/access/acls" throws NPE
Co-authored-by: liang.hua
---
.../src/main/java/org/apache/kylin/rest/service/AccessService.java | 6 --
.../test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java | 4
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 9e8da0d29b..54dcf242c8 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -804,7 +804,7 @@ public class AccessService extends BasicService {
return AclConstants.ADMINISTRATION;
}
Map projectPermissions = getProjectPermission(project);
-int mask = projectPermissions.get(getSid(groupName, false));
+int mask = projectPermissions.getOrDefault(getSid(groupName, false),
0);
return ExternalAclProvider.convertToExternalPermission(mask);
}
@@ -877,7 +877,9 @@ public class AccessService extends BasicService {
SidPermissionWithAclResponse permissionWithAclResponse = principal
? getUserPermissionWithAclResponse(project,
userOrGroupName)
: getGroupPermissionWithAclResponse(project,
userOrGroupName);
-sidPermissionWithAclResponse.add(permissionWithAclResponse);
+if
(!AclConstants.EMPTY.equals(permissionWithAclResponse.getProjectPermission())) {
+sidPermissionWithAclResponse.add(permissionWithAclResponse);
+}
}
return sidPermissionWithAclResponse;
}
diff --git
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
index 9af9f92b14..ca41c60de0 100644
---
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
+++
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
@@ -1334,6 +1334,10 @@ public class AclTCRServiceTest extends
NLocalFileMetadataTestCase {
responses = accessService.getUserOrGroupAclPermissions(projects,
"ANALYST", true);
Assert.assertEquals(1, responses.size());
Assert.assertEquals("OPERATION",
responses.get(0).getProjectPermission());
+
+// test user have not project permission
+responses =
accessService.getUserOrGroupAclPermissions(Lists.newArrayList("ssb"),
"ANALYST", true);
+Assert.assertEquals(0, responses.size());
}
@Test
(kylin) 17/18: KYLIN-5786 Add a write lock when merging v3 dict
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 30c4f28a1616dc80e674c4c4f7017b6cc63114b4
Author: Yaguang Jia
AuthorDate: Fri Aug 11 13:10:02 2023 +0800
KYLIN-5786 Add a write lock when merging v3 dict
---
.../spark/builder/v3dict/DictionaryBuilder.scala | 63 +++---
.../builder/v3dict/GlobalDictionarySuite.scala | 12 +++--
2 files changed, 51 insertions(+), 24 deletions(-)
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
index e47dc7107e..5fd9ae1673 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/v3dict/DictionaryBuilder.scala
@@ -40,6 +40,7 @@ import util.retry.blocking.RetryStrategy.RetryStrategyProducer
import util.retry.blocking.{Failure, Retry, RetryStrategy, Success}
import java.nio.file.Paths
+import java.util.concurrent.locks.ReentrantReadWriteLock
import scala.collection.mutable.ListBuffer
import scala.concurrent.duration.DurationInt
import scala.util.control.NonFatal
@@ -48,6 +49,7 @@ object DictionaryBuilder extends Logging {
implicit val retryStrategy: RetryStrategyProducer =
RetryStrategy.randomBackOff(5.seconds, 15.seconds, maxAttempts = 20)
+ lazy val v3dictMergeLock = new ReentrantReadWriteLock(true)
def buildGlobalDict(
project: String,
@@ -86,11 +88,12 @@ object DictionaryBuilder extends Logging {
private def transformerDictPlan(
spark: SparkSession,
context: DictionaryContext,
- plan: LogicalPlan): LogicalPlan = {
+ plan: LogicalPlan): DictPlanVersion = {
val dictPath = getDictionaryPathAndCheck(context)
-val dictTable: DeltaTable = DeltaTable.forPath(dictPath)
-val maxOffset = dictTable.toDF.count()
+val dictVersion = DeltaLog.forTable(spark, dictPath).snapshot.version
+val dictTableDF = spark.read.format("delta").option("versionAsOf",
dictVersion).load(dictPath);
+val maxOffset = dictTableDF.count()
logInfo(s"Dict $dictPath item count $maxOffset")
plan match {
@@ -99,19 +102,19 @@ object DictionaryBuilder extends Logging {
val windowSpec =
org.apache.spark.sql.expressions.Window.orderBy(col(column))
val joinCondition = createColumn(
EqualTo(col(column).cast(StringType).expr,
-getLogicalPlan(dictTable.toDF).output.head))
-val filterKey = getLogicalPlan(dictTable.toDF).output.head.name
+getLogicalPlan(dictTableDF).output.head))
+val filterKey = getLogicalPlan(dictTableDF).output.head.name
val antiJoinDF = getDataFrame(spark, windowChild)
.filter(col(filterKey).isNotNull)
- .join(dictTable.toDF,
+ .join(dictTableDF,
joinCondition,
"left_anti")
.select(col(column).cast(StringType) as "dict_key",
(row_number().over(windowSpec) + lit(maxOffset)).cast(LongType) as
"dict_value")
logInfo(s"Dict logical plan :
${antiJoinDF.queryExecution.logical.treeString}")
-getLogicalPlan(antiJoinDF)
+DictPlanVersion(getLogicalPlan(antiJoinDF), dictVersion)
- case _ => plan
+ case _ => DictPlanVersion(plan, dictVersion)
}
}
@@ -166,32 +169,52 @@ object DictionaryBuilder extends Logging {
val dictPath = getDictionaryPath(context)
logInfo(s"Save dict values into path $dictPath.")
try {
- dictDF.write.mode(SaveMode.Overwrite).format("delta").save(dictPath)
+ dictDF.write.mode(SaveMode.ErrorIfExists).format("delta").save(dictPath)
} catch {
case e: DeltaConcurrentModificationException =>
logWarning(s"Concurrent modifications occurred: $dictPath", e)
throw e
case NonFatal(e) =>
+logWarning(s"A NoFatal exception occurs when initializing the
dictionary, and will be retried", e)
if (!DeltaTable.isDeltaTable(dictPath)) {
logWarning(s"Try to delete v3dict: $dictPath", e)
HadoopUtil.deletePath(HadoopUtil.getCurrentConfiguration, new
Path(dictPath))
}
throw e
+ case e: Throwable => throw e
}
}
private def mergeIncrementDict(spark: SparkSession, context:
DictionaryContext, plan: LogicalPlan): Unit = {
-val dictPlan = transformerDictPlan(spark, context, plan)
-val incrementDictDF
(kylin) branch kylin5 updated (f6cc0ce32c -> 4ed3729312)
This is an automated email from the ASF dual-hosted git repository. liyang pushed a change to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git from f6cc0ce32c KYLIN-5820,added pagination and add count distinct function in batch measure dialog new 8ab87c93b9 KYLIN-5775 upgrade testcontainers to 1.16.2 new 5d781d1917 KYLIN-5776 Support parameter type is string for timestampadd function new a67e854a1d KYLIN-5777 Change spark.yarn.submit.file.replication from 1 to 3 new 9f269b527f KYLIN-5779 Without project permission，API "/access/acls" throws NPE new 8929bb820c KYLIN-5780 Dimension capacity of aggGroup does not work new cb2a11d503 KYLIN-5781 Adjust the query result order of sql with union new 1b63eefbc9 KYLIN-5782 Duplicated join keys result empty query result new 8aaa87cbb3 KYLIN-5783 Round query result of decimal zero by config new 40a10818a0 Add some code to compatible with the enterprise version new 1367e9ccff KYLIN-5784 Optimization for query history filters new d7d8098598 KYLIN-5785 Support implicit query of split function new dd16079c42 KYLIN-5787 Use t-digest as spark percentile_approx function new 9db3502f80 KYLIN-5774 Optimize Calcite plan to convert spark logical plan new 775efe62b5 KYLIN-5788 Enhance global dict on flat table encoding stage logging & retry new ca370d2dad KYLIN-5789 Clean sparder history and spark history automatically new 2695c5da83 KYLIN-5790 Security of kafka-clients new 30c4f28a16 KYLIN-5786 Add a write lock when merging v3 dict new 4ed3729312 KYLIN-5778 Upgrade spark to 3.2.0-kylin-4.6.14.0 The 18 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml| 10 +- .../src/main/resources/config/init.properties | 2 +- .../src/main/resources/kylinSecurity.xml | 1 + .../kylin/rest/controller/NSystemController.java | 18 +- .../src/main/resources/kylinSecurity.xml | 1 + .../apache/kylin/rest/service/AccessService.java | 6 +- .../apache/kylin/rest/service/ProjectService.java | 6 +- .../apache/kylin/rest/service/SystemService.java | 9 + .../kylin/rest/service/AclTCRServiceTest.java | 4 + .../src/test/resources/kylinSecurity.xml | 1 + .../org/apache/kylin/common/KylinConfigBase.java | 19 ++- .../common/exception/code/ErrorCodeServer.java | 3 + .../src/main/resources/kylin-defaults0.properties | 3 +- .../resources/kylin_error_msg_conf_cn.properties | 3 + .../resources/kylin_error_msg_conf_en.properties | 3 + .../kylin_error_suggestion_conf_cn.properties | 3 + .../kylin_error_suggestion_conf_en.properties | 3 + .../main/resources/kylin_errorcode_conf.properties | 3 + .../measure/percentile/PercentileCounter.java | 1 + .../apache/kylin/metadata/model/NDataModel.java| 9 + .../metadata/query/JdbcQueryHistoryStore.java | 19 ++- .../kylin/metadata/query/QueryHistoryDAO.java | 2 + .../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 5 + .../metadata/query/util/QueryHisStoreUtil.java | 5 + .../recommendation/entity/DimensionRecItemV2.java | 17 +- .../metadata/query/RDBMSQueryHistoryDaoTest.java | 17 +- .../src/main/resources/config/init.properties | 2 +- .../src/main/resources/kylinSecurity.xml | 1 + .../kylin/rest/controller/SegmentController.java | 6 +- .../rest/controller/SegmentControllerTest.java | 43 - .../org/apache/kylin/rest/KafkaCondition.java} | 20 +-- .../apache/kylin/rest/service/KafkaService.java| 8 +- .../apache/kylin/rest/service/TableService.java| 2 +- .../apache/kylin/rest/service/ScheduleService.java | 60 +-- .../kylin/rest/service/ScheduleServiceTest.java| 25 ++- .../kylin/newten/EnhancedAggPushDownTest.java | 2 +- ...itInfoTest.java => SpecialColumnNamesTest.java} | 68 +++- .../apache/kylin/query/engine/QueryExecTest.java | 59 +-- .../java/org/apache/kylin/util/ExecAndComp.java| 42 +++-- .../_global/project/special_column_names.json} | 2 +- .../c41390c5-b35d-4db3-b167-029874b85a2c.json} | 2 +- .../c41390c5-b35d-4db3-b167-029874b85a2c.json} | 2 +- .../c41390c5-b35d-4db3-b167-029874b85a2c.json} | 10 +- .../special_column_names}/table/SSB.CUSTOMER.json | 6 +- .../table/SSB.P_LINEORDER.json | 10 +- .../kylin/rest/controller/KafkaController.java | 2 +- .../kylin/rest/controller/NModelController.java| 11 +- .../rest/controller/NModelControllerTest.java | 107 +++- .../kylin/rest/service/FusionIndexService.java | 10 ++ .../kylin/rest/serv
(kylin) branch kylin5 updated: KYLIN-5820,added pagination and add count distinct function in batch measure dialog
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new f6cc0ce32c KYLIN-5820,added pagination and add count distinct
function in batch measure dialog
f6cc0ce32c is described below
commit f6cc0ce32c82e3102a2fe27db9c3b9a99f43bd47
Author: huangchunyan
AuthorDate: Fri Apr 5 16:02:27 2024 +0800
KYLIN-5820,added pagination and add count distinct function in batch
measure dialog
---
kystudio/src/components/common/pager.vue | 8 +-
.../studio/StudioModel/AddMeasure/index.vue| 2 +-
.../studio/StudioModel/BatchMeasureModal/index.vue | 287 -
3 files changed, 169 insertions(+), 128 deletions(-)
diff --git a/kystudio/src/components/common/pager.vue
b/kystudio/src/components/common/pager.vue
index 927414f2cb..383f926917 100644
--- a/kystudio/src/components/common/pager.vue
+++ b/kystudio/src/components/common/pager.vue
@@ -43,6 +43,10 @@ import { Component } from 'vue-property-decorator'
refTag: {
type: String,
default: ''
+},
+isNeedLocalStorage: {
+ type: Boolean,
+ default: true
}
},
methods: {
@@ -53,7 +57,7 @@ import { Component } from 'vue-property-decorator'
sizeChange (size) {
this.pageSize = size
this.currentPage = 0
- cacheLocalStorage(this.refTag, size)
+ this.isNeedLocalStorage && this.refTag && cacheLocalStorage(this.refTag,
size)
this.$emit('handleCurrentChange', this.currentPage, size)
}
}
@@ -61,7 +65,7 @@ import { Component } from 'vue-property-decorator'
export default class Pager extends Vue {
data () {
return {
- pageSize: +localStorage.getItem(this.refTag) || this.perPageSize ||
bigPageCount,
+ pageSize: this.refTag && +localStorage.getItem(this.refTag) ||
this.perPageSize || bigPageCount,
pageSizes: pageSizes,
currentPage: this.curPage
}
diff --git a/kystudio/src/components/studio/StudioModel/AddMeasure/index.vue
b/kystudio/src/components/studio/StudioModel/AddMeasure/index.vue
index abe837945a..9f304cbff3 100644
--- a/kystudio/src/components/studio/StudioModel/AddMeasure/index.vue
+++ b/kystudio/src/components/studio/StudioModel/AddMeasure/index.vue
@@ -438,7 +438,7 @@ export default class AddMeasure extends Vue {
}
get flattenLookupTables () {
-return this.modelInstance.anti_flatten_lookups
+return this.modelInstance?.anti_flatten_lookups ?? []
}
get showSync () {
diff --git
a/kystudio/src/components/studio/StudioModel/BatchMeasureModal/index.vue
b/kystudio/src/components/studio/StudioModel/BatchMeasureModal/index.vue
index 452c3020bf..a732940d8c 100644
--- a/kystudio/src/components/studio/StudioModel/BatchMeasureModal/index.vue
+++ b/kystudio/src/components/studio/StudioModel/BatchMeasureModal/index.vue
@@ -1,5 +1,5 @@
-
-
-
+
+
-
+
-
+
-
+
+
+
+
+
+
+
@@ -76,32 +82,38 @@
-
-
+
+
-
+
-
+
-
+
+
+
+
+
+
+
@@ -122,28 +134,34 @@
:data="ccTable.columns"
:ref="ccTable.guid">
-
-
+
+
-
+
-
+
-
+
+
+
+
+
+
+
@@ -154,7 +172,7 @@
:empty-text="emptyText"
:k
(kylin) branch kylin5 updated: KYLIN-5813,improve query history page filter item and model list page button text
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 8c91a1a4ed KYLIN-5813,improve query history page filter item and model
list page button text
8c91a1a4ed is described below
commit 8c91a1a4eddbf0ed29f0e81487ea1e088cbf736b
Author: huangchunyan
AuthorDate: Fri Apr 5 15:07:12 2024 +0800
KYLIN-5813,improve query history page filter item and model list page
button text
---
.../common/DropdownFilter/DropdownFilter.vue | 354 +-
kystudio/src/components/query/query_history.vue| 3 +
.../src/components/query/query_history_table.vue | 509 -
.../studio/StudioModel/ModelList/index.vue | 27 +-
kystudio/src/config/index.js | 3 +
kystudio/src/service/datasource.js | 2 +-
6 files changed, 641 insertions(+), 257 deletions(-)
diff --git a/kystudio/src/components/common/DropdownFilter/DropdownFilter.vue
b/kystudio/src/components/common/DropdownFilter/DropdownFilter.vue
index e6c8c9a719..99fbb8528c 100644
--- a/kystudio/src/components/common/DropdownFilter/DropdownFilter.vue
+++ b/kystudio/src/components/common/DropdownFilter/DropdownFilter.vue
@@ -37,26 +37,56 @@ import { getPickerOptions } from './handler'
type: Array,
default: () => []
},
+options2: {
+ type: Array,
+ default: () => []
+},
hideArrow: {
type: Boolean
+},
+isShowfooter: {
+ type: Boolean,
+ default: true
+},
+filterScrollMaxHeight: {
+ type: String,
+ default: '130'
+},
+isLoadingData: Boolean,
+isLoading: Boolean,
+loadingTips: String,
+isShowSearchInput: Boolean,
+filterPlaceholder: String,
+optionsTitle: String,
+totalSizeLabel: String,
+isShowDropDownImme: Boolean,
+isSelectAllOption: {
+ type: Object,
+ default: null
}
},
locales
})
export default class DropdownFilter extends Vue {
isShowDropDown = false
+ startSec = 0
+ endSec = 10
+ searchValue = ''
+ isAll = false
+ timer = null
get resetValue () {
const { type } = this
switch (type) {
case 'checkbox': return []
+ case 'inputNumber': return [null, null]
default: return null
}
}
get isPopoverType () {
const { type } = this
-return ['checkbox'].includes(type)
+return ['checkbox', 'inputNumber'].includes(type)
}
get isDatePickerType () {
@@ -84,6 +114,27 @@ export default class DropdownFilter extends Vue {
}
}
+ handleChangeAll () {
+this.$emit('handleChangeAll')
+ }
+ saveLatencyRange () {
+const latencyFrom = this.startSec
+let latencyTo = null
+if (this.startSec > this.endSec) {
+ latencyTo = this.endSec = this.startSec
+} else {
+ latencyTo = this.endSec
+}
+this.handleInput([latencyFrom, latencyTo])
+this.handleToggleDropdown()
+ }
+ resetLatency () {
+this.startSec = 0
+this.endSec = 10
+this.handleClearValue()
+this.handleToggleDropdown()
+ }
+
handleClearValue () {
this.$emit('input', this.resetValue)
}
@@ -97,28 +148,76 @@ export default class DropdownFilter extends Vue {
handleToggleDropdown () {
this.handleSetDropdown(!this.isShowDropDown)
+if (this.isShowDropDown) {
+ this.bindScrollEvent()
+}
+ }
+
+ removeScrollEvent () {
+const groupBlocks =
this.$refs.$checkBoxGroup.querySelectorAll('.group-block .scroll-content')
+if (groupBlocks.length) {
+ for (let group of groupBlocks) {
+group.removeEventListener('scroll', this.addScrollEvent, false)
+ }
+}
+ }
+ bindScrollEvent () {
+const groupBlocks =
this.$refs.$checkBoxGroup.querySelectorAll('.group-block .scroll-content')
+console.log(groupBlocks)
+if (groupBlocks.length) {
+ for (let group of groupBlocks) {
+group.addEventListener('scroll', this.addScrollEvent, false)
+ }
+}
}
handlerClickEvent () {
this.$refs.$datePicker.$el.click()
}
+ addScrollEvent (e) {
+try {
+ const scrollT = e.target.scrollTop
+ if (scrollT > 0) {
+e.target.parentNode.className = 'group-block is-scrollable-top'
+ } else {
+e.target.parentNode.className = 'group-block'
+ }
+ let scrollH = e.target.scrollHeight
+ let clientH = e.target.clientHeight
+ if (scrollT + clientH === scrollH) {
+this.$emit('filter-scroll-bottom')
+ }
+} catch (e) {
+ console.error(e)
+}
+ }
+
mounted () {
this.isDatePickerType && this.$slots.default && this.$slots.default.length
&& this.$slots.default[0].elm.addEventListener('click', this.handlerClickEvent)
+if (this.isShowDropDownImme) {
+ this.$nextTick(() =>
(kylin) branch kylin5 updated: fix frontend build sh
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new fd3b1313c0 fix frontend build sh
fd3b1313c0 is described below
commit fd3b1313c0fbbd9f4817714e887631538da98100
Author: huangchunyan
AuthorDate: Mon Apr 8 20:52:05 2024 +0800
fix frontend build sh
---
build/release/build.sh | 2 +-
kystudio/build/build.js | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/build/release/build.sh b/build/release/build.sh
index 4549745e7d..f759bd3c52 100755
--- a/build/release/build.sh
+++ b/build/release/build.sh
@@ -29,7 +29,7 @@ echo 'Build front-end'
if [ "${WITH_FRONT}" = "1" ]; then
cd kystudio
echo 'Install front-end dependencies'
-npm install || { exit 1; }
+npm install --registry=https://registry.npmjs.org/ --legacy-peer-deps || {
exit 1; }
echo 'Install front-end end'
npm run build || { exit 1; }
echo 'build front-end dist end'
diff --git a/kystudio/build/build.js b/kystudio/build/build.js
index 6b8add100c..1483a7f430 100644
--- a/kystudio/build/build.js
+++ b/kystudio/build/build.js
@@ -31,5 +31,6 @@ rm(path.join(config.build.assetsRoot,
config.build.assetsSubDirectory), err => {
' Tip: built files are meant to be served over an HTTP server.\n' +
' Opening index.html over file:// won\'t work.\n'
))
+process.exit(0)
})
})
(kylin) 09/14: KYLIN-5768 Refine and adjust some code for localCache
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 60da9e0ce1fa789ad580ca62ae4d47162f9d3201 Author: Yinghao Lin <39019287+yhca...@users.noreply.github.com> AuthorDate: Fri Jul 21 21:09:31 2023 +0800 KYLIN-5768 Refine and adjust some code for localCache - Co-authored-by: sibingzhang <74443791+sibingzh...@users.noreply.github.com> Co-authored-by: sibing.zhang --- pom.xml| 12 +- .../rest/controller/NBasicControllerTest.java | 2 +- .../apache/kylin/rest/service/ProjectService.java | 9 +- .../org/apache/kylin/common/KylinConfigBase.java | 8 ++ .../kylin/metadata/cube/model/IndexPlan.java | 136 +++-- .../cube/optimization/AbstractOptStrategy.java | 4 +- .../cube/optimization/GarbageLayoutType.java | 2 +- .../optimization/IncludedLayoutOptStrategy.java| 2 +- .../metadata/cube/optimization/IndexOptimizer.java | 14 ++- .../cube/optimization/IndexOptimizerFactory.java | 12 +- .../optimization/LowFreqLayoutOptStrategy.java | 2 +- .../cube/optimization/MergedLayoutOptStrategy.java | 59 + .../optimization/SimilarLayoutOptStrategy.java | 2 +- .../cube/optimization/event/ApproveRecsEvent.java} | 27 ++-- .../cube/optimization/event/BuildIndexEvent.java} | 26 ++-- .../cube/storage/GarbageStorageCollector.java | 2 +- .../metadata/cube/utils/IndexPlanReduceUtil.java | 52 +++- .../storage/ProjectStorageInfoCollectorTest.java | 74 +-- .../cube/utils/IndexPlanReduceUtilTest.java| 27 +++- .../kylin/rest/initialize/BuildAppInitializer.java | 5 + .../kylin/rest/service/ModelBuildService.java | 38 -- .../kylin/rest/service/ModelServiceBuildTest.java | 36 +- .../kylin/rest/controller/NProjectController.java | 14 ++- .../rest/controller/NProjectControllerTest.java| 2 +- .../kylin/rest/service/AbstractModelService.java | 114 - .../kylin/rest/service/BaseIndexUpdateHelper.java | 17 ++- .../kylin/rest/service/IndexPlanService.java | 66 +++--- .../apache/kylin/rest/service/ModelService.java| 33 +++-- .../kylin/rest/service/IndexPlanServiceTest.java | 52 .../kylin/rest/service/ProjectServiceTest.java | 2 +- .../plugin/diagnose/DiagnoseExecutorPlugin.scala | 9 +- .../common/logging/AbstractHdfsLogAppender.java| 7 +- .../common/logging/SparkDriverHdfsLogAppender.java | 2 +- .../logging/SparkExecutorHdfsLogAppender.java | 4 +- .../org/apache/spark/utils/SparkHadoopUtils.scala | 32 - .../org/apache/kylin/helper/RoutineToolHelper.java | 8 +- .../kylin/tool/garbage/ExecutableCleaner.java | 10 ++ .../apache/kylin/tool/garbage/GarbageCleaner.java | 11 +- .../apache/kylin/tool/garbage/IndexCleaner.java| 129 ++- .../apache/kylin/tool/garbage/MetadataCleaner.java | 6 + .../apache/kylin/tool/garbage/SnapshotCleaner.java | 10 ++ 41 files changed, 888 insertions(+), 191 deletions(-) diff --git a/pom.xml b/pom.xml index 9b68f4a83d..ff1f965ea3 100644 --- a/pom.xml +++ b/pom.xml @@ -258,7 +258,11 @@ **/org/apache/kylin/tool/hadoop/KapGetPathWithoutSchemeAndAuthorityCLI.java, **/org/apache/kylin/engine/spark/streaming/**/*, **/org/apache/kylin/rest/security/KerberosLoginManager.java, -**/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java + **/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java, +**/org/apache/kylin/tool/garbage/IndexCleaner.java, + **/org/apache/kylin/spark/common/logging/SparkExecutorHdfsLogAppender.java, +**/org/apache/kylin/rest/service/AbstractModelService.java, +**/org/apache/spark/utils/SparkHadoopUtils.scala **/org/apache/kylin/**/*Exception.java, @@ -304,7 +308,11 @@ **/org/apache/kylin/rest/util/InitResourceGroupUtils.java, **/org/apache/kylin/rest/QueryNodeFilter.java, **/org/apache/kylin/rest/config/SwaggerConfig.java, -**/org/apache/kylin/rest/config/SwaggerCompatibilityConfig.java +**/org/apache/kylin/rest/config/SwaggerCompatibilityConfig.java, +**/org/apache/kylin/tool/garbage/IndexCleaner.java, + **/org/apache/kylin/spark/common/logging/SparkExecutorHdfsLogAppender.java, +**/org/apache/kylin/rest/service/AbstractModelService.java, +**/org/apache/spark/utils/SparkHadoopUtils.scala kylin diff --git a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java index 34ac02cab2..586fc
(kylin) 13/14: KYLIN-5772 Fix vulnerability & excluding okio
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 90aa1def3d0bd2c1b2f6548923d9bde8af0d6cee
Author: Yinghao Lin
AuthorDate: Wed Aug 2 11:36:43 2023 +0800
KYLIN-5772 Fix vulnerability & excluding okio
---
pom.xml | 4
1 file changed, 4 insertions(+)
diff --git a/pom.xml b/pom.xml
index 04867d1347..765cbefa09 100644
--- a/pom.xml
+++ b/pom.xml
@@ -963,6 +963,10 @@
org.codehaus.jackson
jackson-mapper-asl
+
+com.squareup.okio
+okio
+
${hadoop.version}
provided
(kylin) 04/14: KYLIN-5764 Sum expression supports if condition
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit cd09148868fd2b4cb5a08d236e1a0c0e8dd5eaea Author: Pengfei Zhan AuthorDate: Wed Jul 5 17:50:34 2023 +0800 KYLIN-5764 Sum expression supports if condition --- .../kylin/query/engine/SumExprPlannerTest.xml | 63 ++ .../resources/query/sql_sum_expr/query00-if.sql| 45 + .../apache/kylin/query/util/AggExpressionUtil.java | 215 - 3 files changed, 141 insertions(+), 182 deletions(-) diff --git a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml index 0796314ef7..5cd88d79e1 100644 --- a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml +++ b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml @@ -80,6 +80,69 @@ KapOLAPToEnumerableConverter KapTableScan(table=[[DEFAULT, TEST_COUNTRY]], ctx=[], fields=[[0, 1, 2, 3]]) KapTableScan(table=[[DEFAULT, TEST_COUNTRY]], ctx=[], fields=[[0, 1, 2, 3]]) ]]> + + + + + + +
(kylin) 11/14: KYLIN-5770 Upgrade spring-security-config from 5.7.8 from 5.7.10
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3dd581ecfa47477892da5590d168ccf6b28e5f13
Author: jlf
AuthorDate: Wed Jul 26 22:02:04 2023 +0800
KYLIN-5770 Upgrade spring-security-config from 5.7.8 from 5.7.10
---
pom.xml | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index ff1f965ea3..04867d1347 100644
--- a/pom.xml
+++ b/pom.xml
@@ -218,6 +218,7 @@
2.7.12
2.6.10
2.6.1-kylin-r4
+
5.7.10
1.0.10.RELEASE
5.3.26
2.6.6
@@ -1359,7 +1360,7 @@
com.twitter
- parquet-hadoop-bundle
+parquet-hadoop-bundle
org.apache.parquet
@@ -2411,6 +2412,11 @@
+
+org.springframework.security
+spring-security-config
+${spring.framework.security.config.version}
+
org.springframework.security.extensions
spring-security-saml2-core
(kylin) 08/14: KYLIN-5745 Using a global thread pool to clean underlying storages
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 180ff07afeaf64da8e565b7aa5882c3a89f1c268
Author: fengguangyuan
AuthorDate: Sat Jul 1 12:07:43 2023 +0800
KYLIN-5745 Using a global thread pool to clean underlying storages
1. Using a global thread pool to clean underlying storages;
2. Launching cleaning tasks in the local thread and to ignore
FileNotFoundException while collecting HDFS files.
Co-authored-by: Guangyuan Feng
---
.../apache/kylin/rest/config/AppInitializer.java | 7 +
.../kylin/rest/service/AsyncTaskService.java | 9 +-
.../org/apache/kylin/common/KylinConfigBase.java | 8 +
.../kylin/metadata/query/QueryHistoryDAO.java | 6 +-
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 13 +-
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 2 +-
.../apache/kylin/rest/service/ScheduleService.java | 10 +-
.../kylin/rest/service/MetaStoreService.java | 10 +-
.../rest/service/MetaStoreTenantServiceTest.java | 10 +-
.../org/apache/kylin/rest/InitConfiguration.java | 1 -
.../org/apache/kylin/rest/HAConfigurationTest.java | 6 +-
.../org/apache/kylin/helper/RoutineToolHelper.java | 55 +++-
.../tool/garbage/AbstractComparableCleanTask.java | 93 ++
.../tool/garbage/CleanTaskExecutorService.java | 236 +++
.../kylin/tool/garbage/PriorityExecutor.java | 71 +
.../apache/kylin/tool/garbage/StorageCleaner.java | 28 +-
.../kylin/tool/garbage/StorageCleanerCLI.java | 18 +-
.../apache/kylin/tool/routine/FastRoutineTool.java | 1 +
.../org/apache/kylin/tool/routine/RoutineTool.java | 24 +-
.../org/apache/kylin/tool/util/MetadataUtil.java | 9 +-
.../garbage/CleanTaskExecutorServiceTests.java | 324 +
21 files changed, 889 insertions(+), 52 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
index 5a2a833305..35af91a415 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
@@ -60,6 +60,8 @@ import
org.apache.kylin.rest.service.task.QueryHistoryTaskScheduler;
import org.apache.kylin.rest.util.JStackDumpTask;
import org.apache.kylin.streaming.jobs.StreamingJobListener;
import org.apache.kylin.tool.daemon.KapGuardianHATask;
+import org.apache.kylin.tool.garbage.CleanTaskExecutorService;
+import org.apache.kylin.tool.garbage.PriorityExecutor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationContext;
@@ -175,6 +177,11 @@ public class AppInitializer {
log.info("KylinConfig in env, ID is {}", kylinConfig.hashCode());
log.info("KylinConfig in env, metadata is {}",
kylinConfig.getMetadataUrl());
log.info("KylinConfig in env, working dir is {}",
kylinConfig.getHdfsWorkingDirectory());
+
+// Init global static instances
+CleanTaskExecutorService.getInstance().bindWorkingPool(
+() -> PriorityExecutor.newWorkingThreadPool(
+"clean-storages-pool",
kylinConfig.getStorageCleanTaskConcurrency()));
}
private void warmUpSystemCache() {
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
b/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
index a343a464c6..8046f0c62a 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/service/AsyncTaskService.java
@@ -27,6 +27,7 @@ import java.time.ZoneOffset;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.servlet.ServletOutputStream;
@@ -50,6 +51,7 @@ import org.apache.kylin.metadata.query.QueryHistoryInfo;
import org.apache.kylin.metadata.query.QueryHistoryRequest;
import org.apache.kylin.metadata.query.QueryHistorySql;
import org.apache.kylin.metadata.query.util.QueryHistoryUtil;
+import org.apache.kylin.tool.garbage.CleanTaskExecutorService;
import org.apache.kylin.tool.garbage.StorageCleaner;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
@@ -69,8 +71,11 @@ public class AsyncTaskService implements
AsyncTaskServiceSupporter {
long startAt = System.currentTimeMillis();
try {
-val storageCleaner = new StorageCle
(kylin) 12/14: KYLIN-5771 Query cannot be interrupted during segment pruning
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 64b52dea660fc5feba34b8650557f1323d55b28b
Author: fengguangyuan
AuthorDate: Thu Jul 27 15:46:27 2023 +0800
KYLIN-5771 Query cannot be interrupted during segment pruning
Co-authored-by: Guangyuan Feng
---
.../kylin/query/util/QueryInterruptChecker.java| 53 +++-
.../apache/kylin/query/util/SlowQueryDetector.java | 2 +-
.../query/routing/PartitionPruningRuleTest.java| 313 +
.../query/routing/SegmentPruningRuleTest.java | 277 ++
.../kylin/query/routing/PartitionPruningRule.java | 15 +
.../kylin/query/routing/SegmentPruningRule.java| 33 ++-
.../kylin/query/util/QueryContextCutter.java | 5 +
7 files changed, 689 insertions(+), 9 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
b/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
index b163cc5a31..df4eb94589 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/query/util/QueryInterruptChecker.java
@@ -20,6 +20,7 @@ package org.apache.kylin.query.util;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.common.exception.KylinRuntimeException;
import org.apache.kylin.common.exception.KylinTimeoutException;
import org.apache.kylin.query.exception.UserStopQueryException;
@@ -32,15 +33,65 @@ public class QueryInterruptChecker {
// This is Utils.
}
+/**
+ * @deprecated Use {@link
this#checkQueryCanceledOrThreadInterrupted(String, String)} instead.
+ * The semantic of this method is confused in some scenarios.
+ */
+@Deprecated
public static void checkThreadInterrupted(String errorMsgLog, String
stepInfo) {
if (Thread.currentThread().isInterrupted()) {
log.error("{} {}", QueryContext.current().getQueryId(),
errorMsgLog);
-if
(SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser())
{
+if
(SlowQueryDetector.getRunningQueries().containsKey(Thread.currentThread())
+&&
SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser())
{
throw new UserStopQueryException("");
}
+
QueryContext.current().getQueryTagInfo().setTimeout(true);
throw new KylinTimeoutException("The query exceeds the set time
limit of "
+
KylinConfig.getInstanceFromEnv().getQueryTimeoutSeconds() + "s. " + stepInfo);
}
}
+
+/**
+ * Within the global context, STOP is same to CANCEL, therefore to stop a
query is equal to
+ * cancel a query.
+ * There are some possible reasons to cancel the query recorded in the
current thread context.
+ * {@link UserStopQueryException} is for stopping the query from the
request, see
+ * {@link SlowQueryDetector#stopQuery(String)}.
+ * {@link KylinTimeoutException} is for run out of the timeout of the
query, see
+ * {@link SlowQueryDetector::checkTimeout()}.
+ * {@link KylinRuntimeException} is risking inconsistent states to stop
the query.
+ * {@link InterruptedException} is for other interruptions.
+ * @param cause the reason of canceling the current query or interrupt the
working thread
+ * @param step the processing point
+ */
+public static void checkQueryCanceledOrThreadInterrupted(String cause,
String step) throws InterruptedException {
+SlowQueryDetector.QueryEntry entry =
SlowQueryDetector.getRunningQueries().getOrDefault(Thread.currentThread(),
+null);
+if (entry != null) {
+if (entry.isStopByUser() &&
entry.getPlannerCancelFlag().isCancelRequested()
+&& Thread.currentThread().isInterrupted()) {
+throw new UserStopQueryException(String.format("Manually stop
the query %s. Caused: %s. Step: %s",
+entry.getQueryId(), cause, step));
+}
+
+if (entry.getPlannerCancelFlag().isCancelRequested() &&
Thread.currentThread().isInterrupted()) {
+QueryContext.current().getQueryTagInfo().setTimeout(true);
+throw new KylinTimeoutException(String.format("Run out of time
of the query %s. Caused: %s. Step: %s",
+entry.getQueryId(), cause, step));
+}
+
+if (entry.isStopByUser() ||
entry.getPlannerCancelFlag().isCancelRequested()) {
+throw new UserStopQueryException(String.format(
+"You are trying to cancel the query %
(kylin) 10/14: KYLIN-5769 Support split function
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 738cf4af01fca030751eaff0b1ea3a59340d83eb
Author: Pengfei Zhan
AuthorDate: Wed Jul 5 22:12:00 2023 +0800
KYLIN-5769 Support split function
---
.../apache/kylin/metadata/datatype/DataType.java | 2 +-
.../rest/controller/NBuildAndQueryMetricsTest.java | 15 ++-
.../kylin/query/engine/QueryRoutingEngine.java | 80 +--
.../org/apache/kylin/query/udf/SparkStringUDF.java | 7 +-
.../engine/spark/builder/CreateFlatTable.scala | 26 ++---
.../job/stage/build/FlatTableAndDictBase.scala | 82 +---
.../spark/smarter/IndexDependencyParser.scala | 7 +-
.../spark/utils/ComputedColumnEvalUtilTest.java| 2 +
.../kylin/query/runtime/ExpressionConverter.scala | 2 +-
.../kylin/query/runtime/plan/TableScanPlan.scala | 15 +--
.../org/apache/spark/sql/SparderTypeUtilTest.scala | 29 +-
.../org/apache/spark/sql/SparderTypeUtil.scala | 107 -
.../apache/spark/sql/common/SparderQueryTest.scala | 75 +++
13 files changed, 254 insertions(+), 195 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 81e5881ef5..20057ad919 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -153,7 +153,7 @@ public class DataType implements Serializable {
LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
}
-private static final ConcurrentMap CACHE = new
ConcurrentHashMap();
+private static final ConcurrentMap CACHE = new
ConcurrentHashMap<>();
public static final DataType ANY = DataType.getType(ANY_STR);
diff --git
a/src/kylin-server-it/src/test/java/org/apache/kylin/rest/controller/NBuildAndQueryMetricsTest.java
b/src/kylin-server-it/src/test/java/org/apache/kylin/rest/controller/NBuildAndQueryMetricsTest.java
index 7ff9b40cf9..00f8522c4a 100644
---
a/src/kylin-server-it/src/test/java/org/apache/kylin/rest/controller/NBuildAndQueryMetricsTest.java
+++
b/src/kylin-server-it/src/test/java/org/apache/kylin/rest/controller/NBuildAndQueryMetricsTest.java
@@ -32,6 +32,9 @@ import org.apache.kylin.common.util.TempMetadataBuilder;
import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.engine.spark.job.NSparkCubingJob;
import org.apache.kylin.engine.spark.merger.AfterBuildResourceMerger;
+import org.apache.kylin.guava30.shaded.common.base.Preconditions;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.NExecutableManager;
@@ -77,10 +80,6 @@ import
org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
import lombok.val;
import lombok.var;
import lombok.extern.slf4j.Slf4j;
@@ -181,7 +180,7 @@ public class NBuildAndQueryMetricsTest extends
AbstractMVCIntegrationTestCase {
projectManager.updateProject(projectInstance,
projectInstanceUpdate.getName(),
projectInstanceUpdate.getDescription(),
projectInstanceUpdate.getOverrideKylinProps());
-Preconditions.checkArgument(projectInstance != null);
+Preconditions.checkNotNull(projectInstance);
for (String table : projectInstance.getTables()) {
if (!"DEFAULT.TEST_KYLIN_FACT".equals(table) &&
!"DEFAULT.TEST_ACCOUNT".equals(table)) {
@@ -254,6 +253,12 @@ public class NBuildAndQueryMetricsTest extends
AbstractMVCIntegrationTestCase {
assertMetric(sql, 30);
}
+@Test
+public void testSplitFunction() throws Exception {
+String sql = "select split(account_id, '-')[0] from test_account limit
30";
+assertMetric(sql, 30);
+}
+
@Test
public void testMetricsScanForTableIndex() throws Exception {
String sql = "select count(distinct case when trans_id > 100 then
order_id else 0 end),"
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index 0aa304edf6..081edc0d0a 100644
---
a/src/query/src/main/java/org/apache/kylin/q
(kylin) branch kylin5 updated (bcd4220ef2 -> ad52aada8f)
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from bcd4220ef2 update spark version to 3.2.0-kylin-4.6.12.0
new d977ce7bfc KYLIN-5757 No way to deselect query objects after selecting
all of them
new 71a7a52d66 KYLIN-5763 Fix query history model filter
new 046c97f54c KYLIN-5759 Log governance of job building log
new cd09148868 KYLIN-5764 Sum expression supports if condition
new b7615c56bb KYLIN-5765 Optimization of registering udf operators
new a8ad269b3c KYLIN-5766 Normalize query cache key
new d3b4469484 KYLIN-5767 Calculate total rows wrongly when connecting
jdbc datasource
new 180ff07afe KYLIN-5745 Using a global thread pool to clean underlying
storages
new 60da9e0ce1 KYLIN-5768 Refine and adjust some code for localCache
new 738cf4af01 KYLIN-5769 Support split function
new 3dd581ecfa KYLIN-5770 Upgrade spring-security-config from 5.7.8 from
5.7.10
new 64b52dea66 KYLIN-5771 Query cannot be interrupted during segment
pruning
new 90aa1def3d KYLIN-5772 Fix vulnerability & excluding okio
new ad52aada8f KYLIN-5773 update spark version to 3.2.0-kylin-4.6.13.0
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pom.xml| 26 +-
.../rest/controller/NBasicControllerTest.java | 2 +-
.../apache/kylin/rest/config/AppInitializer.java | 7 +
.../org/apache/kylin/rest/request/SQLRequest.java | 4 +-
.../kylin/rest/service/AsyncTaskService.java | 9 +-
.../apache/kylin/rest/service/ProjectService.java | 9 +-
.../org/apache/kylin/common/KylinConfigBase.java | 16 +
.../apache/kylin/common/constant/LogConstant.java | 1 +
.../kylin/common/logging/SetLogCategory.java | 22 +-
.../kylin/common/logging/SetLogCategoryTest.java | 131 +
.../kylin/job/execution/ExecutableThread.java | 3 +-
.../org/apache/kylin/job/manager/JobManager.java | 30 +-
.../runners/AbstractDefaultSchedulerRunner.java| 3 +-
.../org/apache/kylin/job/runners/JobRunner.java| 3 +-
.../kylin/metadata/cube/model/IndexPlan.java | 136 ++---
.../cube/optimization/AbstractOptStrategy.java | 4 +-
.../cube/optimization/GarbageLayoutType.java | 2 +-
.../optimization/IncludedLayoutOptStrategy.java| 2 +-
.../metadata/cube/optimization/IndexOptimizer.java | 14 +-
.../cube/optimization/IndexOptimizerFactory.java | 12 +-
.../optimization/LowFreqLayoutOptStrategy.java | 2 +-
.../cube/optimization/MergedLayoutOptStrategy.java | 59
.../optimization/SimilarLayoutOptStrategy.java | 2 +-
.../cube/optimization/event/ApproveRecsEvent.java} | 13 +-
.../cube/optimization/event/BuildIndexEvent.java} | 11 +-
.../cube/storage/GarbageStorageCollector.java | 2 +-
.../metadata/cube/utils/IndexPlanReduceUtil.java | 52 +++-
.../apache/kylin/metadata/datatype/DataType.java | 2 +-
.../metadata/query/JdbcQueryHistoryStore.java | 30 +-
.../kylin/metadata/query/QueryHistoryDAO.java | 8 +-
.../kylin/metadata/query/QueryHistoryRequest.java | 2 +
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 17 +-
.../kylin/query/util/QueryInterruptChecker.java| 53 +++-
.../apache/kylin/query/util/SlowQueryDetector.java | 2 +-
.../storage/ProjectStorageInfoCollectorTest.java | 74 -
.../cube/utils/IndexPlanReduceUtilTest.java| 27 +-
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 44 ++-
.../kylin/rest/initialize/BuildAppInitializer.java | 5 +
.../rest/scheduler/AbstractSchedulerRunnable.java | 3 +-
.../kylin/rest/service/ModelBuildService.java | 38 ++-
.../scheduler/AbstractSchedulerRunnableTest.java | 3 +-
.../kylin/rest/service/ModelServiceBuildTest.java | 36 ++-
.../apache/kylin/rest/service/ScheduleService.java | 10 +-
.../query/routing/PartitionPruningRuleTest.java| 313
.../query/routing/SegmentPruningRuleTest.java | 277 ++
.../kylin/query/engine/SumExprPlannerTest.xml | 63
.../sql_sum_expr/{query00.sql => query00-if.sql} | 8 +-
.../rest/controller/NBuildAndQueryMetricsTest.java | 15 +-
.../kylin/rest/controller/NProjectController.java | 14 +-
.../rest/controller/NProjectControllerTest.java| 2 +-
.../kylin/rest/service/AbstractModelService.java | 114 +++-
.../kylin/rest/service/BaseIndexUpdateHelper.java | 17 +-
.../kylin/rest/service/IndexPlanService.java | 66 +++--
.../kylin/rest/service/MetaStoreService.java | 10 +-
.../apache/kylin/rest/service/ModelService.java| 33
(kylin) 07/14: KYLIN-5767 Calculate total rows wrongly when connecting jdbc datasource
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d3b4469484a79102071db26114f6602b30d70f43
Author: Liang.Hua <36814772+jacob...@users.noreply.github.com>
AuthorDate: Fri Jul 21 16:31:39 2023 +0800
KYLIN-5767 Calculate total rows wrongly when connecting jdbc datasource
-
Co-authored-by: liang.hua
---
.../kylin/engine/spark/NSparkCubingEngine.java | 4
.../engine/spark/builder/SnapshotBuilder.scala | 7 +--
.../engine/spark/job/NSparkCubingJobTest.java | 8
.../kylin/source/jdbc/DefaultSourceConnector.java | 5 +
.../apache/kylin/source/jdbc/ISourceConnector.java | 3 +++
.../apache/kylin/source/jdbc/JdbcSourceInput.java | 11 ++
.../kylin/source/jdbc/JdbcSourceInputTest.java | 24 ++
7 files changed, 60 insertions(+), 2 deletions(-)
diff --git
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/NSparkCubingEngine.java
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/NSparkCubingEngine.java
index 00078a86ad..76a2bcb9ff 100644
---
a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/NSparkCubingEngine.java
+++
b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/NSparkCubingEngine.java
@@ -50,6 +50,10 @@ public class NSparkCubingEngine implements NCubingEngine {
* @return the Dataset, its schema consists of table column's
name, for example, [column1,column2,column3]
*/
Dataset getSourceData(TableDesc table, SparkSession ss,
Map parameters);
+
+default Long getSourceDataCount(TableDesc table, SparkSession ss,
Map parameters) {
+return getSourceData(table, ss, parameters).count();
+}
}
public interface NSparkCubingStorage {
diff --git
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
index e217a31c99..1a9ef63544 100644
---
a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
+++
b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
@@ -199,8 +199,11 @@ class SnapshotBuilder(var jobId: String) extends Logging
with Serializable {
case e: Throwable => logWarning(s"Calculate table
${tableDesc.getIdentity}'s total rows exception", e)
}
logInfo(s"Calculate table ${tableDesc.getIdentity}'s total rows from
source data")
-val sourceData = getSourceData(ss, tableDesc)
-val totalRows = sourceData.count()
+val params = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv)
+ .getProject(tableDesc.getProject).getLegalOverrideKylinProps
+val totalRows = SourceFactory
+ .createEngineAdapter(tableDesc,
classOf[NSparkCubingEngine.NSparkCubingSource])
+ .getSourceDataCount(tableDesc, ss, params)
logInfo(s"Table ${tableDesc.getIdentity}'s total rows is ${totalRows}'")
totalRows
}
diff --git
a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkCubingJobTest.java
b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkCubingJobTest.java
index b86eb333f3..2078e44056 100644
---
a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkCubingJobTest.java
+++
b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkCubingJobTest.java
@@ -163,6 +163,14 @@ public class NSparkCubingJobTest extends
NLocalWithSparkSessionTest {
getLookTables(df).forEach(table ->
Assert.assertNotNull(table.getLastSnapshotPath()));
}
+@Test
+public void testCalculateTableTotalRows() {
+NTableMetadataManager tableMetadataManager =
NTableMetadataManager.getInstance(config, getProject());
+TableDesc tableDesc =
tableMetadataManager.getTableDesc("DEFAULT.TEST_ORDER");
+long totalRows = new SnapshotBuilder().calculateTableTotalRows(null,
tableDesc, ss);
+Assert.assertEquals(5000, totalRows);
+}
+
private Set getLookTables(NDataflow df) {
return
df.getModel().getLookupTables().stream().map(TableRef::getTableDesc).collect(Collectors.toSet());
}
diff --git
a/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/DefaultSourceConnector.java
b/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/DefaultSourceConnector.java
index 1ea35704ed..1723810955 100644
---
a/src/spark-project/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/DefaultSourceConnector.java
+++
b/src/spark-project/source-jdbc/src/main/java/org/
(kylin) 06/14: KYLIN-5766 Normalize query cache key
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a8ad269b3c37d8670f0c5205ecef9d60a2796df3
Author: lixiang <447399...@qq.com>
AuthorDate: Fri Jul 21 15:29:49 2023 +0800
KYLIN-5766 Normalize query cache key
---
.../org/apache/kylin/rest/request/SQLRequest.java | 4 +-
.../apache/kylin/query/util/WhiteSpaceParser.jj| 887 +
.../apache/kylin/rest/service/QueryService.java| 6 +
.../kylin/query/util/WhiteSpaceParserTest.java | 110 +++
4 files changed, 1006 insertions(+), 1 deletion(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
b/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 58d9a82596..2bf34c75c5 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -77,13 +77,15 @@ public class SQLRequest implements Serializable,
ProjectInsensitiveRequest, Vali
@Size(max = 256)
private String user_defined_tag;
+private String normalizedSql;
+
protected volatile Object cacheKey = null;
public Object getCacheKey() {
if (cacheKey != null)
return cacheKey;
-cacheKey = Lists.newArrayList(sql.replaceAll("[ ]", " "), //
+cacheKey = Lists.newArrayList(normalizedSql, //
project, //
offset, //
limit, //
diff --git
a/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/WhiteSpaceParser.jj
b/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/WhiteSpaceParser.jj
new file mode 100644
index 00..c07936792a
--- /dev/null
+++
b/src/query-common/src/main/codegen/javacc/org/apache/kylin/query/util/WhiteSpaceParser.jj
@@ -0,0 +1,887 @@
+options {
+STATIC = false;
+IGNORE_CASE = true;
+UNICODE_INPUT = true;
+}
+
+PARSER_BEGIN(WhiteSpaceParser)
+package org.apache.kylin.query.util;
+
+import java.io.StringReader;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Locale;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Scanner;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class WhiteSpaceParser {
+
+private static final String CEIL_FLOOR_EXCEPTION_MSG = "%s(exp to
timeunit) encounters timeunit %s, but only supports %s";
+private static final String EXTRACT_FROM_EXCEPTION_MSG = "extract(timeunit
from exp) encounters timeunit %s, but only supports %s";
+private static final String TIMESTAMP_DIFF_ADD_EXCEPTION_MSG =
"%s(timeunit, exp1, exp2) encounters timeunit %s, but only supports %s";
+private static final String COMMA_STR = ",";
+private static final String SPACE_STR = " ";
+private static final char SPACE_CHAR = ' ';
+private static final String QUOTE_STR = "'";
+private static final Set OPERATORS = new
HashSet(Arrays.asList("+", "-", "*", "/", "%", "=",">=", "<=", "!=", "<>",
"||"));
+
+private EscapeDialect dialect = EscapeDialect.DEFAULT;
+
+ public static void main(String[] args) throws Exception {
+String sql = "select * col1,\ncol2,col3 from \n tableA \\r
a left \t\n table\rb ( select * from t) ";
+WhiteSpaceParser whiteSpaceParser = new WhiteSpaceParser(new
StringReader(sql));
+System.out.println("transformed sql is ：" + whiteSpaceParser.parse());
+ }
+
+private static String normalize(String str) {
+ if (str.equals(COMMA_STR)) {
+str = str + SPACE_STR;
+ } else if (OPERATORS.contains(str)) {
+str = SPACE_STR + str + SPACE_STR;
+ }
+ return str;
+}
+
+ public WhiteSpaceParser(EscapeDialect newDialect, String SQL) {
+this(new StringReader(SQL));
+this.dialect = newDialect;
+ }
+
+private static boolean needDropPreSpace(String str) {
+return COMMA_STR.equals(str) || OPERATORS.contains(str)
+|| (StringUtils.isBlank(str) && !SPACE_STR.equals(str));
+}
+
+ private static boolean skipSpace(String str) {
+ return COMMA_STR.equals(str) || OPERATORS.contains(str);
+ }
+
+ private static void appendExpression(StringBuilder sb, String innerString) {
+ if (sb.length() != 0) {
+ int last = sb.length() - 1;
+ if (sb.charAt(last) != SPACE_CHAR &&
innerStri
(kylin) 05/14: KYLIN-5765 Optimization of registering udf operators
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b7615c56bb618fc1133d156d12df9fd00ea04a2d
Author: Mingming Ge <7mmi...@gmail.com>
AuthorDate: Mon Jul 10 14:10:51 2023 +0800
KYLIN-5765 Optimization of registering udf operators
---
.../kylin/query/util/RexToTblColRefTranslator.java | 31 +-
1 file changed, 19 insertions(+), 12 deletions(-)
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/util/RexToTblColRefTranslator.java
b/src/query-common/src/main/java/org/apache/kylin/query/util/RexToTblColRefTranslator.java
index 9ce69bc03d..bc5d78109a 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/util/RexToTblColRefTranslator.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/util/RexToTblColRefTranslator.java
@@ -287,6 +287,8 @@ public class RexToTblColRefTranslator {
private final Map rexToSqlMap;
+private static final Set REGISTERED_UDFS = registerUdfs();
+
public OLAPRexSqlStandardConvertletTable(RexCall call, Map rexToSqlMap) {
super();
this.rexToSqlMap = rexToSqlMap;
@@ -303,18 +305,7 @@ public class RexToTblColRefTranslator {
}
private void registerUdfOperator(RexCall call) {
-Set udfs = Sets.newHashSet();
-KylinConfig.getInstanceFromEnv().getUDFs().forEach((key, value) ->
{
-try {
-val methods = Class.forName(value).getMethods();
-for (Method method : methods) {
-udfs.add(method.getName().toLowerCase(Locale.ROOT));
-}
-} catch (Exception e) {
-log.error("registerUdfOperator not found method for :", e);
-}
-});
-if
(udfs.contains(call.getOperator().toString().toLowerCase(Locale.ROOT))) {
+if
(REGISTERED_UDFS.contains(call.getOperator().toString().toLowerCase(Locale.ROOT)))
{
SqlOperator operator = call.getOperator();
this.registerEquivOp(operator);
}
@@ -338,6 +329,22 @@ public class RexToTblColRefTranslator {
}
}
+private static Set registerUdfs() {
+Set udfs = Sets.newHashSet();
+KylinConfig.getInstanceFromEnv().getUDFs().forEach((key, value) ->
{
+try {
+val methods = Class.forName(value).getMethods();
+for (Method method : methods) {
+udfs.add(method.getName().toLowerCase(Locale.ROOT));
+}
+} catch (Exception e) {
+log.error("registerUdfOperator not found method for :", e);
+}
+});
+
+return udfs;
+}
+
private Map initTimeUnitFunctionMap() {
Map rst = Maps.newHashMap();
rst.putIfAbsent(TimeUnit.YEAR, SqlStdOperatorTable.YEAR);
(kylin) 03/14: KYLIN-5759 Log governance of job building log
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 046c97f54c5fa582f8983c794bf0d5a4a0cfd15e
Author: jlf
AuthorDate: Thu Jul 6 16:10:53 2023 +0800
KYLIN-5759 Log governance of job building log
1. move build log in kylin.build.log
2. add LogConstant.SMART_CATEGORY
---
.../apache/kylin/common/constant/LogConstant.java | 1 +
.../kylin/common/logging/SetLogCategory.java | 22 +++-
.../kylin/common/logging/SetLogCategoryTest.java | 131 +
.../kylin/job/execution/ExecutableThread.java | 3 +-
.../org/apache/kylin/job/manager/JobManager.java | 30 +++--
.../runners/AbstractDefaultSchedulerRunner.java| 3 +-
.../org/apache/kylin/job/runners/JobRunner.java| 3 +-
.../rest/scheduler/AbstractSchedulerRunnable.java | 3 +-
.../scheduler/AbstractSchedulerRunnableTest.java | 3 +-
.../kylin/query/routing/RealizationChooser.java| 3 +-
10 files changed, 180 insertions(+), 22 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/constant/LogConstant.java
b/src/core-common/src/main/java/org/apache/kylin/common/constant/LogConstant.java
index 4a5293dffa..5fd9aae424 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/constant/LogConstant.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/constant/LogConstant.java
@@ -27,4 +27,5 @@ public class LogConstant {
public static final String METADATA_CATEGORY = "metadata";
public static final String QUERY_CATEGORY = "query";
public static final String BUILD_CATEGORY = "build";
+public static final String SMART_CATEGORY = "smart";
}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/logging/SetLogCategory.java
b/src/core-common/src/main/java/org/apache/kylin/common/logging/SetLogCategory.java
index eb04aa9414..284380c8c4 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/logging/SetLogCategory.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/logging/SetLogCategory.java
@@ -18,19 +18,35 @@
package org.apache.kylin.common.logging;
import java.io.Closeable;
+import java.util.Deque;
+import java.util.LinkedList;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.ThreadContext;
+import org.springframework.util.CollectionUtils;
public class SetLogCategory implements Closeable {
-private static final String KEY = "logCategory";
+private static final String LOG_CATEGORY = "logCategory";
+private static final ThreadLocal> categoryThreadLocal =
ThreadLocal.withInitial(LinkedList::new);
+// when category exist, and use new category to logger message, will push
oldSetLogCategory
public SetLogCategory(String category) {
-ThreadContext.put(KEY, category);
+String oldCategory = ThreadContext.get(LOG_CATEGORY);
+ThreadContext.put(LOG_CATEGORY, category);
+if (StringUtils.isNotBlank(oldCategory)) {
+categoryThreadLocal.get().offerFirst(oldCategory);
+}
}
@Override
public void close() {
-ThreadContext.remove(KEY);
+ThreadContext.remove(LOG_CATEGORY);
+if (!CollectionUtils.isEmpty(categoryThreadLocal.get())) {
+String oldCategory = categoryThreadLocal.get().pollFirst();
+if (StringUtils.isNotBlank(oldCategory)) {
+ThreadContext.put(LOG_CATEGORY, oldCategory);
+}
+}
}
}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/logging/SetLogCategoryTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/logging/SetLogCategoryTest.java
new file mode 100644
index 00..b447cf65a8
--- /dev/null
+++
b/src/core-common/src/test/java/org/apache/kylin/common/logging/SetLogCategoryTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.logging;
+
+import static org.awaitility.Awaitility.
(kylin) 14/14: KYLIN-5773 update spark version to 3.2.0-kylin-4.6.13.0
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit ad52aada8fb7a5b1620b076680ea48bfecb5e919 Author: huangsheng AuthorDate: Fri Jul 28 15:46:00 2023 +0800 KYLIN-5773 update spark version to 3.2.0-kylin-4.6.13.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 765cbefa09..8f969b9a14 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 2.0.2 -3.2.0-kylin-4.6.12.0 +3.2.0-kylin-4.6.13.0 0.9.2-kylin-r4
(kylin) 02/14: KYLIN-5763 Fix query history model filter
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 71a7a52d669f3c33b6efd7f9bbc80a5b1dfd894a
Author: lixiang <447399...@qq.com>
AuthorDate: Wed Jul 5 19:47:38 2023 +0800
KYLIN-5763 Fix query history model filter
---
.../metadata/query/JdbcQueryHistoryStore.java | 13 +-
.../kylin/metadata/query/QueryHistoryDAO.java | 2 +-
.../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 4 +-
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 5 +--
.../kylin/rest/controller/NQueryController.java| 37 +++
.../rest/controller/NQueryControllerTest.java | 20 ++---
.../rest/response/QueryHistoryFiltersResponse.java | 49
.../kylin/rest/service/QueryHistoryService.java| 52 +++---
.../rest/service/QueryHistoryServiceTest.java | 26 +++
9 files changed, 131 insertions(+), 77 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index da5b152073..6ffbd9eac0 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -214,7 +214,7 @@ public class JdbcQueryHistoryStore {
}
}
-public List
queryQueryHistoriesModelIds(QueryHistoryRequest request, int size) {
+public List
queryQueryHistoriesModelIds(QueryHistoryRequest request) {
try (SqlSession session = sqlSessionFactory.openSession()) {
QueryStatisticsMapper mapper =
session.getMapper(QueryStatisticsMapper.class);
SelectStatementProvider statementProvider1 =
selectDistinct(queryHistoryTable.engineType)
@@ -222,16 +222,7 @@ public class JdbcQueryHistoryStore {
.where(queryHistoryTable.engineType,
isNotEqualTo("NATIVE")) //
.and(queryHistoryTable.projectName,
isEqualTo(request.getProject())) //
.build().render(RenderingStrategies.MYBATIS3);
-List engineTypes =
mapper.selectMany(statementProvider1);
-
-SelectStatementProvider statementProvider2 =
selectDistinct(queryHistoryRealizationTable.model)
-.from(queryHistoryRealizationTable) //
-.where(queryHistoryRealizationTable.projectName,
isEqualTo(request.getProject())) //
-.limit(size) //
-.build().render(RenderingStrategies.MYBATIS3);
-List modelIds =
mapper.selectMany(statementProvider2);
-engineTypes.addAll(modelIds);
-return engineTypes;
+return mapper.selectMany(statementProvider1);
}
}
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
index 882dc78477..66f1d4aa22 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java
@@ -63,7 +63,7 @@ public interface QueryHistoryDAO {
List getQueryHistoriesSubmitters(QueryHistoryRequest
request, int size);
-List getQueryHistoriesModelIds(QueryHistoryRequest
request, int size);
+List getQueryHistoriesModelIds(QueryHistoryRequest
request);
String getRealizationMetricMeasurement();
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
index 64d874d1e6..3592dbec22 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java
@@ -184,8 +184,8 @@ public class RDBMSQueryHistoryDAO implements
QueryHistoryDAO {
return jdbcQueryHisStore.queryQueryHistoriesSubmitters(request, size);
}
-public List getQueryHistoriesModelIds(QueryHistoryRequest
request, int size) {
-return jdbcQueryHisStore.queryQueryHistoriesModelIds(request, size);
+public List getQueryHistoriesModelIds(QueryHistoryRequest
request) {
+return jdbcQueryHisStore.queryQueryHistoriesModelIds(request);
}
public QueryStatistics getQueryCountAndAvgDuration(long startTime, long
endTime, String project) {
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
index 2651f64b81..30cd2d37d0 100644
---
a/src/core-metadata/src/test
(kylin) 01/14: KYLIN-5757 No way to deselect query objects after selecting all of them
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d977ce7bfc9a244d7dbb568ba75a58ab29648908
Author: Hang Jia <754332...@qq.com>
AuthorDate: Wed Jul 5 18:09:55 2023 +0800
KYLIN-5757 No way to deselect query objects after selecting all of them
---
.../metadata/query/JdbcQueryHistoryStore.java | 17 --
.../kylin/metadata/query/QueryHistoryRequest.java | 2 +
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 37 +
.../kylin/rest/controller/NQueryController.java| 12 +++--
.../rest/controller/NQueryControllerTest.java | 2 +-
.../kylin/rest/service/QueryHistoryService.java| 8 +++
.../rest/service/QueryHistoryServiceTest.java | 61 ++
7 files changed, 132 insertions(+), 7 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index 6a696c0fdd..da5b152073 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -29,6 +29,7 @@ import static
org.mybatis.dynamic.sql.SqlBuilder.isLessThanOrEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.isLike;
import static org.mybatis.dynamic.sql.SqlBuilder.isLikeCaseInsensitive;
import static org.mybatis.dynamic.sql.SqlBuilder.isNotEqualTo;
+import static org.mybatis.dynamic.sql.SqlBuilder.isNotIn;
import static org.mybatis.dynamic.sql.SqlBuilder.max;
import static org.mybatis.dynamic.sql.SqlBuilder.or;
import static org.mybatis.dynamic.sql.SqlBuilder.select;
@@ -50,6 +51,7 @@ import java.util.stream.Collectors;
import javax.sql.DataSource;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.jdbc.ScriptRunner;
import org.apache.ibatis.session.ExecutorType;
@@ -743,9 +745,18 @@ public class JdbcQueryHistoryStore {
filterSql = filterSql.and(queryHistoryTable.indexHit,
isEqualTo(false));
}
} else if (selectAllModels) {
-// Process CONSTANTS, HIVE, RDBMS and all model
-filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
-or(queryHistoryTable.indexHit, isEqualTo(true)));
+if (!CollectionUtils.isEmpty(request.getExcludeFilterModelIds())) {
+// Process CONSTANTS, HIVE, RDBMS and not in model1, model2,
model3...
+filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
+or(queryHistoryTable.queryId,
+
isIn(selectDistinct(queryHistoryRealizationTable.queryId)
+
.from(queryHistoryRealizationTable).where(queryHistoryRealizationTable.model,
+
isNotIn(request.getExcludeFilterModelIds());
+} else {
+// Process CONSTANTS, HIVE, RDBMS and all model
+filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
+or(queryHistoryTable.indexHit, isEqualTo(true)));
+}
} else if (request.getFilterModelIds() != null &&
!request.getFilterModelIds().isEmpty()) {
// Process CONSTANTS, HIVE, RDBMS and model1, model2, model3...
filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
index d406f0864d..a5e10c3e07 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
@@ -52,6 +52,8 @@ public class QueryHistoryRequest {
private List queryStatus;
List realizations;
+List excludeRealization;
+private List excludeFilterModelIds;
public QueryHistoryRequest(String project, String startTimeFrom, String
startTimeTo) {
this.project = project;
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
index b30e13d383..2651f64b81 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
@@ -782,6 +782,43 @@ public class RDBMSQueryHistoryDaoTest extends
NLocalFileMe
(kylin) 14/15: KYLIN-5746 model list get optimize fix
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3cbcead2ed5cb3046647ef2a6617c9480254b1b9
Author: liang.huang <83992752+lhuang09287...@users.noreply.github.com>
AuthorDate: Tue Jul 4 17:58:53 2023 +0800
KYLIN-5746 model list get optimize fix
---
.../apache/kylin/rest/service/ModelService.java| 22 --
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 1ad9ce6cd1..59cc0a9203 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -372,10 +372,10 @@ public class ModelService extends AbstractModelService
implements TableModelSupp
private void addOldSegmentParams(NDataModel model, NDataModelOldParams
oldParams,
List executables) {
-if (((model instanceof NDataModelResponse) && !(model instanceof
NDataModelLiteResponse)) || model.isFusionModel()) {
-List segments =
getSegmentsResponse(model.getId(), model.getProject(), "1",
-String.valueOf(Long.MAX_VALUE - 1), null, executables,
LAST_MODIFY, true);
-calculateRecordSizeAndCount(segments, oldParams);
+List segments =
getSegmentsResponse(model.getId(), model.getProject(), "1",
+String.valueOf(Long.MAX_VALUE - 1), null, executables,
LAST_MODIFY, true, (model instanceof NDataModelLiteResponse));
+calculateRecordSizeAndCount(segments, oldParams);
+if (model instanceof NDataModelResponse) {
((NDataModelResponse) model).setSegments(segments);
((NDataModelResponse) model).setHasSegments(
((NDataModelResponse) model).isHasSegments() ||
CollectionUtils.isNotEmpty(segments));
@@ -387,7 +387,7 @@ public class ModelService extends AbstractModelService
implements TableModelSupp
NDataModel batchModel = fusionModel.getBatchModel();
if (!batchModel.isBroken()) {
List batchSegments =
getSegmentsResponse(batchModel.getUuid(), model.getProject(),
-"1", String.valueOf(Long.MAX_VALUE - 1), null,
executables, LAST_MODIFY, true);
+"1", String.valueOf(Long.MAX_VALUE - 1), null,
executables, LAST_MODIFY, true, false);
calculateRecordSizeAndCount(batchSegments, oldParams);
if (model instanceof FusionModelResponse) {
((FusionModelResponse)
model).setBatchSegments(batchSegments);
@@ -925,9 +925,9 @@ public class ModelService extends AbstractModelService
implements TableModelSupp
}
public List getSegmentsResponse(String modelId,
String project, String start, String end,
-String status, List executables, String
sortBy, boolean reverse) {
+String status, List executables, String
sortBy, boolean reverse, boolean lite) {
return getSegmentsResponse(modelId, project, start, end, status, null,
null, executables, false, sortBy,
-reverse, null, null);
+reverse, lite, null, null);
}
private List getAllRunningExecutable(String project) {
@@ -949,12 +949,12 @@ public class ModelService extends AbstractModelService
implements TableModelSupp
String sortBy, boolean reverse, List statuses,
List secondStorageStatuses) {
val executables = getPartialRunningExecutable(project, modelId);
return getSegmentsResponse(modelId, project, start, end, status,
withAllIndexes, withoutAnyIndexes, executables,
-allToComplement, sortBy, reverse, statuses,
secondStorageStatuses);
+allToComplement, sortBy, reverse, false, statuses,
secondStorageStatuses);
}
public List getSegmentsResponse(String modelId,
String project, String start, String end,
String status, Collection withAllIndexes, Collection
withoutAnyIndexes,
-List executables, boolean allToComplement,
String sortBy, boolean reverse,
+List executables, boolean allToComplement,
String sortBy, boolean reverse, boolean lite,
List statuses, List secondStorageStatuses) {
aclEvaluate.checkProjectReadPermission(project);
NDataflowManager dataflowManager = getManager(NDataflowManager.class,
project);
@@ -962,7 +962,9 @@ public class ModelService extends AbstractModelService
implements TableModelSupp
List segmentResponseList =
getSegmentsResponseCore(modelId, project, start, end, status,
withAllIndexes, withoutAnyIndexes, executables,
allToComplement, dataflo
(kylin) 08/15: KYLIN-5740 Equivalent aggregations matches index failed
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0e3354414d89c4b4b81d0ec1db8d39705e760a40
Author: huangsheng
AuthorDate: Fri Jun 30 12:10:57 2023 +0800
KYLIN-5740 Equivalent aggregations matches index failed
Equivalent aggregations matches index failed when using
the strategy of dimension as measure.
---
.../kylin/rest/service/AccessServiceTest.java | 4 +-
.../kylin/rest/service/AclTCRServiceTest.java | 2 +-
.../metadata/cube/cuboid/AggIndexMatcher.java | 13 ++-
.../metadata/project/NProjectManagerTest.java | 2 +-
.../kylin/metrics/HdfsCapacityMetricsTest.java | 2 +-
.../localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv | 3 +
.../metadata/_global/project/dim_as_measure.json | 6 ++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 63 ++
.../4409fb82-8f3a-ba01-9852-0b05f1fe690f.json | 64 ++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 64 ++
.../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 97 ++
.../table/DEFAULT.TEST_DIM_AS_MEASURE.json | 49 +++
.../newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json | 49 +++
.../query/routing/QueryLayoutChooserTest.java | 47 ++-
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/query/engine/AsyncQueryJobTest.java | 2 +-
17 files changed, 460 insertions(+), 15 deletions(-)
diff --git
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
index f5402d462c..769f1dd326 100644
---
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
+++
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
@@ -606,14 +606,14 @@ public class AccessServiceTest extends
NLocalFileMetadataTestCase {
@Test
public void testGetGrantedProjectsOfUser() throws IOException {
List result = accessService.getGrantedProjectsOfUser("ADMIN");
-assertEquals(29, result.size());
+assertEquals(30, result.size());
}
@Test
public void testGetGrantedProjectsOfUserOrGroup() throws IOException {
// admin user
List result =
accessService.getGrantedProjectsOfUserOrGroup("ADMIN", true);
-assertEquals(29, result.size());
+assertEquals(30, result.size());
// normal user
result = accessService.getGrantedProjectsOfUserOrGroup("ANALYST",
true);
diff --git
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
index 803019720d..9af9f92b14 100644
---
a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
+++
b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
@@ -1316,7 +1316,7 @@ public class AclTCRServiceTest extends
NLocalFileMetadataTestCase {
Mockito.when(userService.isGlobalAdmin("ADMIN")).thenReturn(true);
List responses =
accessService.getUserOrGroupAclPermissions(projects, "ADMIN",
true);
-Assert.assertEquals(29, responses.size());
+Assert.assertEquals(30, responses.size());
Assert.assertTrue(responses.stream().allMatch(response ->
"ADMIN".equals(response.getProjectPermission(;
// test normal group
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
index 5561581d72..a7efb17286 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/cuboid/AggIndexMatcher.java
@@ -89,7 +89,7 @@ public class AggIndexMatcher extends IndexMatcher {
return new MatchResult();
}
log.trace("Matching agg index");
-Set unmatchedMetrics =
Sets.newHashSet(sqlDigest.aggregations);
+Collection unmatchedMetrics =
Lists.newArrayList(sqlDigest.aggregations);
Set unmatchedCols = initUnmatchedColumnIds(layout);
final Map needDerive = Maps.newHashMap();
goThruDerivedDims(layout.getIndex(), needDerive, unmatchedCols);
@@ -140,9 +140,8 @@ public class AggIndexMatcher extends IndexMatcher {
functionDescs.addAll(cuboidLayout.getStreamingMeasures().values());
}
functionDescs.addAll(cuboidLayout.getOrderedMeasures().values());
-
for (MeasureDesc measureDesc :
(kylin) 12/15: KYLIN-5744 Answering snapshot first, then answering by metadata
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d13199db7407552226a58060ceb9b41a7f859c36
Author: Pengfei Zhan
AuthorDate: Thu Jun 29 15:28:35 2023 +0800
KYLIN-5744 Answering snapshot first, then answering by metadata
Hitting snapshot has a higher priority than
answering min/max with metadata.
Change the default behavior of routing query to metadata to true
1. Fix timezone problem.
Given the fact of min(cal_dt) = '2012-01-01',
the result of `select min(cal_dt) from test_kylin_fact`
may give '2011-12-31' rather than '2012-01-01'.
2. Fix min/max query hit a index without any dimensions.
For example: `select min(cal_dt) as min_cal_dt from test_kylin_fact `
hits index just has a measure of min(cal_dt),
the SparkPlan is wrong: ` project --- project --- tableScan `;
but the desired SparkPlan is: ` aggregate --- project --- tableScan `.
---
.../org/apache/kylin/common/KylinConfigBase.java | 2 +-
.../kylin/query/engine/QueryExecWithMetaTest.java | 2 +-
src/query-common/pom.xml | 4 ++
.../kylin/query/relnode/KapAggregateRel.java | 58 +-
.../apache/kylin/query/relnode/OLAPContext.java| 24 +
.../org/apache/spark/sql/SparderTypeUtil.scala | 2 +-
6 files changed, 46 insertions(+), 46 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 03c1c06e89..cbd82ed108 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1734,7 +1734,7 @@ public abstract class KylinConfigBase implements
Serializable {
//
public boolean isRouteToMetadataEnabled() {
-return
Boolean.parseBoolean(this.getOptional("kylin.query.using-metadata-answer-minmax-of-dimension",
FALSE));
+return
Boolean.parseBoolean(this.getOptional("kylin.query.using-metadata-answer-minmax-of-dimension",
TRUE));
}
public boolean partialMatchNonEquiJoins() {
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
index ac84de6f06..94d98eb5ea 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
@@ -230,7 +230,7 @@ public class QueryExecWithMetaTest extends
NLocalWithSparkSessionTest {
List result = iterator.next();
Assert.assertEquals("2147483648,21474836483289,2132,2147483647,-128,127,0,9,0.0,1.0,"
+
"0.3255242,85208.3241,10.,201.3235,abc,xyz,,x,abcd,,"
-+ "2000-12-31,2004-04-16,2004-04-01 00:00:00,2004-04-17
00:32:23.032,false,true,"
++ "2001-01-01,2004-04-17,2004-04-01 00:00:00,2004-04-17
00:32:23.032,false,true,"
+
"null,null,null,null,null,null,null,null,null,null,null,null", String.join(",",
result));
}
diff --git a/src/query-common/pom.xml b/src/query-common/pom.xml
index d6af343c34..16af9a8028 100644
--- a/src/query-common/pom.xml
+++ b/src/query-common/pom.xml
@@ -55,6 +55,10 @@
org.apache.kylin
kap-second-storage-core
+
+org.apache.kylin
+kylin-spark-common
+
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
index f44272beb5..5a064dd840 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
@@ -45,6 +45,9 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.measure.corr.CorrMeasureType;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.NDataflow;
@@ -55,10 +58,6 @@ import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.mod
(kylin) branch kylin5 updated (8118afb449 -> bcd4220ef2)
This is an automated email from the ASF dual-hosted git repository. liyang pushed a change to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git from 8118afb449 KYLIN-5683 tree list load more columns issue new 3ec3063135 KYLIN-5732 Upgrade snappy-java to fix vulnerabilities new 387e68d0a6 KYLIN-5733 TDS template modify to English new 6816947de4 KYLIN-5762 Initialize job scheduler encounters NPE new b15367e3e2 KYLIN-5760 Support matching model exactly for list jobs new d008c28156 KYLIN-5737 Fix vulnerabilities, upgrade tomcat-embed-core & netty (#30573) new c396134127 KYLIN-5742 Make the query result of duplicate group sets same as Spark new 1d665a8282 KYLIN-5762 Initialize job scheduler encounters NPE new 0e3354414d KYLIN-5740 Equivalent aggregations matches index failed new 7cd3fd0ecc KYLIN-5741 change all epoch when empty projects request new 5532198301 KYLIN-5743 optimization of SumExpression prevents ProjectJoinTransposeRule to hit AggIndex new 572d78a45b KYLIN-5756 Concat string and number with + gives unexpected result new d13199db74 KYLIN-5744 Answering snapshot first, then answering by metadata new 227e7a0683 KYLIN-5741 punctuation optimized new 3cbcead2ed KYLIN-5746 model list get optimize fix new bcd4220ef2 update spark version to 3.2.0-kylin-4.6.12.0 The 15 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../kap/secondstorage/metadata/TableFlow.java | 13 ++ pom.xml| 39 +++--- .../kylin/rest/controller/NEpochController.java| 9 +- .../rest/controller/NEpochControllerTest.java | 19 +-- .../apache/kylin/rest/KylinPrepareEnvListener.java | 14 ++- .../apache/kylin/rest/service/ProjectService.java | 8 ++ .../kylin/rest/service/AccessServiceTest.java | 4 +- .../kylin/rest/service/AclTCRServiceTest.java | 2 +- .../org/apache/kylin/common/KylinConfigBase.java | 14 ++- .../apache/kylin/common/constant/Constants.java| 1 + .../org/apache/kylin/common/msg/CnMessage.java | 6 + .../java/org/apache/kylin/common/msg/Message.java | 7 ++ .../kylin/job/execution/ExecutableContext.java | 9 +- .../kylin/job/execution/ExecutableThread.java | 1 + .../job/impl/threadpool/NDefaultScheduler.java | 21 ++-- .../apache/kylin/job/runners/FetcherRunner.java| 3 +- .../org/apache/kylin/job/runners/JobRunner.java| 3 +- .../kylin/job/execution/ExecutableContextTest.java | 4 +- .../job/execution/NExecutableManagerTest.java | 2 +- .../job/impl/threadpool/NDefaultSchedulerTest.java | 21 .../metadata/cube/cuboid/AggIndexMatcher.java | 13 +- .../metadata/project/NProjectManagerTest.java | 2 +- .../kylin/metrics/HdfsCapacityMetricsTest.java | 2 +- .../kylin/rest/controller/JobController.java | 4 +- .../kylin/rest/controller/v2/JobControllerV2.java | 2 +- .../kylin/rest/controller/JobControllerTest.java | 6 +- .../rest/controller/v2/JobControllerV2Test.java| 11 +- .../org/apache/kylin/rest/request/JobFilter.java | 2 + .../org/apache/kylin/rest/service/JobService.java | 5 + .../kylin/rest/service/DagJobServiceTest.java | 8 +- .../apache/kylin/rest/service/JobServiceTest.java | 49 ++-- .../localmeta/data/DEFAULT.TEST_DIM_AS_MEASURE.csv | 3 + .../metadata/_global/project/dim_as_measure.json} | 2 +- .../f29d1a92-c115-ec68-5575-9cfcc0d65890.json | 63 ++ .../4409fb82-8f3a-ba01-9852-0b05f1fe690f.json | 64 ++ .../f29d1a92-c115-ec68-5575-9cfcc0d65890.json} | 59 - .../f29d1a92-c115-ec68-5575-9cfcc0d65890.json} | 61 - .../table/DEFAULT.TEST_DIM_AS_MEASURE.json}| 40 +++--- .../newten/table/DEFAULT.TEST_DIM_AS_MEASURE.json} | 40 +++--- src/kylin-it/pom.xml | 9 -- .../kylin/query/engine/QueryExecWithMetaTest.java | 2 +- .../kylin/query/engine/SumExprPlannerTest.java | 11 ++ .../query/routing/QueryLayoutChooserTest.java | 47 ++- .../kylin/query/engine/SumExprPlannerTest.xml | 32 + src/kylin-server-it/pom.xml| 9 -- .../kylin/rest/service/MetaStoreService.java | 24 ++-- .../apache/kylin/rest/service/ModelService.java| 17 +-- .../bisync/tds/tableau.connector.template.xml | 2 +- .../main/resources/bisync/tds/tableau.template.xml | 2 +- .../kylin/rest/service/ProjectServiceTest.java | 32 - .../bisync_tableau/nmodel_basic_all_cols.tds | 2 +- .../bisync_tableau/nmodel_basic_inner_all_cols.tds | 2 +- .../nmodel_full_measure_test.connector.tds | 2 +- .../nmodel_full_measure_
(kylin) 10/15: KYLIN-5743 optimization of SumExpression prevents ProjectJoinTransposeRule to hit AggIndex
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 553219830193ec43f7b6d68fe626b04456633120
Author: Pengfei Zhan
AuthorDate: Thu Jun 29 14:39:05 2023 +0800
KYLIN-5743 optimization of SumExpression prevents ProjectJoinTransposeRule
to hit AggIndex
---
.../kylin/query/engine/SumExprPlannerTest.java | 11
.../kylin/query/engine/SumExprPlannerTest.xml | 32 ++
.../apache/kylin/query/engine/PlannerFactory.java | 14 ++
3 files changed, 51 insertions(+), 6 deletions(-)
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
index 7e00b6cfb2..9692ec4f4e 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
@@ -199,6 +199,17 @@ public class SumExprPlannerTest extends
CalciteRuleTestBase {
KapJoinRule.INSTANCE, KapProjectRule.INSTANCE));
}
+@Test
+public void testSumExpressionSkipProjectJoinTransposeRule() {
+String sql = "select sum( case when LO_COMMITDATE = '20230501' then
LO_DISCOUNT end )\n" //
++ "from (\n" //
++ "select LO_COMMITDATE, LO_DISCOUNT,
P_LINEORDER.LO_ORDERDATE from ssb.P_LINEORDER\n" //
++ " ) a\n" //
++ "where LO_COMMITDATE = ( select max(LO_COMMITDATE) from
ssb.P_LINEORDER )";
+openSumCaseWhen();
+checkSQL("ssb", sql, null, null);
+}
+
@Test
public void testAggPushdownWithCrossJoin1() {
diff --git
a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
index e7b560f9a5..0796314ef7 100644
---
a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
+++
b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
@@ -1377,6 +1377,38 @@ KapOLAPToEnumerableConverter
]]>
+
+
+
+
+
+
+
+
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
b/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
index 8616d6e14a..bac1fbf81b 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/PlannerFactory.java
@@ -65,10 +65,13 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.guava30.shaded.common.base.Function;
+import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.query.engine.meta.PlannerContext;
+import org.apache.kylin.query.relnode.OLAPContext;
+
import io.kyligence.kap.query.optrule.AggregateMultipleExpandRule;
import io.kyligence.kap.query.optrule.AggregateProjectReduceRule;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.engine.meta.PlannerContext;
import io.kyligence.kap.query.optrule.CorrReduceFunctionRule;
import io.kyligence.kap.query.optrule.KAPValuesRule;
import io.kyligence.kap.query.optrule.KapAggregateReduceFunctionsRule;
@@ -89,9 +92,6 @@ import io.kyligence.kap.query.optrule.KapWindowRule;
import io.kyligence.kap.query.optrule.RightJoinToLeftJoinRule;
import io.kyligence.kap.query.optrule.SumConstantConvertRule;
-import org.apache.kylin.guava30.shaded.common.base.Function;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
-
/**
* factory that create optimizers and register opt rules
* TODO sort and register only necessary rules
@@ -240,7 +240,9 @@ public class PlannerFactory {
// UnionMergeRule may slow volcano planner optimization on large
number of union clause
planner.removeRule(UnionMergeRule.INSTANCE);
-planner.addRule(KapProjectJoinTransposeRule.INSTANCE);
+if (!kylinConfig.isConvertSumExpressionEnabled()) {
+planner.addRule(KapProjectJoinTransposeRule.INSTANCE);
+}
planner.removeRule(ProjectRemoveRule.INSTANCE);
// skip corr expandsion during model suggestion
(kylin) 11/15: KYLIN-5756 Concat string and number with + gives unexpected result
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 572d78a45b6e0cabb248b6772a9b434221d9f323
Author: Zhiting Guo <35057824+fre...@users.noreply.github.com>
AuthorDate: Thu Jun 29 18:00:09 2023 +0800
KYLIN-5756 Concat string and number with + gives unexpected result
KYLIN-5747 duplicated
Co-authored-by: Zhiting Guo
---
pom.xml| 2 +-
.../org/apache/kylin/rest/KylinPrepareEnvListener.java | 14 ++
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 12
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/pom.xml b/pom.xml
index 83adcd3e33..8bfc1dd630 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
0.3.0
-1.116.0-kylin-4.x-r031
+1.116.0-kylin-4.x-r033
4.x_1.10-r01
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index e700fb0801..a49c67e33b 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -24,6 +24,7 @@ import java.sql.SQLException;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.TempMetadataBuilder;
import org.apache.kylin.common.util.TimeZoneUtils;
@@ -75,14 +76,19 @@ public class KylinPrepareEnvListener implements
EnvironmentPostProcessor, Ordere
Unsafe.setProperty("needCheckCC", "true");
val config = KylinConfig.getInstanceFromEnv();
if (config.isCalciteInClauseEnabled()) {
-Unsafe.setProperty("calcite.keep-in-clause", "true");
+Unsafe.setProperty("calcite.keep-in-clause", KylinConfigBase.TRUE);
} else {
-Unsafe.setProperty("calcite.keep-in-clause", "false");
+Unsafe.setProperty("calcite.keep-in-clause",
KylinConfigBase.FALSE);
}
if (config.isCalciteConvertMultipleColumnsIntoOrEnabled()) {
-Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or",
"true");
+Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or",
KylinConfigBase.TRUE);
} else {
-Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or",
"false");
+Unsafe.setProperty("calcite.convert-multiple-columns-in-to-or",
KylinConfigBase.FALSE);
+}
+if (config.isCalciteCompatibleWithMsSqlPlusEnabled()) {
+Unsafe.setProperty("calcite.compatible-with-mssql-plus",
KylinConfigBase.TRUE);
+} else {
+Unsafe.setProperty("calcite.compatible-with-mssql-plus",
KylinConfigBase.FALSE);
}
Unsafe.setProperty("calcite.bindable.cache.maxSize",
Integer.toString(config.getCalciteBindableCacheSize()));
Unsafe.setProperty("calcite.bindable.cache.concurrencyLevel",
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2908f05571..03c1c06e89 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2533,6 +2533,10 @@ public abstract class KylinConfigBase implements
Serializable {
return
Integer.parseInt(getOptional("kylin.query.calcite.bindable.cache.concurrencyLevel",
"5"));
}
+public boolean isCalciteCompatibleWithMsSqlPlusEnabled() {
+return
Boolean.parseBoolean(getOptional("kylin.query.calcite.compatible-with-mssql-plus-enabled",
FALSE));
+}
+
public int getEventPollIntervalSecond() {
return
Integer.parseInt(getOptional("kylin.job.event.poll-interval-second", "60"));
}
@@ -2823,6 +2827,14 @@ public abstract class KylinConfigBase implements
Serializable {
return
Boolean.parseBoolean(getOptional("kylin.smart.conf.skip-corr-reduce-rule",
FALSE));
}
+public int getOptimizeTransformerMaxIterations() {
+return
Integer.parseInt(getOptional("kylin.smart.conf.optimize-transformer-max-iterations",
"2"));
+}
+
+public int getOptimizeTransformerConditionCountThreshold() {
+return
Integer.parseInt(getOptional("kylin.smart.conf.optimize-transformer-condition-count-threshold",
"200"));
+}
+
public String getEngineWriteFs() {
String engineWriteFs = getOptional("kylin.env.engine-write-fs", "");
return StringHelper.dropSuffix(engineWriteFs, File.separator);
(kylin) 05/15: KYLIN-5737 Fix vulnerabilities, upgrade tomcat-embed-core & netty (#30573)
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d008c28156b0b2cfea530b08f765613575b2c60e
Author: Yinghao Lin <39019287+yhca...@users.noreply.github.com>
AuthorDate: Thu Jun 29 20:09:22 2023 +0800
KYLIN-5737 Fix vulnerabilities, upgrade tomcat-embed-core & netty (#30573)
---
pom.xml | 27 ++-
src/kylin-it/pom.xml| 9 -
src/kylin-server-it/pom.xml | 9 -
src/server/pom.xml | 1 +
src/streaming/pom.xml | 7 +--
5 files changed, 12 insertions(+), 41 deletions(-)
diff --git a/pom.xml b/pom.xml
index 904f18cb80..a61ca736d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -100,7 +100,7 @@
UTF-8
UTF-8
3.6.7-kyarthas-r3
-9.0.74
+9.0.76
@@ -360,9 +360,8 @@
10.14.2.0
1.75
1.10.11
-4.1.89.Final
-3.10.6.Final
-4.1.89.Final
+3.10.6.Final
+4.1.94.Final
0.13.2
1.0.3
2.6.2
@@ -741,11 +740,6 @@
kylin-soft-affinity-cache
${project.version}
-
-io.netty
-netty-codec-haproxy
-${netty-codec.version}
-
io.dropwizard.metrics
metrics-core
@@ -2788,14 +2782,8 @@
io.netty
-netty-all
-${netty-all.version}
-
-
-io.netty
-netty
-${netty.version}
-provided
+netty-handler
+${netty-handler.version}
org.codehaus.janino
@@ -2961,6 +2949,11 @@
snappy-java
${snappy-java.version}
+
+io.netty
+netty
+${netty3.version}
+
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index 01282cf457..4272d86bf6 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -97,15 +97,6 @@
commons-cli
-
-io.netty
-netty-all
-
-
-io.netty
-netty
-provided
-
org.codehaus.janino
janino
diff --git a/src/kylin-server-it/pom.xml b/src/kylin-server-it/pom.xml
index 9c60c7bb48..8cd256e983 100644
--- a/src/kylin-server-it/pom.xml
+++ b/src/kylin-server-it/pom.xml
@@ -106,15 +106,6 @@
commons-cli
-
-io.netty
-netty-all
-
-
-io.netty
-netty
-provided
-
org.codehaus.janino
janino
diff --git a/src/server/pom.xml b/src/server/pom.xml
index fdf7d8182e..deb02b3ade 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -443,6 +443,7 @@
net.sf.py4j,
com.h2database,
jdk.tools,
+io.netty
unused
diff --git a/src/streaming/pom.xml b/src/streaming/pom.xml
index 105755ad83..e03dc67f0c 100644
--- a/src/streaming/pom.xml
+++ b/src/streaming/pom.xml
@@ -159,11 +159,6 @@
test
3.4.0
-
-io.netty
-netty
-test
-
org.apache.spark
@@ -206,4 +201,4 @@
-
\ No newline at end of file
+
(kylin) 04/15: KYLIN-5760 Support matching model exactly for list jobs
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b15367e3e2682b195966f5b59dbf09f8867a792e
Author: Hang Jia <754332...@qq.com>
AuthorDate: Thu Jun 29 14:24:12 2023 +0800
KYLIN-5760 Support matching model exactly for list jobs
---
.../kap/secondstorage/metadata/TableFlow.java | 13 ++
.../kylin/rest/controller/JobController.java | 4 +-
.../kylin/rest/controller/v2/JobControllerV2.java | 2 +-
.../kylin/rest/controller/JobControllerTest.java | 6 +--
.../rest/controller/v2/JobControllerV2Test.java| 11 ++---
.../org/apache/kylin/rest/request/JobFilter.java | 2 +
.../org/apache/kylin/rest/service/JobService.java | 5 +++
.../apache/kylin/rest/service/JobServiceTest.java | 49 ++
.../apache/kylin/rest/service/ModelService.java| 9 ++--
9 files changed, 77 insertions(+), 24 deletions(-)
diff --git
a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
index a4add8edca..908f591e13 100644
---
a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
+++
b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import com.fasterxml.jackson.annotation.JsonManagedReference;
@@ -72,6 +73,8 @@ public class TableFlow extends RootPersistentEntity
return new Builder();
}
+public static final String TABLEFLOW_RESOURCE_ROOT = "/clickhouse_data";
+
protected transient Manager manager;
@Override
public void setManager(Manager manager) {
@@ -171,6 +174,16 @@ public class TableFlow extends RootPersistentEntity
});
}
+@Override
+public String getResourcePath() {
+return concatResourcePath(getUuid(), manager.project);
+}
+
+public static String concatResourcePath(String name, String project) {
+return new
StringBuilder().append("/").append(project).append(TABLEFLOW_RESOURCE_ROOT)
+
.append("/").append(name).append(MetadataConstants.FILE_SURFIX).toString();
+}
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index ea3e921baa..e47d01b658 100644
---
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -94,6 +94,7 @@ public class JobController extends BaseController {
@RequestParam(value = "time_filter") Integer timeFilter,
@RequestParam(value = "subject", required = false) String subject,
@RequestParam(value = "key", required = false) String key,
+@RequestParam(value = "exact", required = false, defaultValue =
"false") boolean exactMatch,
@RequestParam(value = "project", required = false) String project,
@RequestParam(value = "page_offset", required = false,
defaultValue = "0") Integer pageOffset,
@RequestParam(value = "page_size", required = false, defaultValue
= "10") Integer pageSize,
@@ -101,7 +102,8 @@ public class JobController extends BaseController {
@RequestParam(value = "reverse", required = false, defaultValue =
"true") boolean reverse) {
jobService.checkJobStatus(statuses);
checkRequiredArg("time_filter", timeFilter);
-JobFilter jobFilter = new JobFilter(statuses, jobNames, timeFilter,
subject, key, project, sortBy, reverse);
+JobFilter jobFilter = new JobFilter(statuses, jobNames, timeFilter,
subject, key, exactMatch, project, sortBy,
+reverse);
DataResult> executables;
if (!StringUtils.isEmpty(project)) {
executables = jobService.listJobs(jobFilter, pageOffset, pageSize);
diff --git
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/v2/JobControllerV2.java
index 720238fb53..6844578785 100644
---
a/src/data-loading-server/src/main/java/org/apache/kylin/rest/
(kylin) 03/15: KYLIN-5762 Initialize job scheduler encounters NPE
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6816947de43cc1f3a7101c06a1d32d623a36ef06
Author: sibingzhang <74443791+sibingzh...@users.noreply.github.com>
AuthorDate: Wed Jun 28 18:28:54 2023 +0800
KYLIN-5762 Initialize job scheduler encounters NPE
Co-authored-by: sibing.zhang
---
.../java/org/apache/kylin/job/execution/ExecutableContext.java | 9 ++---
.../java/org/apache/kylin/job/execution/ExecutableThread.java| 1 +
.../src/main/java/org/apache/kylin/job/runners/JobRunner.java| 3 ++-
.../org/apache/kylin/job/execution/ExecutableContextTest.java| 4 +++-
.../org/apache/kylin/job/execution/NExecutableManagerTest.java | 2 +-
.../java/org/apache/kylin/rest/service/DagJobServiceTest.java| 8
6 files changed, 17 insertions(+), 10 deletions(-)
diff --git
a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
index 16c9e9f576..fea0a74f09 100644
---
a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
+++
b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableContext.java
@@ -23,10 +23,10 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.common.KylinConfig;
-
import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@@ -68,7 +68,6 @@ public class ExecutableContext {
// Only used when the job starts scheduling
public void addRunningJob(Executable executable) {
-runningJobThreads.put(executable.getId(), Thread.currentThread());
runningJobs.put(executable.getId(), executable);
runningJobInfos.put(executable.getId(), System.currentTimeMillis());
}
@@ -84,6 +83,10 @@ public class ExecutableContext {
return runningJobThreads.get(executable.getId());
}
+public void addRunningJobThread(Executable executable) {
+runningJobThreads.put(executable.getId(), Thread.currentThread());
+}
+
public Map getRunningJobs() {
return Collections.unmodifiableMap(runningJobs);
}
diff --git
a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
index 61140daeec..162eed1daa 100644
---
a/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
+++
b/src/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableThread.java
@@ -50,6 +50,7 @@ public class ExecutableThread extends Thread {
try (SetThreadName ignored = new
SetThreadName("JobWorker(project:%s,jobid:%s)", project, jobIdSimple);
SetLogCategory logCategory = new SetLogCategory("schedule")) {
context.addRunningJob(executable);
+context.addRunningJobThread(executable);
dagExecutable.executeDagExecutable(dagExecutablesMap, executable,
context);
} finally {
context.removeRunningJob(executable);
diff --git
a/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
b/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
index 3e839f57f4..b8cd19bed0 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/runners/JobRunner.java
@@ -18,13 +18,13 @@
package org.apache.kylin.job.runners;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.logging.SetLogCategory;
import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.JobStoppedNonVoluntarilyException;
import org.apache.kylin.job.exception.JobStoppedVoluntarilyException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
-import org.apache.kylin.common.logging.SetLogCategory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +50,7 @@ public class JobRunner extends AbstractDefaultSchedulerRunner
{
val jobIdSimple = executable.getId().substring(0, 8);
try (SetThreadName ignored = new
SetThreadName("JobWorker(project:%s,jobid:%s)", project, jobIdSimple);
SetLogCategory logCategory = new SetLogCategory("schedule")) {
+context.addRunningJobThread(executable);
executable.execute(context);
// trigger the next step asap
fetcherRunner.scheduleNext();
diff --git
a/s
(kylin) 02/15: KYLIN-5733 TDS template modify to English
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit 387e68d0a669b84d269341a8a4127c5b70d6bf55 Author: fanfanAlice <41991994+fanfanal...@users.noreply.github.com> AuthorDate: Tue Jun 27 17:20:58 2023 +0800 KYLIN-5733 TDS template modify to English --- .../src/main/resources/bisync/tds/tableau.connector.template.xml| 2 +- src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml | 2 +- .../src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds | 2 +- .../src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds | 2 +- .../resources/bisync_tableau/nmodel_full_measure_test.connector.tds | 2 +- .../bisync_tableau/nmodel_full_measure_test.connector_admin.tds | 2 +- .../resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds | 2 +- .../bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds | 2 +- .../bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds | 2 +- .../nmodel_full_measure_test.connector_no_hierarchies.tds | 2 +- .../bisync_tableau/nmodel_full_measure_test.connector_permission.tds| 2 +- .../nmodel_full_measure_test.connector_permission_agg_index_col.tds | 2 +- .../nmodel_full_measure_test.connector_permission_all_col.tds | 2 +- .../nmodel_full_measure_test.connector_permission_no_measure.tds| 2 +- .../bisync_tableau/nmodel_full_measure_test.connector_user.tds | 2 +- .../nmodel_full_measure_test.connector_user_agg_index_col.tds | 2 +- .../bisync_tableau/nmodel_full_measure_test.table_index_connector.tds | 2 +- .../src/test/resources/bisync_tableau/nmodel_full_measure_test.tds | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml index f410cfdb71..a83940f9b6 100755 --- a/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml +++ b/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml @@ -25,6 +25,6 @@ - + diff --git a/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml index a72b454dfc..16a2ef2b7f 100644 --- a/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml +++ b/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml @@ -27,6 +27,6 @@ - + diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds index f1a91e41f4..6477036203 100644 --- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds +++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds @@ -544,6 +544,6 @@ - + diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds index 6b84dddcbb..edf95da259 100644 --- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds +++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds @@ -541,6 +541,6 @@ - + diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds index ac3e77ff62..7b0a31f9b9 100644 --- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds +++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds @@ -120,6 +120,6 @@ - + diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds index f02667d2a2..ceaad43004 100644 --- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds +++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_admin.tds @@ -120,6 +120,6 @@ - + diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds index 4663e549bc..91017c63d1 100644 --- a/src/modeling-service/src/test/resources/bisync_t
(kylin) 01/15: KYLIN-5732 Upgrade snappy-java to fix vulnerabilities
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3ec3063135f6b99cb4f7934d056a6ca7fd4a1e5e
Author: Yinghao Lin <39019287+yhca...@users.noreply.github.com>
AuthorDate: Tue Jun 20 16:23:34 2023 +0800
KYLIN-5732 Upgrade snappy-java to fix vulnerabilities
---
pom.xml | 8
1 file changed, 8 insertions(+)
diff --git a/pom.xml b/pom.xml
index 534e8bcca6..904f18cb80 100644
--- a/pom.xml
+++ b/pom.xml
@@ -371,6 +371,7 @@
2.13.4.2
9.23
1.2.1
+1.1.10.1
@@ -2954,6 +2955,13 @@
${system-lambda.version}
test
+
+
+org.xerial.snappy
+snappy-java
+${snappy-java.version}
+
+
(kylin) 13/15: KYLIN-5741 punctuation optimized
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 227e7a068360978133f8188cff859723cea458bf
Author: zhong.zhu
AuthorDate: Tue Dec 19 11:52:18 2023 +0800
KYLIN-5741 punctuation optimized
---
.../org/apache/kylin/common/msg/CnMessage.java | 2 +-
.../kylin/rest/service/MetaStoreService.java | 24 ++--
.../org/apache/spark/sql/udf/CeilFloorTest.scala | 139 +++--
.../apache/kylin/helper/MetadataToolHelper.java| 20 +--
4 files changed, 94 insertions(+), 91 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index ed626ec5d1..09c3925756 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -32,7 +32,7 @@ public class CnMessage extends Message {
private static final String TASK_TIMEOUT = "执行超时";
private static final String PARAMETER_EMPTY = "请输入参数 “%s” 的值。";
-private static final String PARAMETER_MUST_BE_POSITIVE_NUMBER= "参数 %s
的值必须为非负数.";
+private static final String PARAMETER_MUST_BE_POSITIVE_NUMBER= "参数 %s
的值必须为非负数。";
protected CnMessage() {
diff --git
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
index bf44347d69..4976131faa 100644
---
a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
+++
b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
@@ -75,6 +75,10 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.guava30.shaded.common.collect.Maps;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.io.ByteSource;
import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.metadata.cube.model.IndexEntity;
@@ -99,6 +103,10 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.util.QueryHisStoreUtil;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
+import org.apache.kylin.metadata.recommendation.candidate.RawRecItem;
+import org.apache.kylin.metadata.recommendation.candidate.RawRecManager;
+import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
import org.apache.kylin.metadata.view.LogicalView;
import org.apache.kylin.metadata.view.LogicalViewManager;
import org.apache.kylin.rest.aspect.Transaction;
@@ -121,16 +129,6 @@ import
org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.guava30.shaded.common.io.ByteSource;
-
-import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.metadata.recommendation.candidate.RawRecItem;
-import org.apache.kylin.metadata.recommendation.candidate.RawRecManager;
-import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
-
import lombok.Setter;
import lombok.val;
import lombok.var;
@@ -466,9 +464,9 @@ public class MetaStoreService extends BasicService {
if (config.isDDLLogicalViewEnabled() &&
missTableDesc.isLogicalView()) {
LogicalView logicalView =
LogicalViewManager.getInstance(config).get(missTableDesc.getName());
if (logicalView != null &&
!targetProject.equalsIgnoreCase(logicalView.getCreatedProject())) {
-throw new KylinException(FAILED_CREATE_MODEL,
String.format(Locale.ROOT,
-" Logical View %s can only add in project %s",
-missTableDesc.getName(),
logicalView.getCreatedProject()));
+throw new KylinException(FAILED_CREATE_MODEL,
+String.format(Locale.ROOT, " Logical View %s can
only add in project %s",
+missTableDesc.getName(),
logicalView.getCreatedProject()));
}
}
}
dif
(kylin) 09/15: KYLIN-5741 change all epoch when empty projects request
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7cd3fd0ecc8ce95ef4f21a84abd1ed251d4e54a4
Author: Yu Gan
AuthorDate: Thu Jun 29 19:00:05 2023 +0800
KYLIN-5741 change all epoch when empty projects request
---
.../kylin/rest/controller/NEpochController.java | 9 -
.../kylin/rest/controller/NEpochControllerTest.java | 19 ---
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
index 7f9f1ff00b..ac194caa73 100644
---
a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
+++
b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NEpochController.java
@@ -36,6 +36,9 @@ import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
+import java.util.ArrayList;
+import java.util.Objects;
+
@Controller
@RequestMapping(value = "/api/epoch", produces = { HTTP_VND_APACHE_KYLIN_JSON
})
public class NEpochController extends NBasicController {
@@ -48,7 +51,11 @@ public class NEpochController extends NBasicController {
@PostMapping(value = "", produces = { HTTP_VND_APACHE_KYLIN_JSON,
HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON })
@ResponseBody
public EnvelopeResponse updateEpochOwner(@RequestBody EpochRequest
epochRequest) {
-checkCollectionRequiredArg("projects", epochRequest.getProjects());
+if (Objects.isNull(epochRequest.getProjects())) {
+// Avoid following NPEs.
+epochRequest.setProjects(new ArrayList<>(0));
+}
+// Empty projects has specified meanings: all projects do change epoch.
epochRequest.getProjects().forEach(this::checkProjectName);
checkRequiredArg("force", epochRequest.getForce());
epochService.updateEpoch(epochRequest.getProjects(),
epochRequest.getForce(), epochRequest.isClient());
diff --git
a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
index 80e0b3bf04..523d95e2c8 100644
---
a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
+++
b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NEpochControllerTest.java
@@ -96,17 +96,12 @@ public class NEpochControllerTest extends
NLocalFileMetadataTestCase {
mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
.content(JsonUtil.writeValueAsString(mapRequest2))
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
-.andExpect(MockMvcResultMatchers.status().is5xxServerError());
-request.setProjects(Lists.newArrayList());
-mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
-.content(JsonUtil.writeValueAsString(request))
-
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
-.andExpect(MockMvcResultMatchers.status().is5xxServerError());
+.andExpect(MockMvcResultMatchers.status().isOk());
request.setProjects(Lists.newArrayList());
mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
.content(JsonUtil.writeValueAsString(request))
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
-.andExpect(MockMvcResultMatchers.status().is5xxServerError());
+.andExpect(MockMvcResultMatchers.status().isOk());
request.setProjects(Arrays.asList("DEFAULT"));
mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
.content(JsonUtil.writeValueAsString(request))
@@ -114,6 +109,16 @@ public class NEpochControllerTest extends
NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk());
}
+@Test
+public void testUpdateEpochOwnerWithEmptyProjects() throws Exception {
+val request = new EpochRequest();
+request.setForce(false);
+mockMvc.perform(MockMvcRequestBuilders.post("/api/epoch")
+.content(JsonUtil.writeValueAsString(request))
+
.contentType(MediaType.APPLICATION_JSON).accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+.andExpect(MockMvcResultMatchers.status().isOk());
+}
+
@Test
public void testUpdateAllEpochOwner() throws Exception {
val request = mockStreamingEpochRequest();
(kylin) 15/15: update spark version to 3.2.0-kylin-4.6.12.0
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit bcd4220ef2556eaab318d7e2fa4d6e3c1d9fe312 Author: Yu Gan AuthorDate: Sat Jul 8 10:08:47 2023 +0800 update spark version to 3.2.0-kylin-4.6.12.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8bfc1dd630..9b68f4a83d 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 2.0.2 -3.2.0-kylin-4.6.9.0 +3.2.0-kylin-4.6.12.0 0.9.2-kylin-r4
(kylin) 07/15: KYLIN-5762 Initialize job scheduler encounters NPE
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1d665a82829a82fc9a51181378aa87de1a98dea2
Author: huangsheng
AuthorDate: Fri Jun 30 10:43:50 2023 +0800
KYLIN-5762 Initialize job scheduler encounters NPE
---
.../apache/kylin/rest/service/ProjectService.java | 8 +++
.../apache/kylin/common/constant/Constants.java| 1 +
.../org/apache/kylin/common/msg/CnMessage.java | 6 +
.../java/org/apache/kylin/common/msg/Message.java | 7 ++
.../job/impl/threadpool/NDefaultScheduler.java | 21 ++---
.../apache/kylin/job/runners/FetcherRunner.java| 3 +--
.../job/impl/threadpool/NDefaultSchedulerTest.java | 21 +
.../kylin/rest/service/ProjectServiceTest.java | 26 ++
8 files changed, 83 insertions(+), 10 deletions(-)
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index d32d8b261e..416f577e0e 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -25,6 +25,7 @@ import static
org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_
import static
org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_ENABLE_KEY;
import static
org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_SOURCE_NAME_KEY;
import static
org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_USER_KEY;
+import static
org.apache.kylin.common.constant.Constants.KYLIN_JOB_MAX_CONCURRENT_JOBS;
import static
org.apache.kylin.common.constant.NonCustomProjectLevelConfig.DATASOURCE_TYPE;
import static
org.apache.kylin.common.exception.ServerErrorCode.DATABASE_NOT_EXIST;
import static
org.apache.kylin.common.exception.ServerErrorCode.DUPLICATE_PROJECT_NAME;
@@ -82,6 +83,7 @@ import org.apache.kylin.guava30.shaded.common.base.Strings;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Maps;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.guava30.shaded.common.primitives.Ints;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.NExecutableManager;
@@ -431,6 +433,12 @@ public class ProjectService extends BasicService {
if (projectInstance == null) {
throw new KylinException(PROJECT_NOT_EXIST, project);
}
+if (overrideKylinProps.containsKey(KYLIN_JOB_MAX_CONCURRENT_JOBS)) {
+val maxConcurrentJobs =
Ints.tryParse(overrideKylinProps.get(KYLIN_JOB_MAX_CONCURRENT_JOBS));
+if (Objects.isNull(maxConcurrentJobs) || maxConcurrentJobs < 0)
+throw new KylinException(INVALID_PARAMETER,
+
MsgPicker.getMsg().getIllegalNegative(KYLIN_JOB_MAX_CONCURRENT_JOBS));
+}
encryptJdbcPassInOverrideKylinProps(overrideKylinProps);
projectManager.updateProject(project, copyForWrite ->
copyForWrite.getOverrideKylinProps()
.putAll(KylinConfig.trimKVFromMap(overrideKylinProps)));
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
b/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
index a49d012fa2..6f3a9e1f13 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/constant/Constants.java
@@ -48,6 +48,7 @@ public class Constants {
public static final String KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY =
"kylin.source.jdbc.connection-url";
public static final String KYLIN_SOURCE_JDBC_USER_KEY =
"kylin.source.jdbc.user";
public static final String KYLIN_SOURCE_JDBC_DRIVER_KEY =
"kylin.source.jdbc.driver";
+public static final String KYLIN_JOB_MAX_CONCURRENT_JOBS =
"kylin.job.max-concurrent-jobs";
public static final String UNLIMITED = "Unlimited";
public static final String TRACE_ID = "traceId";
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index c62f52156d..ed626ec5d1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -32,6 +32,7 @@ public class CnMessage extends Message {
private static final String TASK_TIMEOUT = "执行超时";
private static final String PARAMETER_EMPTY = "请输入参数 “%s” 的值。"
(kylin) 06/15: KYLIN-5742 Make the query result of duplicate group sets same as Spark
This is an automated email from the ASF dual-hosted git repository. liyang pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git commit c396134127b17e741e6ead1197589afe7bb773d7 Author: fengguangyuan AuthorDate: Fri Jun 30 15:28:07 2023 +0800 KYLIN-5742 Make the query result of duplicate group sets same as Spark Co-authored-by: Guangyuan Feng --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a61ca736d4..83adcd3e33 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ 0.3.0 -1.116.0-kylin-4.x-r029 +1.116.0-kylin-4.x-r031 4.x_1.10-r01
svn commit: r1915437 [1/5] - in /kylin/site: ./ about/ blog/ blog/2015/01/25/introduce-data-model/ blog/2015/06/10/release-v0.7.1-incubating/ blog/2015/08/13/kylin-dictionary/ blog/2015/08/15/fast-cub
Author: liyang Date: Mon Jan 29 02:47:55 2024 New Revision: 1915437 URL: http://svn.apache.org/viewvc?rev=1915437=rev Log: kylin 4.0.4 release Modified: kylin/site/about/index.html kylin/site/blog/2015/01/25/introduce-data-model/index.html kylin/site/blog/2015/06/10/release-v0.7.1-incubating/index.html kylin/site/blog/2015/08/13/kylin-dictionary/index.html kylin/site/blog/2015/08/15/fast-cubing/index.html kylin/site/blog/2015/09/06/release-v1.0-incubating/index.html kylin/site/blog/2015/09/09/fast-cubing-on-spark/index.html kylin/site/blog/2015/09/25/hybrid-model/index.html kylin/site/blog/2015/10/14/Apache-Kylin-Meetup/index.html kylin/site/blog/2015/10/25/release-v1.1-incubating/index.html kylin/site/blog/2015/12/23/release-v1.2/index.html kylin/site/blog/2015/12/25/support-powerbi-tableau9/index.html kylin/site/blog/2016/02/03/streaming-cubing/index.html kylin/site/blog/2016/02/18/new-aggregation-group/index.html kylin/site/blog/2016/03/16/release-v1.3.0/index.html kylin/site/blog/2016/03/17/release-v1.5.0/index.html kylin/site/blog/2016/03/19/approximate-topn-measure/index.html kylin/site/blog/2016/04/12/release-v1.5.1/index.html kylin/site/blog/2016/05/26/release-v1.5.2/index.html kylin/site/blog/2016/05/29/raw-measure-in-kylin/index.html kylin/site/blog/2016/06/10/diagnosis-tool/index.html kylin/site/blog/2016/06/10/standalone-hbase-cluster/index.html kylin/site/blog/2016/07/28/release-v1.5.3/index.html kylin/site/blog/2016/08/01/count-distinct-in-kylin/index.html kylin/site/blog/2016/08/27/query-metrics-in-kylin/index.html kylin/site/blog/2016/10/18/new-nrt-streaming/index.html kylin/site/blog/2016/11/16/window-function/index.html kylin/site/blog/2016/11/28/intersect-count/index.html kylin/site/blog/2016/12/04/release-v1.6.0/index.html kylin/site/blog/2017/02/23/by-layer-spark-cubing/index.html kylin/site/blog/2017/02/25/v2.0.0-beta-ready/index.html kylin/site/blog/2017/04/01/percentile-measure/index.html kylin/site/blog/2017/07/21/Improving-Spark-Cubing/index.html kylin/site/blog/2018/01/01/kylin-and-superset/index.html kylin/site/blog/2018/03/04/release-v2.3.0/index.html kylin/site/blog/2018/05/08/redash-kylin-plugin-strikingly/index.html kylin/site/blog/2018/07/16/Star-Schema-Benchmark-on-Apache-Kylin/index.html kylin/site/blog/2018/09/20/release-v2.5.0/index.html kylin/site/blog/2018/12/12/why-did-meituan-develop-kylin-on-druid-part1-of-2/index.html kylin/site/blog/2019/01/16/introduce-data-source-sdk-v2.6.0/index.html kylin/site/blog/2019/01/17/cisco-throughput-5x/index.html kylin/site/blog/2019/01/18/release-v2.6.0/index.html kylin/site/blog/2019/04/12/rt-streaming-design/index.html kylin/site/blog/2019/04/19/release-v3.0.0-alpha/index.html kylin/site/blog/2019/05/29/kylin-2.4.1-to-2.6.1/index.html kylin/site/blog/2019/06/26/use-python-for-data-science-with-apache-kylin/index.html kylin/site/blog/2019/07/01/deep-dive-real-time-olap/index.html kylin/site/blog/2019/07/30/detailed-analysis-of-refine-query-cache/index.html kylin/site/blog/2019/09/22/kylin-tableau/index.html kylin/site/blog/2021/06/17/Why-did-Youzan-choose-Kylin4/index.html kylin/site/blog/2021/07/02/Apache-Kylin4-A-new-storage-and-compute-architecture/index.html kylin/site/blog/2021/08/03/How-Meituan-Dominates-Online-Shopping-with-Apache-Kylin/index.html kylin/site/blog/2021/10/21/Local-Cache-and-Soft-Affinity-Scheduling/index.html kylin/site/blog/2022/01/12/The-Future-Of-Kylin/index.html kylin/site/blog/2022/03/17/kylin4-now-supporting-aws-glue-catalog/index.html kylin/site/blog/2022/03/31/how-to-use-excel-to-query-kylin/index.html kylin/site/blog/2022/04/20/kylin4-on-cloud-part1/index.html kylin/site/blog/2022/04/20/kylin4-on-cloud-part2/index.html kylin/site/blog/index.html kylin/site/cn/blog/index.html kylin/site/cn/development/datasource_sdk.html kylin/site/cn/development/howto_setup_frontend_registry.html kylin/site/cn/development40/howto_setup_frontend_registry.html kylin/site/cn/docs24/install/kylin_docker.html kylin/site/cn/docs30/install/kylin_docker.html kylin/site/cn/docs30/install/kylin_on_kubernetes.html kylin/site/cn/docs31/install/kylin_docker.html kylin/site/cn/docs31/install/kylin_on_kubernetes.html kylin/site/cn_blog/2015/10/25/release-v1.1-incubating/index.html kylin/site/cn_blog/2015/12/23/release-v1.2/index.html kylin/site/cn_blog/2015/12/25/support-powerbi-tableau9/index.html kylin/site/cn_blog/2016/03/16/release-v1.3.0/index.html kylin/site/cn_blog/2016/03/17/release-v1.5.0/index.html kylin/site/cn_blog/2016/04/12/release-v1.5.1/index.html kylin/site/cn_blog/2016/05/26/release-v1.5.2/index.html kylin/site/cn_blog/2016/07/28/release-v1.5.3/index.html kylin/site/cn_blog/2016/12/04/release-v1.6.0
svn commit: r1915437 [4/5] - in /kylin/site: ./ about/ blog/ blog/2015/01/25/introduce-data-model/ blog/2015/06/10/release-v0.7.1-incubating/ blog/2015/08/13/kylin-dictionary/ blog/2015/08/15/fast-cub
Modified: kylin/site/docs30/install/index.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/install/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/install/index.html (original) +++ kylin/site/docs30/install/index.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/install/kylin_aws_emr.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/install/kylin_aws_emr.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/install/kylin_aws_emr.html (original) +++ kylin/site/docs30/install/kylin_aws_emr.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/install/kylin_cluster.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/install/kylin_cluster.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/install/kylin_cluster.html (original) +++ kylin/site/docs30/install/kylin_cluster.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/install/kylin_docker.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/install/kylin_docker.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/install/kylin_docker.html (original) +++ kylin/site/docs30/install/kylin_docker.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/install/kylin_on_kubernetes.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/install/kylin_on_kubernetes.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/install/kylin_on_kubernetes.html (original) +++ kylin/site/docs30/install/kylin_on_kubernetes.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/release_notes.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/release_notes.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/release_notes.html (original) +++ kylin/site/docs30/release_notes.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/security.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/security.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/security.html (original) +++ kylin/site/docs30/security.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/tutorial/Qlik.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/tutorial/Qlik.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/tutorial/Qlik.html (original) +++ kylin/site/docs30/tutorial/Qlik.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs30/tutorial/acl.html URL: http://svn.apache.org/viewvc/kylin/site/docs30/tutorial/acl.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs30/tutorial/acl.html (original) +++ kylin/site/docs30/tutorial/acl.html Mon
svn commit: r1915437 [3/5] - in /kylin/site: ./ about/ blog/ blog/2015/01/25/introduce-data-model/ blog/2015/06/10/release-v0.7.1-incubating/ blog/2015/08/13/kylin-dictionary/ blog/2015/08/15/fast-cub
Modified: kylin/site/docs/tutorial/project_level_acl.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/project_level_acl.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/project_level_acl.html (original) +++ kylin/site/docs/tutorial/project_level_acl.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/query_pushdown.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/query_pushdown.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/query_pushdown.html (original) +++ kylin/site/docs/tutorial/query_pushdown.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/quick_start_for_mdx.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/quick_start_for_mdx.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/quick_start_for_mdx.html (original) +++ kylin/site/docs/tutorial/quick_start_for_mdx.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/setup_systemcube.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/setup_systemcube.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/setup_systemcube.html (original) +++ kylin/site/docs/tutorial/setup_systemcube.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/sql_reference.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/sql_reference.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/sql_reference.html (original) +++ kylin/site/docs/tutorial/sql_reference.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/squirrel.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/squirrel.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/squirrel.html (original) +++ kylin/site/docs/tutorial/squirrel.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/superset.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/superset.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/superset.html (original) +++ kylin/site/docs/tutorial/superset.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/tableau.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/tableau.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs/tutorial/tableau.html (original) +++ kylin/site/docs/tutorial/tableau.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs/tutorial/tableau_91.html URL: http://svn.apache.org/viewvc/kylin/site/docs/tutorial/tableau_91.html?rev=1915437=1915436=1915437=diff ==
svn commit: r1915437 [2/5] - in /kylin/site: ./ about/ blog/ blog/2015/01/25/introduce-data-model/ blog/2015/06/10/release-v0.7.1-incubating/ blog/2015/08/13/kylin-dictionary/ blog/2015/08/15/fast-cub
Modified: kylin/site/cn_blog/2015/12/25/support-powerbi-tableau9/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2015/12/25/support-powerbi-tableau9/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2015/12/25/support-powerbi-tableau9/index.html (original) +++ kylin/site/cn_blog/2015/12/25/support-powerbi-tableau9/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2016/03/16/release-v1.3.0/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2016/03/16/release-v1.3.0/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2016/03/16/release-v1.3.0/index.html (original) +++ kylin/site/cn_blog/2016/03/16/release-v1.3.0/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2016/03/17/release-v1.5.0/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2016/03/17/release-v1.5.0/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2016/03/17/release-v1.5.0/index.html (original) +++ kylin/site/cn_blog/2016/03/17/release-v1.5.0/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2016/04/12/release-v1.5.1/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2016/04/12/release-v1.5.1/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2016/04/12/release-v1.5.1/index.html (original) +++ kylin/site/cn_blog/2016/04/12/release-v1.5.1/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2016/05/26/release-v1.5.2/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2016/05/26/release-v1.5.2/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2016/05/26/release-v1.5.2/index.html (original) +++ kylin/site/cn_blog/2016/05/26/release-v1.5.2/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2016/07/28/release-v1.5.3/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2016/07/28/release-v1.5.3/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2016/07/28/release-v1.5.3/index.html (original) +++ kylin/site/cn_blog/2016/07/28/release-v1.5.3/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2016/12/04/release-v1.6.0/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2016/12/04/release-v1.6.0/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2016/12/04/release-v1.6.0/index.html (original) +++ kylin/site/cn_blog/2016/12/04/release-v1.6.0/index.html Mon Jan 29 02:47:55 2024 @@ -143,7 +143,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/cn_blog/2017/02/25/v2.0.0-beta-ready/index.html URL: http://svn.apache.org/viewvc/kylin/site/cn_blog/2017/02/25/v2.0.0-beta-ready/index.html?rev=1915437=1915436=1915437=diff == --- kylin/site/cn_blog/2017/02/25/v2.0.0-beta-ready/index.html (original) +++ kylin/site/cn_blog/2017/02/25/v2.0.0-beta-ready/index.html Mon Jan 29 02:47:55 2024 @@
svn commit: r1915437 [5/5] - in /kylin/site: ./ about/ blog/ blog/2015/01/25/introduce-data-model/ blog/2015/06/10/release-v0.7.1-incubating/ blog/2015/08/13/kylin-dictionary/ blog/2015/08/15/fast-cub
Modified: kylin/site/docs31/tutorial/realtime_olap.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/realtime_olap.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/realtime_olap.html (original) +++ kylin/site/docs31/tutorial/realtime_olap.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/setup_jdbc_datasource.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/setup_jdbc_datasource.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/setup_jdbc_datasource.html (original) +++ kylin/site/docs31/tutorial/setup_jdbc_datasource.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/setup_systemcube.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/setup_systemcube.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/setup_systemcube.html (original) +++ kylin/site/docs31/tutorial/setup_systemcube.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/spark.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/spark.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/spark.html (original) +++ kylin/site/docs31/tutorial/spark.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/sql_reference.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/sql_reference.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/sql_reference.html (original) +++ kylin/site/docs31/tutorial/sql_reference.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/squirrel.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/squirrel.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/squirrel.html (original) +++ kylin/site/docs31/tutorial/squirrel.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/superset.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/superset.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/superset.html (original) +++ kylin/site/docs31/tutorial/superset.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/tableau.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/tableau.html?rev=1915437=1915436=1915437=diff == --- kylin/site/docs31/tutorial/tableau.html (original) +++ kylin/site/docs31/tutorial/tableau.html Mon Jan 29 02:47:55 2024 @@ -144,7 +144,7 @@ var _hmt = _hmt || []; Docs - Kylin 4.0.2 + Kylin 4.0.4 Kylin 3.1.3 Kylin 2.4.0 Archive Modified: kylin/site/docs31/tutorial/tableau_91.html URL: http://svn.apache.org/viewvc/kylin/site/docs31/tutorial/tableau_91.html?rev=1915437=1915436=1915437=diff
svn commit: r66903 - /release/kylin/apache-kylin-4.0.4/
Author: liyang Date: Mon Jan 29 01:09:29 2024 New Revision: 66903 Log: release 4.0.4 Added: release/kylin/apache-kylin-4.0.4/ release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz (with props) release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.asc release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.sha256 release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip (with props) release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.asc release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.sha256 Added: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz == Binary file - no diff available. Propchange: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz -- svn:mime-type = application/octet-stream Added: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.asc == --- release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.asc (added) +++ release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.asc Mon Jan 29 01:09:29 2024 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCgAdFiEEz0iPJCu8Oohdt8bfaF9bXdJU3okFAmW1a4QACgkQaF9bXdJU +3omBFA/9E/W4YcpaPbCM9COtQZcvaUMpDYnvyk1TeSxa/ovGDBVLMUmO+mMPyiOj +6JV2WQ5esiZyR1/Z6aZD96eXgOMnAEIYZPy318xuvKCycqfkjJcE+JmpwNTWODAJ +VDH+PsD/g0HYnDXF+Aqyy6Leez0QIxsVLFuwdskinr/dhR7DRFbdMOD+sOYcC1H0 +tj9dWFCRaaLtJ4zsu7r3eIIACfP/WRvU5fC7jT5LczeRcHiASgg3JE3UsWvZmCew +UucQg6INh+tKilZwpna7fQ4dQHtyX5C2IRdfdHElfYdSnd+skZDC9kexOuX5/1lp +Hp0obF1fqzA/yxqTttZUiqL5BqSzfqzidqAVgH2VpBuIIlOYMuQ7IuO3XN9r9zWa +Bj5vrlO0SU11/xH0mwTRmhtsA6+ovXdJ0gGYdgBQhTtnPu2WD+oXN9V+2cklXGRc +E70aIqaKmc8M8uxaabFw2aqwdFn+FLS3rO867srxQZlJ1xkU+w0Yb5gPQnXBNLeV +v7sU4O/fwGzh3HuNVVnf+G8o5X8wdX0mY3xOUmEUXixSX0n4GKWffAXo4ROGflKe +LG/6yvedqhADJGoviucPgWn3P6DIz3d3uvVXNhx1MLyDRiPuXLfzXN2Cj6gHCutW +q8ohMi6WgfKcyBJ+9XmsLa4xDxnaRbx0uE5fIM/QqQPaC9QWhpU= +=Fz2z +-END PGP SIGNATURE- Added: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.sha256 == --- release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.sha256 (added) +++ release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-bin-spark3.tar.gz.sha256 Mon Jan 29 01:09:29 2024 @@ -0,0 +1 @@ +102cbe845299813e379bb0618935e6b5388157d78acbfb3b78919ae43adf32eb apache-kylin-4.0.4-bin-spark3.tar.gz Added: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip == Binary file - no diff available. Propchange: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip -- svn:mime-type = application/octet-stream Added: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.asc == --- release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.asc (added) +++ release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.asc Mon Jan 29 01:09:29 2024 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2.0.14 (GNU/Linux) + +iQIcBAABCgAGBQJlqq51AAoJEGhfW13SVN6Jzs4P/A4GJezdspBL3q+lbq/vhpzP +3bAc4+3rFwl67tn1n+iaT15NHt3P1Ntn1irt29xTx4V9z/O80/HN6+S3aArLBt1y +Erla6YK2DlxdJKnDqNGalxqonsfVKlG4r8qQMwhnGNiqYkeEQel6pdBgAGfnVKuG +V2LAdBJCs5RhLDpqK8tIT8FpI+ac9Lww27LWyd3dIQ9Z15m/72BG8QQCP/pDobxr +4q8H2irUGh3IVG3Fdk2SuKgeAddlWFtivAQlJjXLULgFZNe4Y5L9z65TrnyG2ZTp +EoHceQkGv92t54p5G7gcNiULsQsbblKiJIH32i4z0skPgp0o+X9wXvcFIGmZdsHn +33KRiqccWQW782PR1tKBaIj2gPfum0evhIp7osYXhqH0bpW4/nnU4Ujc60Dgx2/g +aK+/qg0eWHIDm2sklyeQFwvWBEgdjsbNuI+vbuB8mxsD0lNy1kCogpg7Lo1scPNP +lQraMT9+z+jYWQ0LMH1V3530tBnytDWXCSA1EA6rylETAnOXIk1EgiinCu8yip6U +4aTnqw+xOiO6KYr69kfDtel6LKmahrmqThiaMZ6qy/x09+u2dhMoW7rN+Zs9xXjk +s1z/T89TvRQKEpiBCX9hditAEfNpgytYa4GgOT5oJBwGUf+20jGxU7MbhZF6oqKQ +Y4lOV7eFCuqJMmy3XVNO +=zo2f +-END PGP SIGNATURE- Added: release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.sha256 == --- release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.sha256 (added) +++ release/kylin/apache-kylin-4.0.4/apache-kylin-4.0.4-source-release.zip.sha256 Mon Jan 29 01:09:29 2024 @@ -0,0 +1 @@ +21b338aae14a71357650b35f473381cf325a9781adf1f6c9954cae9b4027cfe2 \ No newline at end of file
