This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d2907d8f017c316376d5ad5e0206b49c4dba0118
Author: liuzx32 <liuz...@163.com>
AuthorDate: Wed Aug 28 14:13:43 2019 +0800
Add JoinedFormatter for Dynamic variables
---
.../java/org/apache/kylin/job/JoinedFlatTable.java | 5 +-
.../java/org/apache/kylin/job/JoinedFormatter.java | 127 +++++++++++++++++++++
.../org/apache/kylin/job/JoinedFormatterTest.java | 62 ++++++++++
3 files changed, 192 insertions(+), 2 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index f729d59..5884421 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -225,8 +225,9 @@ public class JoinedFlatTable {
DataModelDesc model = flatDesc.getDataModel();
if (StringUtils.isNotEmpty(model.getFilterCondition())) {
- String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc,
- model.getFilterCondition());
+ JoinedFormatter formatter = new JoinedFormatter(flatDesc);
+ String fmtFilterCondition =
formatter.formatSentense(model.getFilterCondition());
+ String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc,
fmtFilterCondition);
whereBuilder.append(" AND
(").append(quotedFilterCondition).append(") "); // -> filter condition contains
special character may cause bug
}
if (flatDesc.getSegment() != null) {
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
new file mode 100644
index 0000000..aa7408c
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFormatter.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Joined Formatter for JoinedFlatTable
+ */
+
+public class JoinedFormatter {
+
+ private static String REG_SEPARATOR = "\\$\\{(?<KEY>.*?)\\}";
+ private static Pattern REG_PATTERN = Pattern.compile(REG_SEPARATOR,
+ Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+ //
+ public static String START_DATE = "START_DATE";
+ public static String END_DATE = "END_DATE";
+ public static String ENV_KEY = "KEY";
+ //
+ private Map<String, Object> mapEnv = new HashMap<>();
+
+ public JoinedFormatter() {
+ }
+
+ public JoinedFormatter(IJoinedFlatTableDesc flatDesc) {
+ setDateEnv(flatDesc);
+ }
+
+ public void setDateEnv(IJoinedFlatTableDesc flatDesc) {
+ DataModelDesc model = flatDesc.getDataModel();
+ PartitionDesc partDesc = model.getPartitionDesc();
+ SegmentRange segRange = flatDesc.getSegRange();
+ long startInclusive = (Long) segRange.start.v;
+ long endExclusive = (Long) segRange.end.v;
+ //
+ String startDate = "";
+ String endDate = "";
+ String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
+ if (StringUtils.isBlank(partitionColumnDateFormat)) {
+ startDate = String.valueOf(startInclusive);
+ endDate = String.valueOf(endExclusive);
+ } else {
+ startDate = DateFormat.formatToDateStr(startInclusive,
partitionColumnDateFormat);
+ endDate = DateFormat.formatToDateStr(endExclusive,
partitionColumnDateFormat);
+ }
+ setKeyValue(START_DATE, startDate);
+ setKeyValue(END_DATE, endDate);
+ }
+
+ public Object getValue(String key) {
+ String fmtKey = StringUtils.trimToEmpty(key).toUpperCase(Locale.ROOT);
+ Object value = mapEnv.get(fmtKey);
+ return value == null ? "" : value;
+ }
+
+ public String formatSentense(String sentense) {
+ String[] cArray = REG_PATTERN.split(sentense);
+ StringBuilder sbr = new StringBuilder();
+ List<String> keys = getKeys(sentense);
+ int length = Math.max(cArray.length, keys.size());
+ for (int i = 0; i < length; i++) {
+ if (i < cArray.length) {
+ sbr.append(cArray[i]);
+ }
+ if (i < keys.size()) {
+ sbr.append(getValue(keys.get(i)));
+ }
+ }
+ return sbr.toString();
+ }
+
+ public List<String> getKeys(String condition) {
+ List<String> keys = new ArrayList<>();
+ Matcher matcher = REG_PATTERN.matcher(condition);
+ while (matcher.find()) {
+ keys.add(matcher.group(ENV_KEY));
+ }
+ return keys;
+ }
+
+ public void setKeyValue(String key, Object value) {
+ String fmtKey = StringUtils.trimToEmpty(key).toUpperCase(Locale.ROOT);
+ mapEnv.put(fmtKey, value);
+ }
+
+ public void setStartDate(String dateStr) {
+ setKeyValue(START_DATE, dateStr);
+ }
+
+ public void setEndDate(String dateStr) {
+ setKeyValue(END_DATE, dateStr);
+ }
+
+ public void printEnv() {
+ System.out.println(mapEnv);
+ }
+}
diff --git
a/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
b/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
new file mode 100644
index 0000000..6077011
--- /dev/null
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFormatterTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Zhixin Liu (liuzx32)
+ */
+public class JoinedFormatterTest extends LocalFileMetadataTestCase {
+
+ private JoinedFormatter formatter;
+
+ @Before
+ public void setUp() throws Exception {
+ formatter = new JoinedFormatter();
+ formatter.setStartDate("20190710");
+ formatter.setEndDate("20190711");
+ }
+
+ @After
+ public void after() throws Exception {
+ formatter.printEnv();
+ }
+
+ @Test
+ public void testConditionFormat() {
+ String expected = "date_str>='20190710' and date_str<'20190711'";
+ String condition = "date_str>='${start_date}' and
date_str<'${end_date}'";
+ String fmtCondition = formatter.formatSentense(condition);
+ assertEquals(expected, fmtCondition);
+ }
+
+ @Test
+ public void testSqlFormat() {
+ String expected = "select * from table where date_str>=20190710 and
date_str<20190711";
+ String sql = "select * from table where date_str>=${start_date} and
date_str<${end_date}";
+ String fmtSql = formatter.formatSentense(sql);
+ assertEquals(expected, fmtSql);
+ }
+}
