This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a0ef1895232ab36c3155041dc65a53bd218df603
Author: Zhong, Yanghong <nju_y...@apache.org>
AuthorDate: Mon May 11 15:56:11 2020 +0800
KYLIN-4487 Backend support for auto-migration to allow user to do cube
migration by self service
---
.../org/apache/kylin/common/KylinConfigBase.java | 39 ++
.../apache/kylin/common/restclient/RestClient.java | 34 +-
.../java/org/apache/kylin/job/JoinedFlatTable.java | 3 +-
cube-migration/pom.xml | 169 ++++++++
.../kylin/rest/controller/MigrationController.java | 167 ++++++++
.../kylin/rest/exception/ConflictException.java | 44 ++
.../rest/exception/RuleValidationException.java | 47 +++
.../kylin/rest/request/MigrationRequest.java | 49 +++
.../kylin/rest/service/MigrationRuleSet.java | 469 +++++++++++++++++++++
.../kylin/rest/service/MigrationService.java | 225 ++++++++++
.../kylin/rest/util/MailNotificationUtil.java | 33 ++
.../mail_templates/MIGRATION_APPROVED.ftl | 189 +++++++++
.../mail_templates/MIGRATION_COMPLETED.ftl | 192 +++++++++
.../resources/mail_templates/MIGRATION_FAILED.ftl | 220 ++++++++++
.../mail_templates/MIGRATION_REJECTED.ftl | 196 +++++++++
.../resources/mail_templates/MIGRATION_REQUEST.ftl | 192 +++++++++
pom.xml | 6 +
.../rest/service/ModelSchemaUpdateChecker.java | 8 +-
.../apache/kylin/rest/service/ModelService.java | 21 +-
.../rest/service/TableSchemaUpdateChecker.java | 4 +-
.../apache/kylin/rest/service/TableService.java | 9 +
server/pom.xml | 14 +
server/src/main/resources/kylinSecurity.xml | 3 +
.../tool/migration/CompatibilityCheckRequest.java | 51 +++
.../kylin/tool/query/ProbabilityGenerator.java | 88 ++++
.../kylin/tool/query/ProbabilityGeneratorCLI.java | 99 +++++
.../apache/kylin/tool/query/QueryGenerator.java | 189 +++++++++
.../apache/kylin/tool/query/QueryGeneratorCLI.java | 138 ++++++
.../tool/query/ProbabilityGeneratorCLITest.java | 50 +++
.../kylin/tool/query/QueryGeneratorCLITest.java | 54 +++
30 files changed, 2991 insertions(+), 11 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 13e73d9..ff6138a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2361,6 +2361,45 @@ public abstract class KylinConfigBase implements
Serializable {
}
//
============================================================================
+ // CUBE MIGRATION
+ //
============================================================================
+ public int getMigrationRuleExpansionRateThreshold() {
+ return
Integer.parseInt(getOptional("kylin.cube.migration.expansion-rate", "5"));
+ }
+
+ public int getMigrationRuleQueryGeneratorMaxDimensions() {
+ return
Integer.parseInt(getOptional("kylin.cube.migration.query-generator-max-dimension-number",
"3"));
+ }
+
+ public int getMigrationRuleQueryLatency() {
+ return 1000 *
Integer.parseInt(getOptional("kylin.cube.migration.query-latency-seconds",
"2"));
+ }
+
+ public int getMigrationRuleQueryLatencyMaxThreads() {
+ return
Integer.parseInt(getOptional("kylin.cube.migration.query-latency-max-threads",
"5"));
+ }
+
+ public int getMigrationRuleQueryEvaluationIteration() {
+ return
Integer.parseInt(getOptional("kylin.cube.migration.query-latency-iteration",
"5"));
+ }
+
+ public String getMigrationLocalAddress() {
+ return getOptional("kylin.cube.migration.source-address",
"localhost:80");
+ }
+
+ public String getMigrationTargetAddress() {
+ return getOptional("kylin.cube.migration.target-address",
"sandbox:80");
+ }
+
+ public String getMigrationEmailSuffix() {
+ return getOptional("kylin.cube.migration.mail-suffix", "@mail.com");
+ }
+
+ public boolean isMigrationApplyQueryLatencyRule() {
+ return
Boolean.parseBoolean(getOptional("kylin.cube.migration.rule-query-latency-enabled",
"true"));
+ }
+
+ //
============================================================================
// tool
//
============================================================================
public boolean isAllowAutoMigrateCube() {
diff --git
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 955b0ff..a9971dd 100644
---
a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++
b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -30,6 +30,7 @@ import java.util.regex.Pattern;
import javax.xml.bind.DatatypeConverter;
+import com.google.common.base.Strings;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
@@ -207,12 +208,13 @@ public class RestClient {
try {
response = client.execute(request);
String msg = EntityUtils.toString(response.getEntity());
- Map<String, String> map = JsonUtil.readValueAsMap(msg);
- msg = map.get("config");
if (response.getStatusLine().getStatusCode() != 200)
throw new IOException(INVALID_RESPONSE +
response.getStatusLine().getStatusCode()
+ " with cache wipe url " + url + "\n" + msg);
+
+ Map<String, String> map = JsonUtil.readValueAsMap(msg);
+ msg = map.get("config");
return msg;
} finally {
cleanup(request, response);
@@ -350,6 +352,26 @@ public class RestClient {
return content;
}
+ public void checkCompatibility(String jsonRequest) throws IOException {
+ checkCompatibility(jsonRequest, baseUrl + "/cubes/checkCompatibility");
+ }
+
+ private void checkCompatibility(String jsonRequest, String url) throws
IOException {
+ HttpPost post = newPost(url);
+ try {
+ post.setEntity(new StringEntity(jsonRequest, "UTF-8"));
+ HttpResponse response = client.execute(post);
+ if (response.getStatusLine().getStatusCode() != 200) {
+ String msg = getContent(response);
+ Map<String, String> kvMap = JsonUtil.readValueAsMap(msg);
+ String exception = kvMap.containsKey("exception") ?
kvMap.get("exception") : "unknown";
+ throw new IOException(exception);
+ }
+ } finally {
+ post.releaseConnection();
+ }
+ }
+
private HashMap dealResponse(HttpResponse response) throws IOException {
if (response.getStatusLine().getStatusCode() != 200) {
throw new IOException(INVALID_RESPONSE +
response.getStatusLine().getStatusCode());
@@ -362,9 +384,11 @@ public class RestClient {
private void addHttpHeaders(HttpRequestBase method) {
method.addHeader("Accept", "application/json, text/plain, */*");
method.addHeader("Content-Type", APPLICATION_JSON);
- String basicAuth = DatatypeConverter
- .printBase64Binary((this.userName + ":" +
this.password).getBytes(StandardCharsets.UTF_8));
- method.addHeader("Authorization", "Basic " + basicAuth);
+ if (!Strings.isNullOrEmpty(this.userName) &&
!Strings.isNullOrEmpty(this.password)) {
+ String basicAuth = DatatypeConverter
+ .printBase64Binary((this.userName + ":" +
this.password).getBytes(StandardCharsets.UTF_8));
+ method.addHeader("Authorization", "Basic " + basicAuth);
+ }
}
private HttpPost newPost(String url) {
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 2c4bc6a..27e25ba 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -161,7 +161,8 @@ public class JoinedFlatTable {
return sql.toString();
}
- static void appendJoinStatement(IJoinedFlatTableDesc flatDesc,
StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
+ public static void appendJoinStatement(IJoinedFlatTableDesc flatDesc,
StringBuilder sql, boolean singleLine,
+ SqlDialect sqlDialect) {
final String sep = singleLine ? " " : "\n";
Set<TableRef> dimTableCache = new HashSet<>();
diff --git a/cube-migration/pom.xml b/cube-migration/pom.xml
new file mode 100755
index 0000000..4f75de5
--- /dev/null
+++ b/cube-migration/pom.xml
@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-cube-migration</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Kylin - Cube Migration Service</name>
+ <description>Apache Kylin - Cube Migration Service</description>
+
+ <parent>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin</artifactId>
+ <version>3.1.0-SNAPSHOT</version>
+ </parent>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-server-base</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metadata</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-tool</artifactId>
+ </dependency>
+
+ <!-- Spring Core -->
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-webmvc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ </dependency>
+
+ <!-- Spring Security -->
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-acl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-ldap</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security.extensions</groupId>
+ <artifactId>spring-security-saml2-core</artifactId>
+ </dependency>
+
+ <!-- spring aop -->
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjweaver</artifactId>
+ </dependency>
+
+ <!-- Test & Env -->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ <!--MRUnit relies on older version of mockito, so cannot manage it
globally-->
+ <version>${mockito.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-catalina</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <repositories>
+ <repository>
+ <id>spring-snapshots</id>
+ <url>https://repo.spring.io/libs-snapshot</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ <pluginRepositories>
+ <pluginRepository>
+ <id>spring-snapshots</id>
+ <url>https://repo.spring.io/libs-snapshot</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </pluginRepository>
+ </pluginRepositories>
+</project>
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/controller/MigrationController.java
b/cube-migration/src/main/java/org/apache/kylin/rest/controller/MigrationController.java
new file mode 100644
index 0000000..9588e51
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/controller/MigrationController.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.ConflictException;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.MigrationRequest;
+import org.apache.kylin.rest.service.MigrationRuleSet;
+import org.apache.kylin.rest.service.MigrationService;
+import org.apache.kylin.rest.service.ModelService;
+import org.apache.kylin.rest.service.QueryService;
+import org.apache.kylin.rest.service.TableService;
+import org.apache.kylin.tool.migration.CompatibilityCheckRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+/**
+ * Restful api for cube migration.
+ */
+@Controller
+@RequestMapping(value = "/cubes")
+public class MigrationController extends BasicController {
+ private static final Logger logger =
LoggerFactory.getLogger(MigrationController.class);
+
+ @Autowired
+ private MigrationService migrationService;
+
+ @Autowired
+ private QueryService queryService;
+
+ @Autowired
+ private ModelService modelService;
+
+ @Autowired
+ private TableService tableService;
+
+ private final String targetHost =
KylinConfig.getInstanceFromEnv().getMigrationTargetAddress();
+
+ private CubeInstance getCubeInstance(String cubeName) {
+ CubeInstance cube = queryService.getCubeManager().getCube(cubeName);
+ if (cube == null) {
+ throw new BadRequestException("Cannot find cube " + cubeName);
+ }
+ return cube;
+ }
+
+ @RequestMapping(value = "/{cubeName}/migrateRequest", method = {
RequestMethod.PUT })
+ @ResponseBody
+ public String requestMigration(@PathVariable String cubeName, @RequestBody
MigrationRequest request) {
+ CubeInstance cube = getCubeInstance(cubeName);
+ try {
+ MigrationRuleSet.Context ctx = new
MigrationRuleSet.Context(queryService, cube,
+ getTargetHost(request.getTargetHost()),
request.getProjectName());
+ migrationService.requestMigration(cube, ctx);
+ } catch (Exception e) {
+ logger.error("Request migration failed.", e);
+ throw new BadRequestException(e.getMessage());
+ }
+ return "ok";
+ }
+
+ @RequestMapping(value = "/{cubeName}/migrateReject", method = {
RequestMethod.PUT })
+ @ResponseBody
+ public void reject(@PathVariable String cubeName, @RequestBody
MigrationRequest request) {
+ boolean reject = migrationService.reject(cubeName,
request.getProjectName(), request.getReason());
+ if (!reject) {
+ throw new InternalErrorException("Email send out failed. See
logs.");
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}/migrateApprove", method = {
RequestMethod.PUT })
+ @ResponseBody
+ public String approve(@PathVariable String cubeName, @RequestBody
MigrationRequest request) {
+ CubeInstance cube = getCubeInstance(cubeName);
+ try {
+ MigrationRuleSet.Context ctx = new
MigrationRuleSet.Context(queryService, cube,
+ getTargetHost(request.getTargetHost()),
request.getProjectName());
+ migrationService.approve(cube, ctx);
+ } catch (Exception e) {
+ throw new BadRequestException(e.getMessage());
+ }
+ return "Cube " + cubeName + " migrated.";
+ }
+
+ private String getTargetHost(String targetHost) {
+ return Strings.isNullOrEmpty(targetHost) ? this.targetHost :
targetHost;
+ }
+
+ /**
+ * Check the schema compatibility for table, model desc
+ */
+ @RequestMapping(value = "/checkCompatibility", method = {
RequestMethod.POST })
+ @ResponseBody
+ public void checkCompatibility(@RequestBody CompatibilityCheckRequest
request) {
+ try {
+ List<TableDesc> tableDescList = deserializeTableDescList(request);
+ for (TableDesc tableDesc : tableDescList) {
+ logger.info("Schema compatibility check for table {}",
tableDesc.getName());
+ tableService.checkTableCompatibility(request.getProjectName(),
tableDesc);
+ logger.info("Pass schema compatibility check for table {}",
tableDesc.getName());
+ }
+ DataModelDesc dataModelDesc =
JsonUtil.readValue(request.getModelDescData(), DataModelDesc.class);
+ logger.info("Schema compatibility check for model {}",
dataModelDesc.getName());
+ modelService.checkModelCompatibility(request.getProjectName(),
dataModelDesc, tableDescList);
+ logger.info("Pass schema compatibility check for model {}",
dataModelDesc.getName());
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new ConflictException(e.getMessage(), e);
+ }
+ }
+
+ private List<TableDesc> deserializeTableDescList(CompatibilityCheckRequest
request) {
+ List<TableDesc> result = Lists.newArrayList();
+ try {
+ for (String tableDescData : request.getTableDescDataList()) {
+ TableDesc tableDesc = JsonUtil.readValue(tableDescData,
TableDesc.class);
+ for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+ columnDesc.init(tableDesc);
+ }
+ result.add(tableDesc);
+ }
+ } catch (JsonParseException | JsonMappingException e) {
+ throw new BadRequestException("Fail to parse table description: "
+ e);
+ } catch (IOException e) {
+ throw new InternalErrorException("Failed to deal with the
request:" + e.getMessage(), e);
+ }
+ return result;
+ }
+}
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/exception/ConflictException.java
b/cube-migration/src/main/java/org/apache/kylin/rest/exception/ConflictException.java
new file mode 100644
index 0000000..c6a8f0a
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/exception/ConflictException.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseStatus(value = HttpStatus.CONFLICT)
+public class ConflictException extends RuntimeException {
+ private static final long serialVersionUID = 1L;
+
+ public ConflictException() {
+ super();
+ }
+
+ public ConflictException(String message) {
+ super(message);
+ }
+
+ public ConflictException(Throwable cause) {
+ super(cause);
+ }
+
+ public ConflictException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/exception/RuleValidationException.java
b/cube-migration/src/main/java/org/apache/kylin/rest/exception/RuleValidationException.java
new file mode 100644
index 0000000..5eff71d
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/exception/RuleValidationException.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+/**
+ * Throw RuleValidationException if cube breaks the rules during the migration.
+ */
+@ResponseStatus(value = HttpStatus.BAD_REQUEST)
+public class RuleValidationException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public RuleValidationException() {
+ super();
+ }
+
+ public RuleValidationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public RuleValidationException(String message) {
+ super(message);
+ }
+
+ public RuleValidationException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/request/MigrationRequest.java
b/cube-migration/src/main/java/org/apache/kylin/rest/request/MigrationRequest.java
new file mode 100644
index 0000000..4e8c257
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/request/MigrationRequest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.request;
+
+public class MigrationRequest {
+ private String targetHost;
+ private String projectName;
+ private String reason; // reject reason
+
+ public String getTargetHost() {
+ return targetHost;
+ }
+
+ public void setTargetHost(String targetHost) {
+ this.targetHost = targetHost;
+ }
+
+ public String getProjectName() {
+ return projectName;
+ }
+
+ public void setProjectName(String projectName) {
+ this.projectName = projectName;
+ }
+
+ public String getReason() {
+ return reason;
+ }
+
+ public void setReason(String reason) {
+ this.reason = reason;
+ }
+}
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/service/MigrationRuleSet.java
b/cube-migration/src/main/java/org/apache/kylin/rest/service/MigrationRuleSet.java
new file mode 100644
index 0000000..de158bd
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/service/MigrationRuleSet.java
@@ -0,0 +1,469 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.TableMetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.RuleValidationException;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.apache.kylin.source.SourceManager;
+import org.apache.kylin.tool.migration.CompatibilityCheckRequest;
+import org.apache.kylin.tool.query.QueryGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.SetMultimap;
+import com.google.common.collect.Sets;
+
+/**
+ * Check the pre-defined rules. If not pass, we will throw
+ * {@link RuleValidationException}.
+ */
+public class MigrationRuleSet {
+
+ private static final Logger logger =
LoggerFactory.getLogger(MigrationRuleSet.class);
+
+ public static final Rule DEFAULT_HIVE_TABLE_CONSISTENCY_RULE = new
HiveTableConsistencyRule();
+ public static final Rule DEFAULT_CUBE_STATUS_RULE = new CubeStatusRule();
+ public static final Rule DEFAULT_PROJECT_EXIST_RULE = new
ProjectExistenceRule();
+ public static final Rule DEFAULT_AUTO_MERGE_RULE = new
AutoMergePolicyRule();
+ public static final Rule DEFAULT_EXPANSION_RULE = new ExpansionRateRule();
+ public static final Rule DEFAULT_EMAIL_NOTIFY_RULE = new
NotificationEmailRule();
+ public static final Rule DEFAULT_COMPATIBLE_RULE = new CompatibleRule();
+ public static final Rule DEFAULT_SEGMENT_RULE = new SegmentRule();
+ public static final Rule DEFAULT_CUBE_OVERWRITE_RULE = new
CubeOverwriteRule();
+ public static final Rule DEFAULT_QUERY_LATENCY_RULE = new
QueryLatencyRule();
+
+ private static List<Rule> MUSTTOPASS_RULES = Lists.newLinkedList();
+
+ private static List<Rule> NICETOPASS_RULES = Lists.newLinkedList();
+
+ /**
+ * Register mandatory rules.
+ * @param rules
+ */
+ public static synchronized void register(Rule... rules) {
+ register(true, rules);
+ }
+
+ public static synchronized void register(boolean mandatory, Rule... rules)
{
+ if (mandatory) {
+ for (Rule rule : rules) {
+ MUSTTOPASS_RULES.add(rule);
+ }
+ } else {
+ for (Rule rule : rules) {
+ NICETOPASS_RULES.add(rule);
+ }
+ }
+ }
+
+ // initialize default rules
+ static {
+ register(DEFAULT_HIVE_TABLE_CONSISTENCY_RULE,
DEFAULT_CUBE_STATUS_RULE, DEFAULT_PROJECT_EXIST_RULE,
+ DEFAULT_EMAIL_NOTIFY_RULE, DEFAULT_SEGMENT_RULE,
DEFAULT_CUBE_OVERWRITE_RULE, DEFAULT_COMPATIBLE_RULE);
+ register(false, DEFAULT_AUTO_MERGE_RULE, DEFAULT_EXPANSION_RULE,
DEFAULT_QUERY_LATENCY_RULE);
+ }
+
+ /**
+ * @param ctx
+ * @return warn message if fail to pass some nice to have rules
+ * @throws RuleValidationException
+ */
+ public static String apply(Context ctx) throws RuleValidationException {
+ for (Rule rule : MUSTTOPASS_RULES) {
+ rule.apply(ctx);
+ }
+ StringBuilder sb = new StringBuilder();
+ for (Rule rule : NICETOPASS_RULES) {
+ try {
+ rule.apply(ctx);
+ } catch (RuleValidationException e) {
+ sb.append(e.getMessage());
+ sb.append("\n");
+ }
+ }
+ return sb.toString();
+ }
+
+ public interface Rule {
+ /**
+ * Apply the rule, success if no exception is thrown.
+ *
+ * @param ctx
+ * @throws RuleValidationException
+ * if broke this rule
+ */
+ public void apply(Context ctx) throws RuleValidationException;
+ }
+
+ private static class ProjectExistenceRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ // code from CubeCopyCLI.java
+ ResourceStore dstStore = ctx.getTargetResourceStore();
+ String projectResPath =
ProjectInstance.concatResourcePath(ctx.getTgtProjectName());
+ try {
+ if (!dstStore.exists(projectResPath)) {
+ throw new RuleValidationException("The target project " +
ctx.getTgtProjectName()
+ + " does not exist on " + ctx.getTargetAddress());
+ }
+ } catch (RuleValidationException e) {
+ throw e;
+ } catch (IOException e) {
+ throw new RuleValidationException("Internal error: " +
e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class AutoMergePolicyRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ CubeDesc cubeDesc = ctx.getCubeInstance().getDescriptor();
+ long[] timeRanges = cubeDesc.getAutoMergeTimeRanges();
+ if (timeRanges == null || timeRanges.length == 0) {
+ throw new RuleValidationException(String.format(Locale.ROOT,
+ "Auto merge time range for cube %s is not set.",
cubeDesc.getName()));
+ }
+ }
+ }
+
+ private static class ExpansionRateRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ int expansionRateThr =
KylinConfig.getInstanceFromEnv().getMigrationRuleExpansionRateThreshold();
+
+ CubeInstance cube = ctx.getCubeInstance();
+ if (cube.getInputRecordSizeBytes() == 0 || cube.getSizeKB() == 0) {
+ logger.warn("cube {} has zero input record size.",
cube.getName());
+ throw new RuleValidationException(String.format(Locale.ROOT,
"Cube %s is not built.", cube.getName()));
+ }
+ double expansionRate = cube.getSizeKB() * 1024.0 /
cube.getInputRecordSizeBytes();
+ if (expansionRate > expansionRateThr) {
+ logger.info(
+ "cube {}, size_kb {}, cube record size {}, cube
expansion rate {} larger than threshold {}.",
+ cube.getName(), cube.getSizeKB(),
cube.getInputRecordSizeBytes(), expansionRate,
+ expansionRateThr);
+ throw new RuleValidationException(
+ "ExpansionRateRule: failed on expansion rate check
with exceeding " + expansionRateThr);
+ }
+ }
+ }
+
+ private static class NotificationEmailRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ CubeDesc cubeDesc = ctx.getCubeInstance().getDescriptor();
+ List<String> notifyList = cubeDesc.getNotifyList();
+ if (notifyList == null || notifyList.size() == 0) {
+ throw new RuleValidationException("Cube email notification
list is not set or empty.");
+ }
+ }
+ }
+
+ private static class CompatibleRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ try {
+ checkSchema(ctx);
+ } catch (Exception e) {
+ throw new RuleValidationException(e.getMessage(), e);
+ }
+ }
+
+ public void checkSchema(Context ctx) throws IOException {
+ Set<TableDesc> tableSet = Sets.newHashSet();
+ for (TableRef tableRef :
ctx.getCubeInstance().getModel().getAllTables()) {
+ tableSet.add(tableRef.getTableDesc());
+ }
+
+ List<String> tableDataList = Lists.newArrayList();
+ for (TableDesc table : tableSet) {
+ tableDataList.add(JsonUtil.writeValueAsIndentString(table));
+ }
+
+ DataModelDesc model = ctx.getCubeInstance().getModel();
+ String modelDescData = JsonUtil.writeValueAsIndentString(model);
+
+ CompatibilityCheckRequest request = new
CompatibilityCheckRequest();
+ request.setProjectName(ctx.getTgtProjectName());
+ request.setTableDescDataList(tableDataList);
+ request.setModelDescData(modelDescData);
+
+ String jsonRequest = JsonUtil.writeValueAsIndentString(request);
+ RestClient client = new RestClient(ctx.getTargetAddress());
+ client.checkCompatibility(jsonRequest);
+ }
+ }
+
+ private static class SegmentRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ List<CubeSegment> segments =
ctx.getCubeInstance().getSegments(SegmentStatusEnum.READY);
+ if (segments == null || segments.size() == 0) {
+ throw new RuleValidationException("No built segment found.");
+ }
+ }
+ }
+
+ private static class CubeOverwriteRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ ResourceStore dstStore = ctx.getTargetResourceStore();
+ CubeInstance cube = ctx.getCubeInstance();
+ try {
+ if (dstStore.exists(cube.getResourcePath()))
+ throw new RuleValidationException("The cube named " +
cube.getName()
+ + " already exists on target metadata store.
Please delete it firstly and try again");
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ throw new RuleValidationException(e.getMessage(), e);
+ }
+ }
+ }
+
+ private static class CubeStatusRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ CubeInstance cube = ctx.getCubeInstance();
+ RealizationStatusEnum status = cube.getStatus();
+ if (status != RealizationStatusEnum.READY) {
+ throw new RuleValidationException("The cube named " +
cube.getName() + " is not in READY state.");
+ }
+ }
+ }
+
+ private static class QueryLatencyRule implements Rule {
+
+ @Override
+ public void apply(MigrationRuleSet.Context ctx) throws
RuleValidationException {
+ logger.info("QueryLatencyRule started.");
+ CubeInstance cube = ctx.getCubeInstance();
+
+ int latency =
KylinConfig.getInstanceFromEnv().getMigrationRuleQueryLatency();
+ int iteration =
KylinConfig.getInstanceFromEnv().getMigrationRuleQueryEvaluationIteration();
+ int maxDimension =
cube.getConfig().getMigrationRuleQueryGeneratorMaxDimensions();
+
+ try {
+ List<String> queries =
QueryGenerator.generateQueryList(cube.getDescriptor(), iteration, maxDimension);
+ assert queries.size() > 0;
+ long avg = executeQueries(queries, ctx);
+ logger.info("QueryLatencyRule ended: average time cost " + avg
+ "ms.");
+ if (avg > latency) {
+ throw new RuleValidationException(
+ "Failed on query latency check with average cost "
+ avg + " exceeding " + latency + "ms.");
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ if (e instanceof RuleValidationException) {
+ throw (RuleValidationException) e;
+ } else {
+ throw new RuleValidationException(e.getMessage(), e);
+ }
+ }
+ }
+
+ private long executeQueries(final List<String> queries, final Context
ctx) throws Exception {
+ int maxThreads =
KylinConfig.getInstanceFromEnv().getMigrationRuleQueryLatencyMaxThreads();
+ int threadNum = Math.min(maxThreads, queries.size());
+ ExecutorService threadPool =
Executors.newFixedThreadPool(threadNum);
+ CompletionService<Long> completionService = new
ExecutorCompletionService<Long>(threadPool);
+ final Authentication auth =
SecurityContextHolder.getContext().getAuthentication();
+ long start = System.currentTimeMillis();
+ for (final String query : queries) {
+ completionService.submit(new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+
SecurityContextHolder.getContext().setAuthentication(auth);
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(ctx.getSrcProjectName());
+ sqlRequest.setSql(query);
+ SQLResponse sqlResponse =
ctx.getQueryService().doQueryWithCache(sqlRequest, false);
+ if (sqlResponse.getIsException()) {
+ throw new
RuleValidationException(sqlResponse.getExceptionMessage());
+ }
+ return sqlResponse.getDuration();
+ }
+
+ });
+ }
+ long timeCostSum = 0L;
+ for (int i = 0; i < queries.size(); ++i) {
+ try {
+ timeCostSum += completionService.take().get();
+ } catch (InterruptedException | ExecutionException e) {
+ threadPool.shutdownNow();
+ throw e;
+ }
+ }
+ long end = System.currentTimeMillis();
+ logger.info("Execute" + queries.size() + " queries took " + (end -
start) + " ms, query time cost sum "
+ + timeCostSum + " ms.");
+ return timeCostSum / queries.size();
+ }
+ }
+
+ // check if table schema on Kylin is updated to date with external Hive
table
+ private static class HiveTableConsistencyRule implements Rule {
+
+ @Override
+ public void apply(Context ctx) throws RuleValidationException {
+ // de-dup
+ SetMultimap<String, String> db2tables =
LinkedHashMultimap.create();
+ for (TableRef tableRef :
ctx.getCubeInstance().getModel().getAllTables()) {
+
db2tables.put(tableRef.getTableDesc().getDatabase().toUpperCase(Locale.ROOT),
+
tableRef.getTableDesc().getName().toUpperCase(Locale.ROOT));
+ }
+
+ // load all tables first
+ List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList();
+ ISourceMetadataExplorer explr =
SourceManager.getDefaultSource().getSourceMetadataExplorer();
+ try {
+ for (Map.Entry<String, String> entry : db2tables.entries()) {
+ Pair<TableDesc, TableExtDesc> pair =
explr.loadTableMetadata(entry.getKey(), entry.getValue(),
+ ctx.getSrcProjectName());
+ TableDesc tableDesc = pair.getFirst();
+
Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey()));
+
Preconditions.checkState(tableDesc.getName().equals(entry.getValue()));
+
Preconditions.checkState(tableDesc.getIdentity().equals(entry.getKey() + "." +
entry.getValue()));
+ TableExtDesc extDesc = pair.getSecond();
+
Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity()));
+ allMeta.add(pair);
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new RuleValidationException(
+ "Internal error when checking
HiveTableConsistencyRule: " + e.getMessage());
+ }
+
+ // do schema check
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ TableSchemaUpdateChecker checker = new
TableSchemaUpdateChecker(TableMetadataManager.getInstance(config),
+ CubeManager.getInstance(config),
DataModelManager.getInstance(config));
+ for (Pair<TableDesc, TableExtDesc> pair : allMeta) {
+ try {
+ TableSchemaUpdateChecker.CheckResult result =
checker.allowReload(pair.getFirst(),
+ ctx.getSrcProjectName());
+ result.raiseExceptionWhenInvalid();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new RuleValidationException("Table " +
pair.getFirst().getIdentity()
+ + " has incompatible changes on Hive, please
reload the hive table and update your model/cube if needed.");
+ }
+ }
+ logger.info("Cube " + ctx.getCubeInstance().getName() + " Hive
table consistency check passed.");
+ }
+
+ }
+
+ public static class Context {
+ private final QueryService queryService;
+ private final CubeInstance cubeInstance;
+ private final String targetAddress; // the target kylin host with port
+ private final ResourceStore targetResourceStore;
+ private final String tgtProjectName; // the target project name
+ private final String srcProjectName; // the source project name
+
+ public Context(QueryService queryService, CubeInstance cubeInstance,
String targetHost, String tgtProjectName) {
+ this.queryService = queryService;
+ this.cubeInstance = cubeInstance;
+ this.targetAddress = targetHost;
+ KylinConfig targetConfig =
KylinConfig.createInstanceFromUri(targetHost);
+ this.targetResourceStore = ResourceStore.getStore(targetConfig);
+ this.tgtProjectName = tgtProjectName;
+
+ List<ProjectInstance> projList =
ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
+ .findProjects(cubeInstance.getType(),
cubeInstance.getName());
+ if (projList.size() != 1) {
+ throw new InternalErrorException("Cube " +
cubeInstance.getName()
+ + " should belong to only one project. However, it's
belong to " + projList);
+ }
+ this.srcProjectName = projList.get(0).getName();
+ }
+
+ public QueryService getQueryService() {
+ return queryService;
+ }
+
+ public CubeInstance getCubeInstance() {
+ return cubeInstance;
+ }
+
+ public String getTargetAddress() {
+ return targetAddress;
+ }
+
+ public ResourceStore getTargetResourceStore() {
+ return targetResourceStore;
+ }
+
+ public String getTgtProjectName() {
+ return tgtProjectName;
+ }
+
+ public String getSrcProjectName() {
+ return srcProjectName;
+ }
+ }
+}
\ No newline at end of file
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/service/MigrationService.java
b/cube-migration/src/main/java/org/apache/kylin/rest/service/MigrationService.java
new file mode 100644
index 0000000..ca1e71b
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/service/MigrationService.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.service;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.AclEntity;
+import org.apache.kylin.common.util.MailService;
+import org.apache.kylin.common.util.MailTemplateProvider;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.RuleValidationException;
+import org.apache.kylin.rest.util.MailNotificationUtil;
+import org.apache.kylin.tool.CubeMigrationCLI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.security.acls.domain.PrincipalSid;
+import org.springframework.security.acls.model.AccessControlEntry;
+import org.springframework.security.acls.model.Acl;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * Provide migration logic implementation.
+ */
+@Component("migrationService")
+public class MigrationService extends BasicService {
+
+ private static final Logger logger =
LoggerFactory.getLogger(MigrationService.class);
+
+ @Autowired
+ private AccessService accessService;
+
+ @Autowired
+ private CubeService cubeService;
+
+ private final String localHost =
KylinConfig.getInstanceFromEnv().getMigrationLocalAddress();
+ private final String envName =
KylinConfig.getInstanceFromEnv().getDeployEnv();
+
+ public String checkRule(MigrationRuleSet.Context context) throws
RuleValidationException {
+ return MigrationRuleSet.apply(context);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube,
'ADMINISTRATION')")
+ public void requestMigration(CubeInstance cube, MigrationRuleSet.Context
ctx) throws Exception {
+ Map<String, String> root = Maps.newHashMap();
+ root.put("projectname", ctx.getTgtProjectName());
+ root.put("cubename", ctx.getCubeInstance().getName());
+ root.put("status", "NEED APPROVE");
+ root.put("envname", envName);
+ sendMigrationMail(MailNotificationUtil.MIGRATION_REQUEST,
getEmailRecipients(cube), root);
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ public boolean reject(String cubeName, String projectName, String reason) {
+ try {
+ Map<String, String> root = Maps.newHashMap();
+ root.put("cubename", cubeName);
+ root.put("rejectedReason", reason);
+ root.put("status", "REJECTED");
+ root.put("envname", envName);
+
+ sendMigrationMail(MailNotificationUtil.MIGRATION_REJECTED,
getEmailRecipients(cubeName), root);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return false;
+ }
+ return true;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ public void approve(CubeInstance cube, MigrationRuleSet.Context ctx)
throws Exception {
+ checkRule(ctx);
+
+ String cubeName = cube.getName();
+ String projectName = ctx.getTgtProjectName();
+ try {
+ sendApprovedMailQuietly(cubeName, projectName);
+
+ // do cube migration
+ new CubeMigrationCLI().moveCube(localHost, ctx.getTargetAddress(),
cubeName, projectName, "true", "false",
+ "true", "true", "false");
+
+ sendCompletedMailQuietly(cubeName, projectName);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ sendMigrationFailedMailQuietly(cubeName, projectName,
e.getMessage());
+ throw e;
+ }
+ }
+
+ private boolean sendApprovedMailQuietly(String cubeName, String
projectName) {
+ try {
+ Map<String, String> root = Maps.newHashMap();
+ root.put("projectname", projectName);
+ root.put("cubename", cubeName);
+ root.put("status", "APPROVED");
+ root.put("envname", envName);
+
+ sendMigrationMail(MailNotificationUtil.MIGRATION_APPROVED,
getEmailRecipients(cubeName), root);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return false;
+ }
+ return true;
+ }
+
+ private boolean sendCompletedMailQuietly(String cubeName, String
projectName) {
+ try {
+ Map<String, String> root = Maps.newHashMap();
+ root.put("projectname", projectName);
+ root.put("cubename", cubeName);
+ root.put("status", "COMPLETED");
+ root.put("envname", envName);
+
+ sendMigrationMail(MailNotificationUtil.MIGRATION_COMPLETED,
getEmailRecipients(cubeName), root);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return false;
+ }
+ return true;
+ }
+
+ private boolean sendMigrationFailedMailQuietly(String cubeName, String
projectName, String reason) {
+ try {
+ Map<String, String> root = Maps.newHashMap();
+ root.put("projectname", projectName);
+ root.put("cubename", cubeName);
+ root.put("status", "FAILED");
+ root.put("failedReason", reason);
+ root.put("envname", envName);
+
+ sendMigrationMail(MailNotificationUtil.MIGRATION_FAILED,
getEmailRecipients(cubeName), root);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return false;
+ }
+ return true;
+ }
+
+ public List<String> getCubeAdmins(CubeInstance cubeInstance) {
+ ProjectInstance prjInstance = cubeInstance.getProjectInstance();
+ AclEntity ae = accessService.getAclEntity("ProjectInstance",
prjInstance.getUuid());
+ logger.info("ProjectUUID : " + prjInstance.getUuid());
+ Acl acl = accessService.getAcl(ae);
+
+ String mailSuffix =
KylinConfig.getInstanceFromEnv().getMigrationEmailSuffix();
+ List<String> cubeAdmins = Lists.newArrayList();
+ if (acl != null) {
+ for (AccessControlEntry ace : acl.getEntries()) {
+ if (ace.getPermission().getMask() == 16) {
+ PrincipalSid ps = (PrincipalSid) ace.getSid();
+ cubeAdmins.add(ps.getPrincipal() + mailSuffix);
+ }
+ }
+ }
+
+ if (cubeAdmins.isEmpty()) {
+ throw new BadRequestException("Cube access list is null, please
add at least one role in it.");
+ }
+ return cubeAdmins;
+ }
+
+ public List<String> getEmailRecipients(String cubeName) throws Exception {
+ CubeInstance cubeInstance =
cubeService.getCubeManager().getCube(cubeName);
+ return getEmailRecipients(cubeInstance);
+ }
+
+ public List<String> getEmailRecipients(CubeInstance cubeInstance) throws
Exception {
+ List<String> recipients = Lists.newArrayList();
+ recipients.addAll(getCubeAdmins(cubeInstance));
+ recipients.addAll(cubeInstance.getDescriptor().getNotifyList());
+ String[] adminDls = KylinConfig.getInstanceFromEnv().getAdminDls();
+ if (adminDls != null) {
+ recipients.addAll(Lists.newArrayList(adminDls));
+ }
+ return recipients;
+ }
+
+ public void sendMigrationMail(String state, List<String> recipients,
Map<String, String> root) {
+ String submitter =
SecurityContextHolder.getContext().getAuthentication().getName();
+ root.put("requester", submitter);
+
+ String title;
+
+ // No project name for rejected title
+ if (state == MailNotificationUtil.MIGRATION_REJECTED) {
+ title = MailNotificationUtil.getMailTitle("MIGRATION",
root.get("status"), root.get("envname"),
+ root.get("cubename"));
+ } else {
+ title = MailNotificationUtil.getMailTitle("MIGRATION",
root.get("status"), root.get("envname"),
+ root.get("projectname"), root.get("cubename"));
+ }
+
+ String content =
MailTemplateProvider.getInstance().buildMailContent(state,
+ Maps.<String, Object> newHashMap(root));
+
+ new MailService(KylinConfig.getInstanceFromEnv()).sendMail(recipients,
title, content);
+ }
+}
diff --git
a/cube-migration/src/main/java/org/apache/kylin/rest/util/MailNotificationUtil.java
b/cube-migration/src/main/java/org/apache/kylin/rest/util/MailNotificationUtil.java
new file mode 100644
index 0000000..6ef4e28
--- /dev/null
+++
b/cube-migration/src/main/java/org/apache/kylin/rest/util/MailNotificationUtil.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.util;
+
+import com.google.common.base.Joiner;
+
+public class MailNotificationUtil {
+ public static final String MIGRATION_REQUEST = "MIGRATION_REQUEST";
+ public static final String MIGRATION_REJECTED = "MIGRATION_REJECTED";
+ public static final String MIGRATION_APPROVED = "MIGRATION_APPROVED";
+ public static final String MIGRATION_COMPLETED = "MIGRATION_COMPLETED";
+ public static final String MIGRATION_FAILED = "MIGRATION_FAILED";
+
+ public static String getMailTitle(String... titleParts) {
+ return "[" + Joiner.on("]-[").join(titleParts) + "]";
+ }
+}
diff --git
a/cube-migration/src/main/resources/mail_templates/MIGRATION_APPROVED.ftl
b/cube-migration/src/main/resources/mail_templates/MIGRATION_APPROVED.ftl
new file mode 100644
index 0000000..03c76bf
--- /dev/null
+++ b/cube-migration/src/main/resources/mail_templates/MIGRATION_APPROVED.ftl
@@ -0,0 +1,189 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative;
charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="
+line-height: 1;font-size: 16px;">
+ <p style="text-align:left;">Dear ${requester},</p>
+ <p>Your cube migration request has been approved. Please wait for
migration completed. </p>
+</span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <span style="display: inline;
+ background-color: #337ab7;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Approved </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
style="border-collapse: collapse;border:1px solid #bce8f1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ border: 1px solid #bce8f1;
+ background-color: #d9edf7;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 14px;
+ color: #31708f;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Request detail
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica,
sans-serif;">
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Requester</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${requester}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Project Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${projectname}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Cube Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${cubename}</h4>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ </table>
+ <hr style="margin-top: 20px;
+margin-bottom: 20px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <h4 style="font-weight: 500;
+line-height: 1;font-size: 16px;">
+
+ <p>For any question, please contact support team <a
href="mailto:dl-ebay-kylin-c...@ebay.com "
+ style="color:
#337ab7;text-decoration: none;"><b>DL-eBay-Kylin-DISupport</b></a>
+ or Kylin <a href="https://ebay-eng.slack.com/messages/C1ZG2TN7J";
+ style="color: #337ab7;text-decoration:
none;"><b>Slack</b></a> channel.</p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git
a/cube-migration/src/main/resources/mail_templates/MIGRATION_COMPLETED.ftl
b/cube-migration/src/main/resources/mail_templates/MIGRATION_COMPLETED.ftl
new file mode 100644
index 0000000..58db588
--- /dev/null
+++ b/cube-migration/src/main/resources/mail_templates/MIGRATION_COMPLETED.ftl
@@ -0,0 +1,192 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative;
charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <span style="
+line-height: 1;font-size: 16px;">
+ <p style="text-align:left;">Dear ${requester},</p>
+ <p>Your cube migration request has completed. Please check the
cube in Kylin production.</p>
+ </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <span style="display: inline;
+ background-color: #5cb85c;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Completed </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+
+ <table cellpadding="0" cellspacing="0" width="100%"
style="border-collapse: collapse;border:1px solid #d6e9c6;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #dff0d8;
+ border:1px solid #d6e9c6;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 14px;
+ color: #3c763d;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Request detail
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica,
sans-serif;">
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Requester</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${requester}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style:
normal;">Project Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">${projectname}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Cube
Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">${cubename}</h4>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ height:0px;
+ border-top: 1px solid #eee;
+ border-right:0px;
+ border-bottom:0px;
+ border-left:0px;">
+ <h4 style="font-weight: 500;
+line-height: 1;font-size: 16px;">
+
+ <p>For any question, please contact support team
+ <a href="mailto:dl-ebay-kylin-c...@ebay.com " style="color:
#337ab7; text-decoration: none;">
+ <b>DL-eBay-Kylin-DISupport</b>
+ </a>
+ or Kylin
+ <a href="https://ebay-eng.slack.com/messages/C1ZG2TN7J";
style="color: #337ab7; text-decoration: none;">
+ <b>Slack</b>
+ </a> channel.</p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git
a/cube-migration/src/main/resources/mail_templates/MIGRATION_FAILED.ftl
b/cube-migration/src/main/resources/mail_templates/MIGRATION_FAILED.ftl
new file mode 100644
index 0000000..2a02f92
--- /dev/null
+++ b/cube-migration/src/main/resources/mail_templates/MIGRATION_FAILED.ftl
@@ -0,0 +1,220 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative;
charset=UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <span style="
+ line-height: 1;font-size: 16px;">
+ <p style="text-align:left;">Dear ${requester},</p>
+ <p>Your cube migration request has failed. Please engage support
team to check the reason. You can file a support
+ ticket through
+ <a href="http://go/dis/submit"; style="color:
#337ab7;text-decoration: none;">
+ <b>JIRA</b>
+ </a>.</p>
+ </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <span style="display: inline;
+ background-color: #d9534f;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Failed </span>
+ <hr style="margin-top: 10px;
+ margin-bottom: 10px;
+ height:0px;
+ border-top: 1px solid #eee;
+ border-right:0px;
+ border-bottom:0px;
+ border-left:0px;">
+
+ <table cellpadding="0" cellspacing="0" width="100%"
style="border-collapse: collapse;border:1px solid #ebccd1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ border:1px solid #ebccd1;
+ background-color: #f2dede;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 14px;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Request detail
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
style="margin-bottom: 20px;border:1 solid #ddd;border-collapse:
collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style:
normal;">Requester</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${requester}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style:
normal;">Project Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${projectname}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Cube
Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${cubename}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style:
normal;">Failed Reason</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${failedReason}</h4>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ height:0px;
+ border-top: 1px solid #eee;
+ border-right:0px;
+ border-bottom:0px;
+ border-left:0px;">
+ <h4 style="font-weight: 500;
+ line-height: 1;font-size:16px;">
+
+ <p>For any question, please contact support team
+ <a href="mailto:dl-ebay-kylin-c...@ebay.com " style="color:
#337ab7;text-decoration: none;">
+ <b>DL-eBay-Kylin-DISupport</b>
+ </a>
+ or Kylin
+ <a href="https://ebay-eng.slack.com/messages/C1ZG2TN7J";
style="color: #337ab7;text-decoration: none;">
+ <b>Slack</b>
+ </a> channel.</p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git
a/cube-migration/src/main/resources/mail_templates/MIGRATION_REJECTED.ftl
b/cube-migration/src/main/resources/mail_templates/MIGRATION_REJECTED.ftl
new file mode 100644
index 0000000..ea9eff6
--- /dev/null
+++ b/cube-migration/src/main/resources/mail_templates/MIGRATION_REJECTED.ftl
@@ -0,0 +1,196 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative;
charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <span style="line-height: 1;font-size: 16px;">
+ <p style="text-align:left;">Dear ${requester},</p>
+ <p>Your cube migration request has been rejected. Please check the
reason below and re-submit the request after
+ making related changes.</p>
+ </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <span style="display: inline;
+ background-color: #d9534f;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Rejected </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="border-collapse: collapse;border:1px solid
#ebccd1;table-layout:fixed">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 14px;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Request detail
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica,
sans-serif;">
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Requester</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${requester}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Cube Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${cubename}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Rejected
Reason</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ <pre style="white-space:
pre-wrap;">${rejectedReason}</pre>
+ </h4>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ </table>
+ <hr style="margin-top: 20px;
+margin-bottom: 20px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <h4 style="font-weight: 500;
+ line-height: 1;font-size:16px;">
+
+ <p>For any question, please contact support team
+ <a href="mailto:dl-ebay-kylin-c...@ebay.com " style="color:
#337ab7;text-decoration: none;">
+ <b>DL-eBay-Kylin-DISupport</b>
+ </a>
+ or Kylin
+ <a href="https://ebay-eng.slack.com/messages/C1ZG2TN7J";
style="color: #337ab7;text-decoration: none;">
+ <b>Slack</b>
+ </a> channel.</p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git
a/cube-migration/src/main/resources/mail_templates/MIGRATION_REQUEST.ftl
b/cube-migration/src/main/resources/mail_templates/MIGRATION_REQUEST.ftl
new file mode 100644
index 0000000..6c50f9c
--- /dev/null
+++ b/cube-migration/src/main/resources/mail_templates/MIGRATION_REQUEST.ftl
@@ -0,0 +1,192 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative;
charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="ont-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="
+ line-height: 1;font-size: 16px;">
+ <p style="text-align:left;">Dear ${requester},</p>
+ <p>Your cube migration request has been submitted. It has been sent to
Kylin PM for approval. Please stay tuned.</p>
+</span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <span style="display: inline;
+ background-color: #337ab7;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Waiting for approval </span>
+ <hr style="margin-top: 10px;
+margin-bottom: 10px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+
+ <table cellpadding="0" cellspacing="0" width="100%"
style="border-collapse: collapse;border:1px solid #bce8f1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ border-bottom: 1px solid transparent;
+ background-color: #d9edf7;
+ border-color: #bce8f1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 14px;
+ color: #31708f;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Request detail
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid
#ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica,
sans-serif;">
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Requester</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${requester}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Project Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${projectname}</h4>
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="
+ margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;">Cube Name</h4>
+ </th>
+ <td style="border: 1px solid #ddd;
+ padding: 8px;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ line-height: 1.5;
+ text-align: left;
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 300;">
+ ${cubename}</h4>
+ </td>
+ </tr>
+
+ </table>
+ </td>
+ </tr>
+
+ </table>
+ <hr style="margin-top: 20px;
+margin-bottom: 20px;
+height:0px;
+border-top: 1px solid #eee;
+border-right:0px;
+border-bottom:0px;
+border-left:0px;">
+ <h4 style="font-weight: 500;
+ line-height: 1;font-size:16px;">
+
+ <p>For any question, please contact support team <a
href="mailto:dl-ebay-kylin-c...@ebay.com "
+ style="color:
#337ab7;text-decoration: none;"><b>DL-eBay-Kylin-DISupport</b></a>
+ or Kylin <a href="https://ebay-eng.slack.com/messages/C1ZG2TN7J";
+ style="color: #337ab7;text-decoration:
none;"><b>Slack</b></a> channel.</p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 20481e6..c73bf28 100644
--- a/pom.xml
+++ b/pom.xml
@@ -363,6 +363,11 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-cube-migration</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-tool</artifactId>
<version>${project.version}</version>
</dependency>
@@ -1374,6 +1379,7 @@
<module>metrics-reporter-hive</module>
<module>metrics-reporter-kafka</module>
<module>cache</module>
+ <module>cube-migration</module>
<module>datasource-sdk</module>
<module>storage-stream</module>
<module>stream-receiver</module>
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelSchemaUpdateChecker.java
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelSchemaUpdateChecker.java
index e8c04fd..63d2e78 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelSchemaUpdateChecker.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelSchemaUpdateChecker.java
@@ -169,6 +169,10 @@ public class ModelSchemaUpdateChecker {
}
public CheckResult allowEdit(DataModelDesc modelDesc, String prj) {
+ return allowEdit(modelDesc, prj, !modelDesc.isDraft());
+ }
+
+ public CheckResult allowEdit(DataModelDesc modelDesc, String prj, boolean
needInit) {
final String modelName = modelDesc.getName();
// No model
@@ -176,7 +180,9 @@ public class ModelSchemaUpdateChecker {
if (existing == null) {
return CheckResult.validOnFirstCreate(modelName);
}
- modelDesc.init(metadataManager.getConfig(),
metadataManager.getAllTablesMap(prj));
+ if (needInit) {
+ modelDesc.init(metadataManager.getConfig(),
metadataManager.getAllTablesMap(prj));
+ }
// No cube
List<CubeInstance> cubes = findCubeByModel(modelName);
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index d1c583f..5a3a1ca 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -148,12 +148,12 @@ public class ModelService extends BasicService {
public DataModelDesc updateModelAndDesc(String project, DataModelDesc
desc) throws IOException {
aclEvaluate.checkProjectWritePermission(project);
validateModel(project, desc);
- checkModelCompatible(project, desc);
+ checkModelCompatibility(project, desc);
getDataModelManager().updateDataModelDesc(desc);
return desc;
}
- public void checkModelCompatible(String project, DataModelDesc
dataModalDesc) {
+ public void checkModelCompatibility(String project, DataModelDesc
dataModalDesc) {
ProjectInstance prjInstance = getProjectManager().getProject(project);
if (prjInstance == null) {
throw new BadRequestException("Project " + project + " does not
exist");
@@ -168,6 +168,23 @@ public class ModelService extends BasicService {
result.raiseExceptionWhenInvalid();
}
+ public void checkModelCompatibility(String project, DataModelDesc
dataModalDesc, List<TableDesc> tableDescList) {
+ ProjectInstance prjInstance = getProjectManager().getProject(project);
+ if (prjInstance == null) {
+ throw new BadRequestException("Project " + project + " does not
exist");
+ }
+ ModelSchemaUpdateChecker checker = new
ModelSchemaUpdateChecker(getTableManager(), getCubeManager(),
+ getDataModelManager());
+
+ Map<String, TableDesc> tableDescMap =
Maps.newHashMapWithExpectedSize(tableDescList.size());
+ for (TableDesc tableDesc : tableDescList) {
+ tableDescMap.put(tableDesc.getIdentity(), tableDesc);
+ }
+ dataModalDesc.init(getConfig(), tableDescMap);
+ ModelSchemaUpdateChecker.CheckResult result =
checker.allowEdit(dataModalDesc, project, false);
+ result.raiseExceptionWhenInvalid();
+ }
+
public void validateModel(String project, DataModelDesc desc) throws
IllegalArgumentException {
String factTableName = desc.getRootFactTableName();
TableDesc tableDesc = getTableManager().getTableDesc(factTableName,
project);
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
index 1b205be..f03acb8 100644
---
a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
+++
b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
@@ -50,7 +50,7 @@ public class TableSchemaUpdateChecker {
private final CubeManager cubeManager;
private final DataModelManager dataModelManager;
- static class CheckResult {
+ public static class CheckResult {
private final boolean valid;
private final String reason;
@@ -59,7 +59,7 @@ public class TableSchemaUpdateChecker {
this.reason = reason;
}
- void raiseExceptionWhenInvalid() {
+ public void raiseExceptionWhenInvalid() {
if (!valid) {
throw new RuntimeException(reason);
}
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index cd18d2b..764a32a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -117,6 +117,15 @@ public class TableService extends BasicService {
@Autowired
private AclEvaluate aclEvaluate;
+ public TableSchemaUpdateChecker getSchemaUpdateChecker() {
+ return new TableSchemaUpdateChecker(getTableManager(),
getCubeManager(), getDataModelManager());
+ }
+
+ public void checkTableCompatibility(String prj, TableDesc tableDesc) {
+ TableSchemaUpdateChecker.CheckResult result =
getSchemaUpdateChecker().allowReload(tableDesc, prj);
+ result.raiseExceptionWhenInvalid();
+ }
+
public List<TableDesc> getTableDescByProject(String project, boolean
withExt) throws IOException {
aclEvaluate.checkProjectReadPermission(project);
List<TableDesc> tables =
getProjectManager().listDefinedTables(project);
diff --git a/server/pom.xml b/server/pom.xml
index e0ec203..517c1cd 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -54,6 +54,20 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-cube-migration</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-shaded-guava</artifactId>
<scope>compile</scope>
</dependency>
diff --git a/server/src/main/resources/kylinSecurity.xml
b/server/src/main/resources/kylinSecurity.xml
index 0003a5f..baf7172 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -236,6 +236,7 @@
<scr:http-basic entry-point-ref="unauthorisedEntryPoint"/>
<scr:intercept-url pattern="/api/user/authentication*/**"
access="permitAll"/>
+ <scr:intercept-url pattern="/api/cubes/checkCompatibility"
access="permitAll"/>
<scr:intercept-url pattern="/api/query/runningQueries"
access="hasRole('ROLE_ADMIN')"/>
<scr:intercept-url pattern="/api/query/*/stop"
access="hasRole('ROLE_ADMIN')"/>
<scr:intercept-url pattern="/api/query*/**"
access="isAuthenticated()"/>
@@ -251,6 +252,7 @@
<scr:intercept-url pattern="/api/streaming*/**"
access="isAuthenticated()"/>
<scr:intercept-url pattern="/api/job*/**"
access="isAuthenticated()"/>
<scr:intercept-url pattern="/api/admin/public_config"
access="permitAll"/>
+ <scr:intercept-url pattern="/api/admin/config" access="permitAll"/>
<scr:intercept-url pattern="/api/admin/version"
access="permitAll"/>
<scr:intercept-url pattern="/api/projects" access="permitAll"/>
<scr:intercept-url pattern="/api/admin*/**"
access="hasRole('ROLE_ADMIN')"/>
@@ -287,6 +289,7 @@
<scr:http-basic entry-point-ref="unauthorisedEntryPoint"/>
<scr:intercept-url pattern="/api/user/authentication*/**"
access="permitAll"/>
+ <scr:intercept-url pattern="/api/cubes/checkCompatibility"
access="permitAll"/>
<scr:intercept-url pattern="/api/query/runningQueries"
access="hasRole('ROLE_ADMIN')"/>
<scr:intercept-url pattern="/api/query/*/stop"
access="hasRole('ROLE_ADMIN')"/>
<scr:intercept-url pattern="/api/query*/**"
access="isAuthenticated()"/>
diff --git
a/tool/src/main/java/org/apache/kylin/tool/migration/CompatibilityCheckRequest.java
b/tool/src/main/java/org/apache/kylin/tool/migration/CompatibilityCheckRequest.java
new file mode 100644
index 0000000..7144f7d
--- /dev/null
+++
b/tool/src/main/java/org/apache/kylin/tool/migration/CompatibilityCheckRequest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool.migration;
+
+import java.util.List;
+
+public class CompatibilityCheckRequest {
+ private List<String> tableDescDataList;
+ private String modelDescData;
+ private String projectName;
+
+ public List<String> getTableDescDataList() {
+ return tableDescDataList;
+ }
+
+ public void setTableDescDataList(List<String> tableDescDataList) {
+ this.tableDescDataList = tableDescDataList;
+ }
+
+ public String getModelDescData() {
+ return modelDescData;
+ }
+
+ public void setModelDescData(String modelDescData) {
+ this.modelDescData = modelDescData;
+ }
+
+ public String getProjectName() {
+ return projectName;
+ }
+
+ public void setProjectName(String projectName) {
+ this.projectName = projectName;
+ }
+}
diff --git
a/tool/src/main/java/org/apache/kylin/tool/query/ProbabilityGenerator.java
b/tool/src/main/java/org/apache/kylin/tool/query/ProbabilityGenerator.java
new file mode 100644
index 0000000..9f32de1
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/query/ProbabilityGenerator.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool.query;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProbabilityGenerator {
+
+ private static final Logger logger =
LoggerFactory.getLogger(ProbabilityGenerator.class);
+
+ public static double[] generate(int size) {
+ double[] probArray = generateProbabilityList(size);
+ return generateProbabilityCumulative(probArray);
+ }
+
+ public static int searchIndex(double p, double[] pCumArray) {
+ return binarySearchIndex(p, pCumArray, 0, pCumArray.length - 1);
+ }
+
+ private static int binarySearchIndex(double key, double[] array, int from,
int to) {
+ if (from < 0 || to < 0) {
+ throw new IllegalArgumentException("params from & length must
larger than 0 .");
+ }
+ if (key < array[from]) {
+ return from - 1;
+ } else if (key >= array[to]) {
+ return to;
+ }
+
+ int middle = (from >>> 1) + (to >>> 1);
+ double temp = array[middle];
+ if (temp > key) {
+ to = middle - 1;
+ } else if (temp < key) {
+ from = middle + 1;
+ } else {
+ return middle;
+ }
+ return binarySearchIndex(key, array, from, to);
+ }
+
+ public static double[] generateProbabilityCumulative(double[] pQueryArray)
{
+ double[] pCumArray = new double[pQueryArray.length];
+ pCumArray[0] = 0;
+ for (int i = 0; i < pQueryArray.length - 1; i++) {
+ pCumArray[i + 1] = pCumArray[i] + pQueryArray[i];
+ }
+ return pCumArray;
+ }
+
+ public static double[] generateProbabilityList(int nOfEle) {
+ Integer[] nHitArray = new Integer[nOfEle];
+ double[] pQueryArray = new double[nOfEle];
+
+ int sumHit = generateHitNumberList(nHitArray);
+ for (int i = 0; i < nOfEle; i++) {
+ pQueryArray[i] = nHitArray[i] * 1.0 / sumHit;
+ }
+ return pQueryArray;
+ }
+
+ public static int generateHitNumberList(Integer[] nHitArray) {
+ int sumHit = 0;
+ for (int i = 0; i < nHitArray.length; i++) {
+ int randomNum = 1 + (int) (Math.random() * nHitArray.length);
+ nHitArray[i] = randomNum * randomNum;
+ sumHit += nHitArray[i];
+ }
+ return sumHit;
+ }
+}
diff --git
a/tool/src/main/java/org/apache/kylin/tool/query/ProbabilityGeneratorCLI.java
b/tool/src/main/java/org/apache/kylin/tool/query/ProbabilityGeneratorCLI.java
new file mode 100644
index 0000000..c248ad9
--- /dev/null
+++
b/tool/src/main/java/org/apache/kylin/tool/query/ProbabilityGeneratorCLI.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool.query;
+
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProbabilityGeneratorCLI extends AbstractApplication {
+
+ private static final Logger logger =
LoggerFactory.getLogger(ProbabilityGeneratorCLI.class);
+
+ private static final Option OPTION_SIZE =
OptionBuilder.withArgName("size").hasArg().isRequired(true)
+ .withDescription("Specify the size of query set to be
generated").create("size");
+ private static final Option OPTION_OUTPUT =
OptionBuilder.withArgName("output").hasArg().isRequired(false)
+ .withDescription("Specify the output path for generated
probability set").create("output");
+
+ protected final Options options;
+ private int size;
+ private String outputPath;
+
+ public ProbabilityGeneratorCLI() {
+ options = new Options();
+ options.addOption(OPTION_SIZE);
+ options.addOption(OPTION_OUTPUT);
+ }
+
+ protected Options getOptions() {
+ return options;
+ }
+
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ this.size =
Integer.parseInt(optionsHelper.getOptionValue(OPTION_SIZE));
+ this.outputPath = optionsHelper.getOptionValue(OPTION_OUTPUT);
+
+ run();
+ }
+
+ public double[] execute(int sizeOfQueryList, String outputPath) throws
Exception {
+ this.size = sizeOfQueryList;
+ this.outputPath = outputPath;
+
+ return run();
+ }
+
+ private double[] run() throws Exception {
+ double[] probArray =
ProbabilityGenerator.generateProbabilityList(this.size);
+ storeProbability(probArray, outputPath);
+ return ProbabilityGenerator.generateProbabilityCumulative(probArray);
+ }
+
+ public double[] execute(int size) throws Exception {
+ this.size = size;
+ double[] pQueryArray =
ProbabilityGenerator.generateProbabilityList(this.size);
+ return ProbabilityGenerator.generateProbabilityCumulative(pQueryArray);
+ }
+
+ public static void storeProbability(double[] probArray, String outputPath)
throws IOException {
+ try (BufferedWriter bufferedWriter = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(outputPath +
".prob"), StandardCharsets.UTF_8))) {
+ for (double elem : probArray) {
+ bufferedWriter.append(String.valueOf(elem));
+ bufferedWriter.append("\n");
+ logger.info(String.valueOf(elem));
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ ProbabilityGeneratorCLI probabilityGeneratorCLI = new
ProbabilityGeneratorCLI();
+ probabilityGeneratorCLI.execute(args);
+ }
+}
diff --git a/tool/src/main/java/org/apache/kylin/tool/query/QueryGenerator.java
b/tool/src/main/java/org/apache/kylin/tool/query/QueryGenerator.java
new file mode 100644
index 0000000..83fca53
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/query/QueryGenerator.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool.query;
+
+import java.math.BigInteger;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
+import org.apache.kylin.job.JoinedFlatTable;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public class QueryGenerator {
+
+ private static final Logger logger =
LoggerFactory.getLogger(QueryGenerator.class);
+
+ public static List<String> generateQueryList(CubeDesc cubeDesc, int
nOfQuery, int maxNumOfDimension) {
+ int nDimension = cubeDesc.getDimensions().size();
+ if (maxNumOfDimension > nDimension) {
+ maxNumOfDimension = nDimension;
+ } else if (maxNumOfDimension < 1) {
+ maxNumOfDimension = 1;
+ }
+
+ int queryMaxSize = getQueryMaxSize(maxNumOfDimension, nDimension);
+ queryMaxSize = (int) Math.ceil(queryMaxSize * 0.5);
+ if (nOfQuery > queryMaxSize) {
+ nOfQuery = queryMaxSize;
+ }
+
+ logger.info("Will generate {} queries", nOfQuery);
+
+ List<String> sqlList = Lists.newArrayListWithExpectedSize(nOfQuery);
+ Set<BitSet> selected = Sets.newHashSetWithExpectedSize(nOfQuery);
+ while (sqlList.size() < nOfQuery) {
+ sqlList.add(generateQuery(cubeDesc, selected, maxNumOfDimension));
+ }
+
+ return sqlList;
+ }
+
+ public static int getQueryMaxSize(int m, int nDimension) {
+ int a = nDimension - m >= m ? nDimension - m : m;
+ int b = nDimension - a;
+
+ BigInteger result = new BigInteger(String.valueOf(1));
+ for (int i = a + 1; i <= nDimension; i++) {
+ result = result.multiply(new BigInteger(String.valueOf(i)));
+ }
+ for (int i = 2; i <= b; i++) {
+ result = result.divide(new BigInteger(String.valueOf(i)));
+ }
+ return result.intValue();
+ }
+
+ public static String generateQuery(CubeDesc cubeDesc, Set<BitSet>
selected, int maxNumOfDimension) {
+ IJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(cubeDesc);
+
+ String dimensionStatement =
createDimensionStatement(cubeDesc.getDimensions(), selected, maxNumOfDimension);
+ String measureStatement =
createMeasureStatement(cubeDesc.getMeasures());
+
+ StringBuilder sql = new StringBuilder();
+ sql.append("SELECT" + "\n");
+ sql.append(dimensionStatement);
+ sql.append(measureStatement);
+
+ StringBuilder joinPart = new StringBuilder();
+ JoinedFlatTable.appendJoinStatement(flatDesc, joinPart, false, null);
+ sql.append(joinPart.toString().replaceAll("DEFAULT\\.", ""));
+
+ sql.append("GROUP BY" + "\n");
+ sql.append(dimensionStatement);
+ String ret = sql.toString();
+ ret = ret.replaceAll("`", "\"");
+ return ret;
+ }
+
+ public static String createMeasureStatement(List<MeasureDesc> measureList)
{
+ StringBuilder sql = new StringBuilder();
+
+ for (MeasureDesc measureDesc : measureList) {
+ FunctionDesc functionDesc = measureDesc.getFunction();
+ if (functionDesc.isSum() || functionDesc.isMax() ||
functionDesc.isMin()) {
+ sql.append("," + functionDesc.getExpression() + "(" +
functionDesc.getParameter().getValue() + ")\n");
+ break;
+ } else if (functionDesc.isCountDistinct()) {
+ sql.append(",COUNT" + "(DISTINCT " +
functionDesc.getParameter().getValue() + ")\n");
+ break;
+ }
+ }
+
+ return sql.toString();
+ }
+
+ public static String createDimensionStatement(List<DimensionDesc>
dimensionList, Set<BitSet> selected,
+ final int maxNumOfDimension) {
+ StringBuilder sql = new StringBuilder();
+
+ BitSet bitSet;
+
+ do {
+ bitSet = generateIfSelectList(dimensionList.size(),
+ Math.ceil(maxNumOfDimension * Math.random()) /
dimensionList.size());
+ } while (bitSet.cardinality() > maxNumOfDimension ||
bitSet.cardinality() <= 0 || selected.contains(bitSet));
+ selected.add(bitSet);
+
+ List<String> selectedCols = Lists.newArrayList();
+ int j = 0;
+ for (int i = 0; i < dimensionList.size(); i++) {
+ if (bitSet.get(i)) {
+ DimensionDesc dimensionDesc = dimensionList.get(i);
+ String tableName = getTableName(dimensionDesc.getTable());
+ String columnName = dimensionDesc.getColumn();
+ if (Strings.isNullOrEmpty(columnName) ||
columnName.equals("{FK}")) {
+ String[] derivedCols = dimensionDesc.getDerived();
+ BitSet subBitSet;
+ do {
+ subBitSet = generateIfSelectList(derivedCols.length,
0.5);
+ } while (subBitSet.cardinality() <= 0);
+
+ for (int k = 0; k < derivedCols.length; k++) {
+ if (subBitSet.get(k)) {
+ if (j > 0) {
+ sql.append(",");
+ }
+ sql.append(tableName + ".\"" + derivedCols[k] +
"\"\n");
+ selectedCols.add(derivedCols[k]);
+ j++;
+ }
+ }
+ } else {
+ if (j > 0) {
+ sql.append(",");
+ }
+ sql.append(tableName + ".\"" + columnName + "\"\n");
+ selectedCols.add(columnName);
+ j++;
+ }
+ }
+ }
+
+ return sql.toString();
+ }
+
+ public static BitSet generateIfSelectList(int n, double threshold) {
+ BitSet bitSet = new BitSet(n);
+ for (int i = 0; i < n; i++) {
+ if (Math.random() < threshold) {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+
+ public static String getTableName(String name) {
+ int lastIndexOfDot = name.lastIndexOf(".");
+ if (lastIndexOfDot >= 0) {
+ name = name.substring(lastIndexOfDot + 1);
+ }
+ return name;
+ }
+}
diff --git
a/tool/src/main/java/org/apache/kylin/tool/query/QueryGeneratorCLI.java
b/tool/src/main/java/org/apache/kylin/tool/query/QueryGeneratorCLI.java
new file mode 100644
index 0000000..d1b7f98
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/query/QueryGeneratorCLI.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool.query;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Strings;
+
+public class QueryGeneratorCLI extends AbstractApplication {
+
+ private static final Logger logger =
LoggerFactory.getLogger(QueryGeneratorCLI.class);
+
+ private static final Option OPTION_MAX_DIM_NUM =
OptionBuilder.withArgName("maxDimNum").hasArg().isRequired(false)
+ .withDescription("Specify the maximum number of dimensions for
generating a query").create("maxDimNum");
+ private static final Option OPTION_CUBE =
OptionBuilder.withArgName("cube").hasArg().isRequired(true)
+ .withDescription("Specify for which cube to generate
query").create("cube");
+ private static final Option OPTION_SIZE =
OptionBuilder.withArgName("size").hasArg().isRequired(true)
+ .withDescription("Specify the size of query set to be
generated").create("size");
+ private static final Option OPTION_OUTPUT =
OptionBuilder.withArgName("output").hasArg().isRequired(true)
+ .withDescription("Specify the output path for generated query
set").create("output");
+
+ public static final String SQL_SEPARATOR = "#############";
+
+ protected final Options options;
+
+ private int sizeOfQueryList;
+ private String outputPath;
+ private int maxNumOfDim = 3;
+
+ public QueryGeneratorCLI() {
+ options = new Options();
+ options.addOption(OPTION_MAX_DIM_NUM);
+ options.addOption(OPTION_CUBE);
+ options.addOption(OPTION_SIZE);
+ options.addOption(OPTION_OUTPUT);
+ }
+
+ protected Options getOptions() {
+ return options;
+ }
+
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ String temp = optionsHelper.getOptionValue(OPTION_MAX_DIM_NUM);
+ if (!Strings.isNullOrEmpty(temp)) {
+ this.maxNumOfDim = Integer.parseInt(temp);
+ }
+
+ this.outputPath = optionsHelper.getOptionValue(OPTION_OUTPUT);
+ this.sizeOfQueryList =
Integer.parseInt(optionsHelper.getOptionValue(OPTION_SIZE));
+
+ String cubeName = optionsHelper.getOptionValue(OPTION_CUBE);
+ run(cubeName, true);
+ }
+
+ public Pair<List<String>, double[]> execute(String cubeName, int
sizeOfQueryList, String outputPath)
+ throws Exception {
+ this.outputPath = outputPath;
+ this.sizeOfQueryList = sizeOfQueryList;
+
+ return run(cubeName, true);
+ }
+
+ public Pair<List<String>, double[]> execute(String cubeName, int
sizeOfQueryList) throws Exception {
+ this.sizeOfQueryList = sizeOfQueryList;
+ return run(cubeName, false);
+ }
+
+ private Pair<List<String>, double[]> run(String cubeName, boolean
needToStore) throws Exception {
+ CubeDesc cubeDesc =
CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()).getCubeDesc(cubeName);
+
+ //Generate query list
+ List<String> queryList = QueryGenerator.generateQueryList(cubeDesc,
sizeOfQueryList, maxNumOfDim);
+ ProbabilityGeneratorCLI probabilityGeneratorCLI = new
ProbabilityGeneratorCLI();
+ double[] pCumArray;
+ if (needToStore) {
+ storeQuery(queryList, outputPath + "/" + cubeName);
+ pCumArray = probabilityGeneratorCLI.execute(queryList.size(),
outputPath + "/" + cubeName);
+ } else {
+ pCumArray = probabilityGeneratorCLI.execute(queryList.size());
+ }
+ return new Pair<>(queryList, pCumArray);
+ }
+
+ public static void storeQuery(List<String> querySet, String outputPath)
throws IOException {
+ String fileName = outputPath + ".sql";
+ File parentFile = new File(fileName).getParentFile();
+ if (!parentFile.exists()) {
+ parentFile.mkdirs();
+ }
+ try (BufferedWriter bufferedWriter = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(fileName),
StandardCharsets.UTF_8))) {
+ for (String query : querySet) {
+ bufferedWriter.append(query);
+ bufferedWriter.append(SQL_SEPARATOR + "\n");
+ logger.info(query);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ QueryGeneratorCLI queryGeneratorCLI = new QueryGeneratorCLI();
+ queryGeneratorCLI.execute(args);
+ }
+}
diff --git
a/tool/src/test/java/org/apache/kylin/tool/query/ProbabilityGeneratorCLITest.java
b/tool/src/test/java/org/apache/kylin/tool/query/ProbabilityGeneratorCLITest.java
new file mode 100755
index 0000000..a9a79af
--- /dev/null
+++
b/tool/src/test/java/org/apache/kylin/tool/query/ProbabilityGeneratorCLITest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool.query;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ProbabilityGeneratorCLITest extends LocalFileMetadataTestCase {
+
+ public final String cubeName = "test_kylin_cube_with_slr_desc";
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testExecute() throws Exception {
+ ProbabilityGeneratorCLI probabilityGeneratorCLI = new
ProbabilityGeneratorCLI();
+ double[] pCumArray = probabilityGeneratorCLI.execute(100, cubeName);
+ for (double pCum : pCumArray) {
+ System.out.print(pCum + " ");
+ }
+ int pIdx = ProbabilityGenerator.searchIndex(0.5, pCumArray);
+ System.out.print("\n" + pIdx);
+ }
+}
diff --git
a/tool/src/test/java/org/apache/kylin/tool/query/QueryGeneratorCLITest.java
b/tool/src/test/java/org/apache/kylin/tool/query/QueryGeneratorCLITest.java
new file mode 100755
index 0000000..ff71ac0
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/query/QueryGeneratorCLITest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool.query;
+
+import java.util.List;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class QueryGeneratorCLITest extends LocalFileMetadataTestCase {
+
+ public final String cubeName = "test_kylin_cube_with_slr_desc";
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testExecute() throws Exception {
+ QueryGeneratorCLI queryGeneratorCLI = new QueryGeneratorCLI();
+ Pair<List<String>, double[]> result =
queryGeneratorCLI.execute(cubeName, 10);
+ List<String> sqls = result.getFirst();
+ double[] probs = result.getSecond();
+ for (int i = 0; i < sqls.size(); i++) {
+ System.out.println("Accumulate Probability: " + probs[i]);
+ System.out.println("SQL: " + sqls.get(i));
+ }
+ }
+}
