This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f9255a515d38edafd099c81712c0b6ff8e7038d7
Author: Pengfei Zhan <dethr...@gmail.com>
AuthorDate: Mon Jun 19 11:41:51 2023 +0800
KYLIN-5829 Add a tool for generating sql from dumped metadata
---
build/bin/create-table.sh | 1 -
build/conf/kylin-tools-diag-log4j.xml | 6 ++
.../kylin/tool}/util/CreateTableFromJson.java | 31 +++++-----
.../kylin/tool/util/CreateTableFromJsonTest.java | 68 ++++++++++++++++++++++
4 files changed, 89 insertions(+), 17 deletions(-)
diff --git a/build/bin/create-table.sh b/build/bin/create-table.sh
index 5fbc49541d..a21e39036e 100755
--- a/build/bin/create-table.sh
+++ b/build/bin/create-table.sh
@@ -16,7 +16,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
if [ -z $KYLIN_HOME ];then
export KYLIN_HOME=$(cd -P -- "$(dirname -- "$0")"/../ && pwd -P)
fi
diff --git a/build/conf/kylin-tools-diag-log4j.xml
b/build/conf/kylin-tools-diag-log4j.xml
index 728e50d2ca..227e71cdd1 100644
--- a/build/conf/kylin-tools-diag-log4j.xml
+++ b/build/conf/kylin-tools-diag-log4j.xml
@@ -28,6 +28,9 @@
<Console name="stdout" target="SYSTEM_OUT">
<PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
+ <File name="createTableSql"
fileName="${env:KYLIN_HOME}/logs/create_table.sql">
+ <PatternLayout pattern="%msg%n"/>
+ </File>
</Appenders>
<Loggers>
<Root level="DEBUG">
@@ -38,5 +41,8 @@
<Logger name="org.springframework.security" level="INFO"/>
<Logger name="io.kyligence" level="DEBUG"/>
<Logger name="org.apache.kylin" level="DEBUG"/>
+ <Logger name="org.apache.kylin.tool.util.CreateTableFromJson"
level="INFO">
+ <AppenderRef ref="createTableSql"/>
+ </Logger>
</Loggers>
</Configuration>
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
b/src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
similarity index 81%
rename from
src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
rename to
src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
index 2ee7efef22..67148ad608 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/rest/util/CreateTableFromJson.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/CreateTableFromJson.java
@@ -16,10 +16,13 @@
* limitations under the License.
*/
-package org.apache.kylin.rest.util;
+package org.apache.kylin.tool.util;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -29,20 +32,16 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* A tool to generate database and tables from metadata backups of customer
*/
+@Slf4j
public class CreateTableFromJson {
private static final String QUOTE = "`";
- private static final Map<String, String> TYPE_MAP = Maps.newHashMap();
-
- private static final Logger logger =
LoggerFactory.getLogger(CreateTableFromJson.class);
+ private static final Map<String, String> TYPE_MAP = new HashMap<>();
static {
TYPE_MAP.put("integer", "int");
@@ -54,23 +53,23 @@ public class CreateTableFromJson {
String pathDir = args[0];
Map<String, List<String>> map = createDbAndTables(pathDir);
- map.forEach((k, v) -> {
- logger.info(k);
- v.forEach(logger::info);
+ map.forEach((db, tables) -> {
+ for (String s : db.split("\n")) {
+ log.info(s);
+ }
+ tables.forEach(log::info);
});
-
- logger.info("\n\n\n\n\n");
}
// the path is /{metadata_backup_path}/{project_name}/table/
private static Map<String, List<String>> createDbAndTables(String pathDir)
throws IOException {
- Map<String, List<String>> map = Maps.newHashMap();
+ Map<String, List<String>> map = new LinkedHashMap<>();
File file = new File(pathDir).getAbsoluteFile();
File[] files = file.listFiles();
for (File f : Objects.requireNonNull(files)) {
final TableDesc tableDesc = JsonUtil.readValue(f, TableDesc.class);
- List<String> columnNameTypeList = Lists.newArrayList();
+ List<String> columnNameTypeList = new ArrayList<>();
for (ColumnDesc column : tableDesc.getColumns()) {
String name = column.getName();
String type = convert(column.getDatatype());
@@ -79,7 +78,7 @@ public class CreateTableFromJson {
String databaseSql = String.format(Locale.ROOT, "create database
%s;%nuse %s;",
quote(tableDesc.getDatabase()),
quote(tableDesc.getDatabase()));
- map.putIfAbsent(databaseSql, Lists.newArrayList());
+ map.putIfAbsent(databaseSql, new ArrayList<>());
String tableSql = createTableSql(tableDesc.getName(),
columnNameTypeList);
map.get(databaseSql).add(tableSql);
}
diff --git
a/src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java
b/src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java
new file mode 100644
index 0000000000..7d729aee9f
--- /dev/null
+++
b/src/tool/src/test/java/org/apache/kylin/tool/util/CreateTableFromJsonTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class CreateTableFromJsonTest {
+
+ String path =
"src/test/resources/table_cc_cleanup/metadata/AL_4144/table/";
+ private final ByteArrayOutputStream outContent = new
ByteArrayOutputStream();
+ private PrintStream originPrintStream;
+ private static final String REGEX = "util.CreateTableFromJson :";
+
+ @BeforeEach
+ void setUpStreams() {
+ originPrintStream = System.out;
+ System.setOut(new PrintStream(outContent));
+ }
+
+ @AfterEach
+ void cleanUpStreams() {
+ System.setOut(originPrintStream);
+ }
+
+ @Test
+ void test() throws IOException {
+ CreateTableFromJson.main(new String[] { path });
+ String result = outContent.toString();
+ if (StringUtils.isBlank(result)) {
+ Assertions.assertTrue(true);
+ } else if ("true".equals(System.getProperty("junit.run.local"))) {
+ // with -Djunit.run.local=true, this branch is more convenient for
dev
+ String[] splits = result.split("\n");
+ for (String str : splits) {
+ String s = str.split(REGEX)[1];
+ originPrintStream.println(s);
+ }
+ } else {
+ String[] splits = result.split("\n");
+ Assertions.assertTrue(splits[0].contains("create database
`CAP`;"));
+ Assertions.assertTrue(splits[1].contains("use `CAP`;"));
+ }
+ }
+}
