This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to tag ide-run-2023
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9034caaaff30c8657f5a99f58af1b49068975885
Author: XiaoxiangYu <x...@apache.org>
AuthorDate: Thu Sep 21 17:33:03 2023 +0800
Verify run Kylin in IDEA guide
- dev-support/contributor/README.md is fully rewrote and tested on my
laptop based this commit
---
build/bin/guardian.sh | 4 +-
build/sbin/bootstrap.sh | 12 +-
dev-support/contributor/README.md | 428 +++++++++++++++++++++
dev-support/{ => contributor}/local/README.md | 12 +-
.../{ => contributor}/local/docker-compose.yml | 0
.../local/images/docker_container.png | Bin
.../local/images/docker_resources.png | Bin
.../local/images/executing_result.png | Bin
.../local/images/idea_configuration.png | Bin
.../{ => contributor}/local/images/kylin_ui.png | Bin
.../local/images/run_configuration_replace.png | Bin
.../{ => contributor}/local/images/sparder.png | Bin
dev-support/{ => contributor}/local/local.sh | 0
.../runConfigurations/BootstrapServer_local.xml | 0
.../BootstrapServer_local_community.xml | 0
dev-support/{ => contributor}/sandbox/.gitignore | 0
.../sandbox/conf/capacity-scheduler.xml | 0
.../{ => contributor}/sandbox/conf/core-site.xml | 0
.../sandbox/conf/fairscheduler.xml | 0
.../sandbox/conf/hadoop/core-site.xml | 0
.../{ => contributor}/sandbox/conf/hdfs-site.xml | 0
.../{ => contributor}/sandbox/conf/hive-site.xml | 0
.../sandbox/conf/kylin.properties | 0
.../{ => contributor}/sandbox/conf/yarn-site.xml | 0
.../{ => contributor}/sandbox/docker-compose.yml | 0
dev-support/{ => contributor}/sandbox/hadoop.env | 0
dev-support/{ => contributor}/sandbox/hive.env | 0
.../BootstrapServer_docker_sandbox_.xml | 4 +-
.../BootstrapServer_docker_sandbox_ultimate.xml | 0
dev-support/{ => contributor}/sandbox/sandbox.sh | 2 +-
dev-support/release-manager/README.md | 1 +
.../release-machine/Dockerfile_1 | 0
.../release-machine/Dockerfile_2 | 0
.../release-machine/README.md | 0
.../release-machine/conf/settings.xml | 0
.../release-machine/create-release-machine.sh | 0
.../release-machine/entrypoint.sh | 0
.../release-machine/release-publish.sh | 0
.../release-manager/release-process-definition.md | 13 +
.../release-manager/standalone-docker}/.gitignore | 0
.../all_in_one/Dockerfile_hadoop | 0
.../standalone-docker}/all_in_one/Dockerfile_kylin | 0
.../standalone-docker}/all_in_one/build_image.sh | 0
.../all_in_one/conf/hadoop/capacity-scheduler.xml | 0
.../all_in_one/conf/hadoop/core-site.xml | 0
.../all_in_one/conf/hadoop/hdfs-site.xml | 0
.../all_in_one/conf/hadoop/mapred-site.xml | 0
.../all_in_one/conf/hadoop/yarn-site.xml | 0
.../all_in_one/conf/hive/hive-site.xml | 0
.../conf/kylin/kylin.properties.override | 0
.../standalone-docker}/all_in_one/conf/zk/zoo.cfg | 0
.../standalone-docker}/all_in_one/entrypoint.sh | 0
.../all_in_one/scripts/install_mysql.sh | 0
dev-support/sandbox/README.md | 114 ------
dev-support/sandbox/images/6nP8shbaNX.png | Bin 49891 -> 0 bytes
.../sandbox/images/image-20211102105411171.png | Bin 279505 -> 0 bytes
.../sandbox/images/image-20211102105446286.png | Bin 101816 -> 0 bytes
.../sandbox/images/image-20211102110553460.png | Bin 296430 -> 0 bytes
.../sandbox/images/image-20211102111037483.png | Bin 364056 -> 0 bytes
.../sandbox/images/image-20211102111253713.png | Bin 138867 -> 0 bytes
.../sandbox/images/image-20211102111745414.png | Bin 223073 -> 0 bytes
.../sandbox/images/image-20211102111806627.png | Bin 225910 -> 0 bytes
.../sandbox/images/image-20211102112221619.png | Bin 298560 -> 0 bytes
.../sandbox/images/image-20211102112351003.png | Bin 337499 -> 0 bytes
.../sandbox/images/image-20211102113005133.png | Bin 304205 -> 0 bytes
.../sandbox/images/image-20211102113055804.png | Bin 308948 -> 0 bytes
.../sandbox/images/image-20211102113605007.png | Bin 291798 -> 0 bytes
dev-support/sandbox/images/img.png | Bin 140288 -> 0 bytes
dev-support/sandbox/images/img_1.png | Bin 57617 -> 0 bytes
.../img_8835f30f-7805-4951-808d-45469370fa7g.png | Bin 210172 -> 0 bytes
.../scalastyle-config.xml | 0
pom.xml | 18 +-
.../apache/kylin/rest/KylinPrepareEnvListener.java | 2 +-
src/spark-project/engine-spark/pom.xml | 4 +
74 files changed, 480 insertions(+), 134 deletions(-)
diff --git a/build/bin/guardian.sh b/build/bin/guardian.sh
index 435fea7726..09e6cde1e9 100755
--- a/build/bin/guardian.sh
+++ b/build/bin/guardian.sh
@@ -45,8 +45,8 @@ function startKG() {
fi
fi
- KE_PID_FILE=${KYLIN_HOME}/pid
- if [[ ! -f ${KE_PID_FILE} ]]; then
+ KYLIN_PID_FILE=${KYLIN_HOME}/pid
+ if [[ ! -f ${KYLIN_PID_FILE} ]]; then
quit "Kylin is not running, will not start guardian process"
fi
diff --git a/build/sbin/bootstrap.sh b/build/sbin/bootstrap.sh
index 0d18cf4bff..ab53616e3b 100755
--- a/build/sbin/bootstrap.sh
+++ b/build/sbin/bootstrap.sh
@@ -272,7 +272,7 @@ function setLogRotate() {
fi
}
-function startKE(){
+function startKylin(){
setLogRotate
clearRedundantProcess
@@ -346,7 +346,7 @@ function startKE(){
recordKylinStartOrStop "start success" "${START_TIME}"
}
-function stopKE(){
+function stopKylin(){
sh ${KYLIN_HOME}/bin/guardian.sh stop
clearCrontab
@@ -395,11 +395,11 @@ if [[ "$1" == org.apache.kylin.* ]]; then
# start command
elif [ "$1" == "start" ]; then
echo "Starting Kylin..."
- startKE
+ startKylin
# stop command
elif [ "$1" == "stop" ]; then
echo `date '+%Y-%m-%d %H:%M:%S '`"Stopping Kylin..."
- stopKE
+ stopKylin
if [[ $? == 0 ]]; then
exit 0
else
@@ -409,12 +409,12 @@ elif [ "$1" == "stop" ]; then
elif [ "$1" == "restart" ]; then
echo "Restarting Kylin..."
echo "--> Stopping Kylin first if it's running..."
- stopKE
+ stopKylin
if [[ $? != 0 ]]; then
echo " Kylin is not running, now start it"
fi
echo "--> Starting Kylin..."
- startKE
+ startKylin
else
quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop' or 'kylin.sh
[-v] restart'"
fi
diff --git a/dev-support/contributor/README.md
b/dev-support/contributor/README.md
new file mode 100644
index 0000000000..44640d5e1d
--- /dev/null
+++ b/dev-support/contributor/README.md
@@ -0,0 +1,428 @@
+# How to Run/Debug Kylin in IDE(with some backgrounds)
+> This guide is an enhanced version for [How to debug Kylin in IDEA with
Hadoop](https://kylin.apache.org/5.0/docs/development/how_to_debug_kylin_in_ide)
, <br>
+> it is not only step-by-step guide, but give more background knowledge and
explanation.
+
+## Part One - Before your read
+
+### STEP 1: Who need this guide
+
+If you are any developer who do following things, then this guide is for you.
+
+- want to **deep dive into technical detail** of Kylin 5 and learn how it works
+- want to fix bugs or develop new features of Kylin 5 on you own
+
+To be specific, this guide will help you to:
+
+- set up your development software/env of Kylin 5
+- understand the source code of Kylin 5
+- learn how to reproduce some bugs and try to find its root cause
+- learn how to verify if your patch can fix bugs before you submits your PR
+
+Read [background
part](https://kylin.apache.org/5.0/docs/development/how_to_debug_kylin_in_ide#background).
+
+### Step 2: Why you need Kylin 5
+
+to be updated
+
+### Step 3: Understand Project structure
+
+| Folder Name | Comment
|
+|:------------|------------------------------------------------------------------------------------------------------|
+| build | Scripts for building, packaging, running Kylin
|
+| dev-support | Scripts and guides for contributors to develop/debug/test, for
committers to release/publish website |
+| kystudio | Frontend source code, mainly using Vue.js
|
+| src | Backend source code, mainly using Java & Scala, managed by
Maven |
+| pom.xml | Project definition
|
+| README.md | General guide to Kylin 5 project
|
+| LICENSE | A must-to-have file by ASF
|
+| NOTICE | A must-to-have file by ASF
|
+
+### Step 4: Understand Maven modules
+
+
+| Module Name | Brief Description
| Tags |
+|:---------------------------|:---------------------------------------------------------------------------------------|:------------------------:|
+| Core Common | todo
| Core |
+| Core Metadata | todo
| Core |
+| Core Metrics | todo
| Core |
+| Core Job | todo
| Core |
+| Core Storage | todo
| Core |
+| Query Common | todo
| Core |
+| Local Data Cache | Improve query performance by caching parquet
files in spark executor's disk/memory | Add-on |
+| Spark Common | todo
|
+| Query Engine Spark | todo
|
+| Hive Source | todo
|
+| Build Engine | todo
| Core |
+| Distributed Lock Extension | todo
| Add-on |
+| Build Engine Spark | todo
|
+| Query | Transfer sql text to logical/physical plan and
optimize using Apache Calcite. | Core |
+| Streaming SDK | Not ready. Used to parse Kafka message in
custom way. | Add-on, Not-Ready-Module |
+| Streaming | Not ready. Make Apache Kafka as a data source
for Kylin 5. | Not-Ready-Module |
+| Tool | Different tools for metadata backup, Diagnose
etc. | Tool |
+| Common Service | todo
|
+| Datasource Service | todo
|
+| Modeling Service | todo
|
+| Data Loading Service | todo
|
+| Query Service | todo
|
+| Common Server | todo
|
+| Job Service | todo
|
+| Streaming Service | todo
| Not-Ready-Module |
+| Data Loading Server | todo
|
+| Query Server | todo
|
+| Metadata Server | todo
|
+| REST Server | Main entry of Kylin process, including Spring
config files. | Spring |
+| Datasource SDK | Not ready. Framework to add data source for
Kylin 5. | Add-on, Not-Ready-Module |
+| JDBC Source | Not ready. Make some RDBMS as a data source fro
Kylin 5. | Not-Ready-Module |
+| Integration Test | Major code for Integration Test
| Testing |
+| Integration Test Spark | Some code for Integration Test
| Testing |
+| Source Assembly | Used to create jars for build engine in
spark-submit cmd. | Build |
+| Integration Test Server | Some code for Integration Test
| Testing |
+| Data loading Booter | For micro-service deployment such as k8s.
Process build/refresh index/segment request. | Micro-service |
+| Query Booter | For micro-service deployment such as k8s.
Process query request. | Micro-service |
+| Common Booter | For micro-service deployment such as k8s.
Process crud of metadata request. | Micro-service |
+| JDBC Driver | Connect Kylin using JDBC, for SQL Client or BI
| Tool |
+
+### STEP 5: Install required software in laptop
+
+| Component | Version |
Comment/Link |
+|--------------------------|--------------------------------------------|---------------------------------|
+| JDK | JDK8 | n/a
|
+| Apache Maven | 3.5+ | n/a
|
+| IntelliJ IDEA | IntelliJ IDEA 2023.2.2 (Community Edition) | n/a
|
+| Docker Desktop (for Mac) | 4.22.1 (118664) | n/a
|
+| NodeJs | latest |
https://nodejs.org/en/download/ |
+| nvm | latest |
https://github.com/nvm-sh/nvm |
+
+After install nvm, please use `nvm install 12.14.0` to install correct version
of Node.js for `kystudio`.
+
+I checked software version in my laptop(macbook pro, 15-inch, 2018) using
following command. You can try it.
+
+```shell
+(base) xiaoxiang.yu@XXYU-MBP ~ % uname -a
+Darwin XXYU-MBP.local 22.6.0 Darwin Kernel Version 22.6.0: Wed Jul 5 22:21:56
PDT 2023; root:xnu-8796.141.3~6/RELEASE_X86_64 x86_64
+(base) xiaoxiang.yu@XXYU-MBP ~ % java -version
+java version "1.8.0_301"
+Java(TM) SE Runtime Environment (build 1.8.0_301-b09)
+Java HotSpot(TM) 64-Bit Server VM (build 25.301-b09, mixed mode)
+(base) xiaoxiang.yu@XXYU-MBP ~ % mvn -version
+Apache Maven 3.8.2 (ea98e05a04480131370aa0c110b8c54cf726c06f)
+Maven home: /Users/xiaoxiang.yu/LacusDir/lib/apache-maven-3.8.2
+Java version: 1.8.0_301, vendor: Oracle Corporation, runtime:
/Library/Java/JavaVirtualMachines/jdk1.8.0_301.jdk/Contents/Home/jre
+Default locale: en_CN, platform encoding: UTF-8
+OS name: "mac os x", version: "10.16", arch: "x86_64", family: "mac"
+(base) xiaoxiang.yu@XXYU-MBP ~ % nvm -v
+0.39.1
+(base) xiaoxiang.yu@XXYU-MBP ~ % nvm list
+-> v12.14.0
+ v16.16.0
+ system
+default -> 12.14.0 (-> v12.14.0)
+(base) xiaoxiang.yu@XXYU-MBP ~ % node -v
+v12.14.0
+(base) xiaoxiang.yu@XXYU-MBP ~ % docker -v
+Docker version 24.0.5, build ced0996
+(base) xiaoxiang.yu@XXYU-MBP ~ % date
+Wed Sep 20 11:15:45 CST 2023
+```
+
+Make sure these port is available to use.
+
+### STEP 6: Prepare a linux machine(Optional)
+kylin@worker-03:~$ docker -v
+Docker version 20.10.17, build 100c701
+kylin@worker-03:~$ uname -a
+Linux worker-03 5.4.0-135-generic #152-Ubuntu SMP Wed Nov 23 20:19:22 UTC 2022
x86_64 x86_64 x86_64 GNU/Linux
+kylin@worker-03:~$ free -h
+total used free shared buff/cache available
+Mem: 47Gi 2.8Gi 37Gi 2.0Mi 8.0Gi 44Gi
+Swap: 8.0Gi 0B 8.0Gi
+
+### Terms
+
+| Term | Comment |
+|:--------------|:---------|
+| Build Engine | |
+| Job Engine | |
+| Query Engine | |
+| Metadata | |
+
+### Attention
+This guide is verified using this [verified commit](todo) in 2023-09-28
+by xxyu, requiring macOS and my recommend software.
+
+Since future commits may change behavior or have a chance to break something,
+you are suggested to go through this guide using commit which I verified.
+
+----
+## Part Two - Set up Development
+
+
+### STEP 1: Fetch source code
+
+```shell
+git clone https://github.com/apache/kylin.git --single-branch --branch
kylin5-tmp demo-kylin5-local-run
+export PROJECT_DIR=~/demo-kylin5-local-run
+```
+
+> **Attention**: <br>
+> And root path of source code is replaced with $PROJECT_DIR at following doc.
<br>
+> All following commands are executed in $PROJECT_DIR .
+
+### STEP 2: Build source using maven
+
+```shell
+mvn clean install -DskipTests
+```
+
+Why you need run `mvn install/package` ?
+1. Some jars are built by `maven-shade-plugin` in `$PROJECT_DIR/src/assembly`,
and it is required in later
step(`$PROJECT_DIR/src/assembly/target/kylin-assembly-5.0.0-SNAPSHOT-job.jar`)
+2. Some source to be generated by javacc-plugin.
+
+This process may take 10-20 minutes.
+
+```shell
+[INFO] ------------------------------------------------------------------------
+[INFO] Reactor Summary for Apache Kylin 5 5.0.0-SNAPSHOT:
+[INFO]
+[INFO] Apache Kylin 5 ..................................... SUCCESS [ 19.699 s]
+[INFO] Kylin - Core Common ................................ SUCCESS [02:10 min]
+[INFO] Kylin - Core Metadata .............................. SUCCESS [01:45 min]
+[INFO] Kylin - Core Metrics ............................... SUCCESS [ 27.640 s]
+[INFO] Kylin - Core Job ................................... SUCCESS [ 27.562 s]
+[INFO] Kylin - Core Storage ............................... SUCCESS [ 5.791 s]
+[INFO] Kylin - Query Common ............................... SUCCESS [ 17.910 s]
+[INFO] Kylin - Local Data Cache ........................... SUCCESS [ 33.482 s]
+[INFO] Kylin - Spark Common ............................... SUCCESS [01:14 min]
+[INFO] Kylin - Query Engine Spark ......................... SUCCESS [02:04 min]
+[INFO] Kylin - Hive Source ................................ SUCCESS [ 11.103 s]
+[INFO] Kylin - Build Engine ............................... SUCCESS [ 3.453 s]
+[INFO] Kylin - Distributed Lock Extension ................. SUCCESS [ 6.703 s]
+[INFO] Kylin - Build Engine Spark ......................... SUCCESS [01:45 min]
+[INFO] Kylin - Query ...................................... SUCCESS [ 34.497 s]
+[INFO] Kylin - Streaming SDK .............................. SUCCESS [ 3.539 s]
+[INFO] Kylin - Streaming .................................. SUCCESS [ 57.014 s]
+[INFO] Kylin - Tool ....................................... SUCCESS [ 28.052 s]
+[INFO] Kylin - Common Service ............................. SUCCESS [ 26.044 s]
+[INFO] Kylin - Datasource Service ......................... SUCCESS [ 26.970 s]
+[INFO] Kylin - Modeling Service ........................... SUCCESS [ 29.045 s]
+[INFO] Kylin - Data Loading Service ....................... SUCCESS [ 21.783 s]
+[INFO] Kylin - Query Service .............................. SUCCESS [ 24.721 s]
+[INFO] Kylin - Common Server .............................. SUCCESS [ 11.979 s]
+[INFO] Kylin - Job Service ................................ SUCCESS [ 11.572 s]
+[INFO] Kylin - Streaming Service .......................... SUCCESS [ 8.576 s]
+[INFO] Kylin - Data Loading Server ........................ SUCCESS [ 9.981 s]
+[INFO] Kylin - Query Server ............................... SUCCESS [ 22.769 s]
+[INFO] Kylin - Metadata Server ............................ SUCCESS [ 12.105 s]
+[INFO] Kylin - REST Server ................................ SUCCESS [ 16.878 s]
+[INFO] Kylin - Datasource SDK ............................. SUCCESS [ 6.534 s]
+[INFO] Kylin - JDBC Source ................................ SUCCESS [ 16.812 s]
+[INFO] Kylin - Integration Test ........................... SUCCESS [ 26.048 s]
+[INFO] Kylin - Integration Test Spark ..................... SUCCESS [ 17.564 s]
+[INFO] Kylin - Source Assembly ............................ SUCCESS [ 51.755 s]
+[INFO] Kylin - Integration Test Server .................... SUCCESS [ 15.409 s]
+[INFO] Kylin - Data loading Booter ........................ SUCCESS [ 6.654 s]
+[INFO] Kylin - Query Booter ............................... SUCCESS [ 5.126 s]
+[INFO] Kylin - Common Booter .............................. SUCCESS [ 4.365 s]
+[INFO] Kylin - JDBC Driver ................................ SUCCESS [ 7.785 s]
+[INFO] ------------------------------------------------------------------------
+[INFO] BUILD SUCCESS
+[INFO] ------------------------------------------------------------------------
+[INFO] Total time: 20:02 min
+[INFO] Finished at: 2023-09-20T11:25:25+08:00
+[INFO] ------------------------------------------------------------------------
+```
+
+### STEP 3: Build source of front-end
+
+```shell
+cd kystudio
+npm install
+```
+
+### Step 4: Download Spark
+
+```shell
+bash build/release/download-spark.sh
+```
+
+Result of `ls -al build/spark/jars | wc -l` should be 282.
+By default, the `SPARK_HOME` is pointed to `$PROJECT_DIR/build/spark` .
+
+You are free to move spark directory to other place.
+
+### STEP 5: Import and configure project in IDE
+
+#### 1. Install required IDEA plugins
+- Scala Plugin
+- Lombok Plugin
+- JavaCC Plugin
+
+#### 2. Set up Java SDK and Scala SDK for project
+
+- JDK 1.8
+- Scala 2.12
+
+#### 3. Load Maven modules and build project
+
+1. Load Maven modules
+ 1. Find Maven tool window <br>
+ Find the tools bar of IDE, click following text in order: <br>
+ "View" -> "Tool Windows" -> "Maven"
+ 2. Click "Reload All Maven Projects"
+ 3. Wait all process finished
+2. Build project in IDE
+ 1. Find the tools bar of IDE, click following text in order:<br>
+ "Build" -> "Build Project"
+ 2. Wait all process finished
+3. Check build status by running any ut (for example `HashFunctionTest`).
+
+### STEP 7: Start Hadoop Containers
+
+1. If you decided to running Hadoop processes in docker containers, you can
export `DOCKER_HOST` to correct host/ip.
+ ```shell
+ export DOCKER_HOST=ssh://kylin@worker-03
+ ```
+2. Start Hadoop processes in Docker by:
+ ```shell
+ docker compose -f
"${PROJECT_DIR}/dev-support/contributor/sandbox/docker-compose.yml" up
+ ```
+3. Check status of Hadoop processes in Docker by:
+ ```shell
+ docker compose -f
"${PROJECT_DIR}/dev-support/contributor/sandbox/docker-compose.yml" ps
+ ```
+4. Init metadata(MySQL) for Kylin
+ Use `docker exec -it mysql bash`
+ ```shell
+ mysql -uroot -proot
+ CREATE DATABASE IF NOT EXISTS kylin;
+ ```
+5. Init working dir for Kylin in HDFS
+ Use `docker exec -it namenode bash`
+ ```shell
+ hadoop dfs -mkdir -p '/kylin/spark-history'
+ ```
+6. Prepare sample data
+ ```shell
+ docker cp ${PROJECT_DIR}/src/examples/sample_cube/data datanode:/tmp/ssb
+ docker cp
${PROJECT_DIR}/src/examples/sample_cube/create_sample_ssb_tables.sql
hiveserver:/tmp/
+
+ docker exec datanode bash -c "hdfs dfs -mkdir -p /tmp/sample_cube/data \
+ && hdfs dfs -put /tmp/ssb/* /tmp/sample_cube/data/"
+ docker exec hiveserver bash -c "hive -e 'CREATE DATABASE IF NOT EXISTS SSB'
\
+ && hive --hivevar hdfs_tmp_dir=/tmp --database SSB -f
/tmp/create_sample_ssb_tables.sql"
+ ```
+
+### STEP 8: Running Configuration
+
+IDEA official reference is
https://www.jetbrains.com/help/idea/run-debug-configuration.html
https://www.jetbrains.com/help/idea/run-debug-configuration-java-application.html#more_options.
+
+#### ENVIRONMENT VAR
+
+```shell
+KYLIN_HOME=$PROJECT_DIR
+KYLIN_CONF=$PROJECT_DIR/dev-support/contributor/sandbox/conf
+SPARK_HOME=/Users/xiaoxiang.yu/LacusDir/kyspark
+HADOOP_CONF_DIR=$PROJECT_DIR/dev-support/contributor/sandbox/conf
+HADOOP_USER_NAME=root
+```
+
+#### SYSTEM PROPERTY
+- Spring Framework
+spring.profiles.active=sandbox,docker
+
+#### MAIN CLASS
+org.apache.kylin.rest.BootstrapServer
+
+#### CLASSPATH
+Module **kylin-server**, and add option **INCLUDE_PROVIDED_SCOPE**
+
+#### WORKING DIR
+Module **kylin-server**, or %MODULE_WORKING_DIR% .
+
+### STEP 8.1: More Misc Configuration
+
+#### Log4j2 Configuration
+Modify $PROJECT_DIR/src/server/src/resources/log4j2.xml to change log level
etc.
+Add `-Dlog4j2.debug` in vm options if you need debug.
+
+#### Kylin Configuration
+$PROJECT_DIR/dev-support/contributor/sandbox/conf/kylin.properties
+
+#### Calcite Debug
+
+#### Spring Configuration
+
+### Important Breakpoints
+
+| Class Name | Desc |
+|----------------------------------|------|
+| KylinPrepareEnvListener | |
+| AppInitializer | |
+| BootstrapServer | |
+| KylinPropertySourceConfiguration | |
+----
+
+## Troubles shooting & Tips
+
+#### How to clean up
+
+- Remove docker containers
+```shell
+docker compose -f
"${PROJECT_DIR}/dev-support/contributor/sandbox/docker-compose.yml" down
+```
+
+- Uninstall IDEA https://www.jetbrains.com/help/idea/uninstall.html#standalone
+```shell
+rm -rf /Applications/IntelliJ\ IDEA\ CE.app
+rm -rf ~/Library/Application\ Support/JetBrains/IdeaIC*
+rm -rf ~/Library/Caches/JetBrains/IdeaIC*
+```
+
+- Clean local installed snapshot artificial
+```shell
+rm -rf ~/.m2/repository/org/apache/kylin
+```
+
+- Clean source code
+```shell
+rm -rf $PROJECT_DIR
+```
+
+#### How to check out a specific git commit
+
+```shell
+// todo
+```
+
+#### Fix Invalid method name: 'get_all_functions'
+This is usually caused by hive .
+```sh
+org.apache.hadoop.hive.ql.metadata.HiveException:
org.apache.thrift.TApplicationException: Invalid method name:
'get_all_functions'
+ at
org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3904)
~[hive-exec-2.3.9.jar:2.3.9]
+```
+
+Maybe spark.sql.hive.metastore.jars is pointed to correct path in
`$PROJECT_DIR/dev-support/contributor/sandbox/conf/kylin.properties`.
+
+#### Performance Improvement
+
+When you submit multiple jobs at the same times, you pc might suffer from bad
performance.
+- Reduce the max concurrent number of job.
+
+#### Cannot download logs of build engine
+
+```shell
+ERROR [local_run] [http-nio-7070-exec-9] execution.NExecutableManager : get
sample data from hdfs log file
[/kylin/kylin_metadata/local_run/job_tmp/4edaf665-430a-8cc0-620e-b9f7ae2458bc-98cbd8b9-fee2-44d4-0383-b8b603348e94/01//execute_output.json.1695364459595.log]
failed!
+org.apache.hadoop.hdfs.CannotObtainBlockLengthException: Cannot obtain block
length for
LocatedBlock{BP-258157769-172.22.0.6-1695103912779:blk_1073742602_1778;
getBlockSize()=26457; corrupt=false; offset=0;
locs=[DatanodeInfoWithStorage[172.22.0.3:50010,DS-ca2d8134-89bd-49d1-af55-680cea736111,DISK]]}
of
/kylin/kylin_metadata/local_run/job_tmp/4edaf665-430a-8cc0-620e-b9f7ae2458bc-98cbd8b9-fee2-44d4-0383-b8b603348e94/01/execute_output.json.1695364459595.log
+ at
org.apache.hadoop.hdfs.DFSInputStream.readBlockLength(DFSInputStream.java:470)
~[hadoop-hdfs-client-2.10.1.jar:?]
+```
+
+todo
+
+#### Query Failed by InvalidClassException
+
+```shell
+Caused by: java.io.InvalidClassException:
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat;
+local class incompatible: stream classdesc serialVersionUID =
8961733539262042287, local class serialVersionUID = -27198871445502271
+```
+
+You are using different version of Spark, unify them(1. SPARK_HOME 2.
`spark.version` in $PROJECT_DIR/pom.xml).
\ No newline at end of file
diff --git a/dev-support/local/README.md
b/dev-support/contributor/local/README.md
similarity index 85%
rename from dev-support/local/README.md
rename to dev-support/contributor/local/README.md
index b28a3123d4..23c888bc33 100644
--- a/dev-support/local/README.md
+++ b/dev-support/contributor/local/README.md
@@ -6,7 +6,7 @@
3. Install Docker Desktop on local machine and configure the Docker resource,
as follows is the lowest requirement：
- 
+ 
### Usage
@@ -23,29 +23,29 @@
*Executing Result*
-
+
* Run Configuration for IDEA Debug*
-
+
3) Executing `./dev-support/local/local.sh up` to download images and start
the container。
*Container started successfully*
-
+
4) After start IDEA Debug and access to `http://localhost:8080/` to check
Kylin started successfully.
* Run into Kylin web and enjoy.
-
+
* Access to `http://localhost:4041/` to check sparder started successfully.
-
+
### Other Usage
diff --git a/dev-support/local/docker-compose.yml
b/dev-support/contributor/local/docker-compose.yml
similarity index 100%
rename from dev-support/local/docker-compose.yml
rename to dev-support/contributor/local/docker-compose.yml
diff --git a/dev-support/local/images/docker_container.png
b/dev-support/contributor/local/images/docker_container.png
similarity index 100%
rename from dev-support/local/images/docker_container.png
rename to dev-support/contributor/local/images/docker_container.png
diff --git a/dev-support/local/images/docker_resources.png
b/dev-support/contributor/local/images/docker_resources.png
similarity index 100%
rename from dev-support/local/images/docker_resources.png
rename to dev-support/contributor/local/images/docker_resources.png
diff --git a/dev-support/local/images/executing_result.png
b/dev-support/contributor/local/images/executing_result.png
similarity index 100%
rename from dev-support/local/images/executing_result.png
rename to dev-support/contributor/local/images/executing_result.png
diff --git a/dev-support/local/images/idea_configuration.png
b/dev-support/contributor/local/images/idea_configuration.png
similarity index 100%
rename from dev-support/local/images/idea_configuration.png
rename to dev-support/contributor/local/images/idea_configuration.png
diff --git a/dev-support/local/images/kylin_ui.png
b/dev-support/contributor/local/images/kylin_ui.png
similarity index 100%
rename from dev-support/local/images/kylin_ui.png
rename to dev-support/contributor/local/images/kylin_ui.png
diff --git a/dev-support/local/images/run_configuration_replace.png
b/dev-support/contributor/local/images/run_configuration_replace.png
similarity index 100%
rename from dev-support/local/images/run_configuration_replace.png
rename to dev-support/contributor/local/images/run_configuration_replace.png
diff --git a/dev-support/local/images/sparder.png
b/dev-support/contributor/local/images/sparder.png
similarity index 100%
rename from dev-support/local/images/sparder.png
rename to dev-support/contributor/local/images/sparder.png
diff --git a/dev-support/local/local.sh b/dev-support/contributor/local/local.sh
similarity index 100%
rename from dev-support/local/local.sh
rename to dev-support/contributor/local/local.sh
diff --git a/dev-support/local/runConfigurations/BootstrapServer_local.xml
b/dev-support/contributor/local/runConfigurations/BootstrapServer_local.xml
similarity index 100%
rename from dev-support/local/runConfigurations/BootstrapServer_local.xml
rename to
dev-support/contributor/local/runConfigurations/BootstrapServer_local.xml
diff --git
a/dev-support/local/runConfigurations/BootstrapServer_local_community.xml
b/dev-support/contributor/local/runConfigurations/BootstrapServer_local_community.xml
similarity index 100%
rename from
dev-support/local/runConfigurations/BootstrapServer_local_community.xml
rename to
dev-support/contributor/local/runConfigurations/BootstrapServer_local_community.xml
diff --git a/dev-support/sandbox/.gitignore
b/dev-support/contributor/sandbox/.gitignore
similarity index 100%
rename from dev-support/sandbox/.gitignore
rename to dev-support/contributor/sandbox/.gitignore
diff --git a/dev-support/sandbox/conf/capacity-scheduler.xml
b/dev-support/contributor/sandbox/conf/capacity-scheduler.xml
similarity index 100%
rename from dev-support/sandbox/conf/capacity-scheduler.xml
rename to dev-support/contributor/sandbox/conf/capacity-scheduler.xml
diff --git a/dev-support/sandbox/conf/core-site.xml
b/dev-support/contributor/sandbox/conf/core-site.xml
similarity index 100%
rename from dev-support/sandbox/conf/core-site.xml
rename to dev-support/contributor/sandbox/conf/core-site.xml
diff --git a/dev-support/sandbox/conf/fairscheduler.xml
b/dev-support/contributor/sandbox/conf/fairscheduler.xml
similarity index 100%
rename from dev-support/sandbox/conf/fairscheduler.xml
rename to dev-support/contributor/sandbox/conf/fairscheduler.xml
diff --git a/dev-support/sandbox/conf/hadoop/core-site.xml
b/dev-support/contributor/sandbox/conf/hadoop/core-site.xml
similarity index 100%
rename from dev-support/sandbox/conf/hadoop/core-site.xml
rename to dev-support/contributor/sandbox/conf/hadoop/core-site.xml
diff --git a/dev-support/sandbox/conf/hdfs-site.xml
b/dev-support/contributor/sandbox/conf/hdfs-site.xml
similarity index 100%
rename from dev-support/sandbox/conf/hdfs-site.xml
rename to dev-support/contributor/sandbox/conf/hdfs-site.xml
diff --git a/dev-support/sandbox/conf/hive-site.xml
b/dev-support/contributor/sandbox/conf/hive-site.xml
similarity index 100%
rename from dev-support/sandbox/conf/hive-site.xml
rename to dev-support/contributor/sandbox/conf/hive-site.xml
diff --git a/dev-support/sandbox/conf/kylin.properties
b/dev-support/contributor/sandbox/conf/kylin.properties
similarity index 100%
rename from dev-support/sandbox/conf/kylin.properties
rename to dev-support/contributor/sandbox/conf/kylin.properties
diff --git a/dev-support/sandbox/conf/yarn-site.xml
b/dev-support/contributor/sandbox/conf/yarn-site.xml
similarity index 100%
rename from dev-support/sandbox/conf/yarn-site.xml
rename to dev-support/contributor/sandbox/conf/yarn-site.xml
diff --git a/dev-support/sandbox/docker-compose.yml
b/dev-support/contributor/sandbox/docker-compose.yml
similarity index 100%
rename from dev-support/sandbox/docker-compose.yml
rename to dev-support/contributor/sandbox/docker-compose.yml
diff --git a/dev-support/sandbox/hadoop.env
b/dev-support/contributor/sandbox/hadoop.env
similarity index 100%
rename from dev-support/sandbox/hadoop.env
rename to dev-support/contributor/sandbox/hadoop.env
diff --git a/dev-support/sandbox/hive.env
b/dev-support/contributor/sandbox/hive.env
similarity index 100%
rename from dev-support/sandbox/hive.env
rename to dev-support/contributor/sandbox/hive.env
diff --git
a/dev-support/sandbox/runConfigurations/BootstrapServer_docker_sandbox_.xml
b/dev-support/contributor/sandbox/runConfigurations/BootstrapServer_docker_sandbox_.xml
similarity index 93%
rename from
dev-support/sandbox/runConfigurations/BootstrapServer_docker_sandbox_.xml
rename to
dev-support/contributor/sandbox/runConfigurations/BootstrapServer_docker_sandbox_.xml
index c361f984f0..09022c8536 100644
--- a/dev-support/sandbox/runConfigurations/BootstrapServer_docker_sandbox_.xml
+++
b/dev-support/contributor/sandbox/runConfigurations/BootstrapServer_docker_sandbox_.xml
@@ -18,9 +18,9 @@
<configuration default="false" name="BootstrapServer[docker-sandbox]"
type="Application" factoryName="Application">
<envs>
<env name="KYLIN_HOME" value="$PROJECT_DIR" />
- <env name="KYLIN_CONF" value="$PROJECT_DIR/dev-support/sandbox/conf" />
+ <env name="KYLIN_CONF"
value="$PROJECT_DIR/dev-support/contributor/sandbox/conf" />
<env name="SPARK_HOME" value="$PROJECT_DIR/build/spark" />
- <env name="HADOOP_CONF_DIR"
value="$PROJECT_DIR/dev-support/sandbox/conf" />
+ <env name="HADOOP_CONF_DIR"
value="$PROJECT_DIR/dev-support/contributor/sandbox/conf" />
<env name="HADOOP_USER_NAME" value="root" />
</envs>
<option name="INCLUDE_PROVIDED_SCOPE" value="true" />
diff --git
a/dev-support/sandbox/runConfigurations/BootstrapServer_docker_sandbox_ultimate.xml
b/dev-support/contributor/sandbox/runConfigurations/BootstrapServer_docker_sandbox_ultimate.xml
similarity index 100%
rename from
dev-support/sandbox/runConfigurations/BootstrapServer_docker_sandbox_ultimate.xml
rename to
dev-support/contributor/sandbox/runConfigurations/BootstrapServer_docker_sandbox_ultimate.xml
diff --git a/dev-support/sandbox/sandbox.sh
b/dev-support/contributor/sandbox/sandbox.sh
similarity index 98%
rename from dev-support/sandbox/sandbox.sh
rename to dev-support/contributor/sandbox/sandbox.sh
index 9c2420f01a..76e886da43 100755
--- a/dev-support/sandbox/sandbox.sh
+++ b/dev-support/contributor/sandbox/sandbox.sh
@@ -48,7 +48,7 @@ function error() {
}
PROJECT_DIR=$(cd -P -- "$(dirname -- "$0")/../.." && pwd -P)
-WORKDIR=$(cd -P -- "${PROJECT_DIR}/dev-support/sandbox" && pwd -P)
+WORKDIR=$(cd -P -- "${PROJECT_DIR}/dev-support/contributor/sandbox" && pwd -P)
KYLIN_HOME="${PROJECT_DIR}"
warn "# KYLIN_HOME=${KYLIN_HOME}"
diff --git a/dev-support/release-manager/README.md
b/dev-support/release-manager/README.md
new file mode 100644
index 0000000000..0eecd95bab
--- /dev/null
+++ b/dev-support/release-manager/README.md
@@ -0,0 +1 @@
+# How to make an apache release for Kylin 5
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/Dockerfile_1
b/dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_1
similarity index 100%
rename from build/release/release-pipeline-docker/release-machine/Dockerfile_1
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_1
diff --git a/build/release/release-pipeline-docker/release-machine/Dockerfile_2
b/dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_2
similarity index 100%
rename from build/release/release-pipeline-docker/release-machine/Dockerfile_2
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/Dockerfile_2
diff --git a/build/release/release-pipeline-docker/release-machine/README.md
b/dev-support/release-manager/release-pipeline-docker/release-machine/README.md
similarity index 100%
rename from build/release/release-pipeline-docker/release-machine/README.md
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/README.md
diff --git
a/build/release/release-pipeline-docker/release-machine/conf/settings.xml
b/dev-support/release-manager/release-pipeline-docker/release-machine/conf/settings.xml
similarity index 100%
rename from
build/release/release-pipeline-docker/release-machine/conf/settings.xml
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/conf/settings.xml
diff --git
a/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
b/dev-support/release-manager/release-pipeline-docker/release-machine/create-release-machine.sh
similarity index 100%
rename from
build/release/release-pipeline-docker/release-machine/create-release-machine.sh
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/create-release-machine.sh
diff --git
a/build/release/release-pipeline-docker/release-machine/entrypoint.sh
b/dev-support/release-manager/release-pipeline-docker/release-machine/entrypoint.sh
similarity index 100%
rename from build/release/release-pipeline-docker/release-machine/entrypoint.sh
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/entrypoint.sh
diff --git
a/build/release/release-pipeline-docker/release-machine/release-publish.sh
b/dev-support/release-manager/release-pipeline-docker/release-machine/release-publish.sh
similarity index 100%
rename from
build/release/release-pipeline-docker/release-machine/release-publish.sh
rename to
dev-support/release-manager/release-pipeline-docker/release-machine/release-publish.sh
diff --git a/dev-support/release-manager/release-process-definition.md
b/dev-support/release-manager/release-process-definition.md
new file mode 100644
index 0000000000..bccb2caeb6
--- /dev/null
+++ b/dev-support/release-manager/release-process-definition.md
@@ -0,0 +1,13 @@
+## Definition of release process
+
+
+### All Steps
+- [ ] Release maven artificial
+- [ ] Release source code
+- [ ] Release binary
+- [ ] Call a vote for RC
+- [ ] Announce RC 's result of a vote
+- [ ] Upload release
+- [ ] Publish website
+- [ ] Announce release
+- [ ] Build and publish standalone docker image for release
\ No newline at end of file
diff --git a/build/release/all-in-one-docker/.gitignore
b/dev-support/release-manager/standalone-docker/.gitignore
similarity index 100%
rename from build/release/all-in-one-docker/.gitignore
rename to dev-support/release-manager/standalone-docker/.gitignore
diff --git a/build/release/all-in-one-docker/all_in_one/Dockerfile_hadoop
b/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_hadoop
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/Dockerfile_hadoop
rename to
dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_hadoop
diff --git a/build/release/all-in-one-docker/all_in_one/Dockerfile_kylin
b/dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/Dockerfile_kylin
rename to
dev-support/release-manager/standalone-docker/all_in_one/Dockerfile_kylin
diff --git a/build/release/all-in-one-docker/all_in_one/build_image.sh
b/dev-support/release-manager/standalone-docker/all_in_one/build_image.sh
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/build_image.sh
rename to
dev-support/release-manager/standalone-docker/all_in_one/build_image.sh
diff --git
a/build/release/all-in-one-docker/all_in_one/conf/hadoop/capacity-scheduler.xml
b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/capacity-scheduler.xml
similarity index 100%
rename from
build/release/all-in-one-docker/all_in_one/conf/hadoop/capacity-scheduler.xml
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/capacity-scheduler.xml
diff --git
a/build/release/all-in-one-docker/all_in_one/conf/hadoop/core-site.xml
b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/core-site.xml
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/conf/hadoop/core-site.xml
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/core-site.xml
diff --git
a/build/release/all-in-one-docker/all_in_one/conf/hadoop/hdfs-site.xml
b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/hdfs-site.xml
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/conf/hadoop/hdfs-site.xml
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/hdfs-site.xml
diff --git
a/build/release/all-in-one-docker/all_in_one/conf/hadoop/mapred-site.xml
b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/mapred-site.xml
similarity index 100%
rename from
build/release/all-in-one-docker/all_in_one/conf/hadoop/mapred-site.xml
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/mapred-site.xml
diff --git
a/build/release/all-in-one-docker/all_in_one/conf/hadoop/yarn-site.xml
b/dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/yarn-site.xml
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/conf/hadoop/yarn-site.xml
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/hadoop/yarn-site.xml
diff --git a/build/release/all-in-one-docker/all_in_one/conf/hive/hive-site.xml
b/dev-support/release-manager/standalone-docker/all_in_one/conf/hive/hive-site.xml
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/conf/hive/hive-site.xml
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/hive/hive-site.xml
diff --git
a/build/release/all-in-one-docker/all_in_one/conf/kylin/kylin.properties.override
b/dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
similarity index 100%
rename from
build/release/all-in-one-docker/all_in_one/conf/kylin/kylin.properties.override
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/kylin/kylin.properties.override
diff --git a/build/release/all-in-one-docker/all_in_one/conf/zk/zoo.cfg
b/dev-support/release-manager/standalone-docker/all_in_one/conf/zk/zoo.cfg
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/conf/zk/zoo.cfg
rename to
dev-support/release-manager/standalone-docker/all_in_one/conf/zk/zoo.cfg
diff --git a/build/release/all-in-one-docker/all_in_one/entrypoint.sh
b/dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/entrypoint.sh
rename to dev-support/release-manager/standalone-docker/all_in_one/entrypoint.sh
diff --git
a/build/release/all-in-one-docker/all_in_one/scripts/install_mysql.sh
b/dev-support/release-manager/standalone-docker/all_in_one/scripts/install_mysql.sh
similarity index 100%
rename from build/release/all-in-one-docker/all_in_one/scripts/install_mysql.sh
rename to
dev-support/release-manager/standalone-docker/all_in_one/scripts/install_mysql.sh
diff --git a/dev-support/sandbox/README.md b/dev-support/sandbox/README.md
deleted file mode 100644
index 3588991f20..0000000000
--- a/dev-support/sandbox/README.md
+++ /dev/null
@@ -1,114 +0,0 @@
-### 使用条件：
-
-1. 本地安装 JDK 1.8 版本, Docker 3.8 及以上版本
-
-2. 手动执行 `mvn clean install -DskipTests` 并确保通过编译
-
-3. 本地安装 Docker Desktop，并配置 Docker 容器资源，如下为最低要求：
-
- 
-
-
-
-### 使用步骤：
-
-1）在 `/etc/hosts` 中添加一下内容
-
-```bash
-127.0.0.1 namenode datanode resourcemanager nodemanager historyserver mysql
zookeeper hivemetastore hiveserver
-```
-
-
-2）执行 `./dev-support/sandbox/sandbox.sh init` 初始化本地沙箱环境。
-
-- 该步骤会自动下载 Spark 至本地，若下载失败会提示开发者手动进行下载。需要注意，如果当前 Spark 已经存在，程序会删除它并重新下载
-- 该步骤会自动装载 IDEA Debug 配置
-
-*执行结果如图：*
-
-
-
-*自动装载 IDEA Debug 配置如图：*
-
-
-
-
-
-3）执行 `./dev-support/sandbox/sandbox.sh up` 下载镜像并启动容器。
-
-- 执行过程完成 KE 元数据库的创建，如果失败，可以按照提示手动创建（默认会自动重试 5 次）
-
-*容器启动成功如图：*
-
-
-
-
-
-4）启动 IDEA Debug，启动成功后访问 `http://resourcemanager:8088/` 能够看到 Sparder 已经启动，如图：
-
-
-
-
-
-5）执行 `./dev-support/sandbox/sandbox.sh sample` 加载 SSB 数据
-
-- 该过程会往 Hive 和 Kafka 里面分别灌入 SSB 数据
-
-
-
-*进入到 KE 数据源页面，能够正常加载 Hive 数据：*
-
-
-
-
-*查看批数据任务列表，数据采样成功*
-
-
-
-
-
-*在查询分析页面，也能正常 Pushdown 到 Hive 获取数据*
-
-
-
-
-
-*创建模型并构建索引*
-
-
-
-
-
-*重新查询数据，能够正常命中索引*
-
-
-
-
-
-### 其他使用方式
-
-1）`./dev-support/sandbox/sandbox.sh stop` 停止正在运行的所有容器
-
-2）`./dev-support/sandbox/sandbox.sh start` 启动所有停止了的容器
-
-3）`./dev-support/sandbox/sandbox.sh ps` 查看所有正在运行的容器
-
-4）`./dev-support/sandbox/sandbox.sh down` 停止所有容器，并删除它们
-
-5）`./dev-support/sandbox/sandbox.sh interactive` 进入指定容器
-
-### FAQ
-
-1）当出现如下图 `No Filesystem for schema: hdfs` 时，重新刷新 Maven Profile 即可
-
-
-
-
-
-2）当执行 `sandbox.sh init` 后，runConfiguration 没有生成可用的配置时，替换
`.idea/runconfigurations/BootstrapServer_sandbox_.xml` 中的
`type="SpringBootApplicationConfigurationType" factoryName="Spring Boot"` 为
`type="Application" factoryName="Application"`
-
-
-
-3）当在本地开发和 Spark 相关功能时，需要访问 Spark Web UI，可以临时修改 `KylinSession.scala` 中的代码，如图：
-
-
diff --git a/dev-support/sandbox/images/6nP8shbaNX.png
b/dev-support/sandbox/images/6nP8shbaNX.png
deleted file mode 100644
index be22bb7e99..0000000000
Binary files a/dev-support/sandbox/images/6nP8shbaNX.png and /dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102105411171.png
b/dev-support/sandbox/images/image-20211102105411171.png
deleted file mode 100644
index ce043d9eaf..0000000000
Binary files a/dev-support/sandbox/images/image-20211102105411171.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102105446286.png
b/dev-support/sandbox/images/image-20211102105446286.png
deleted file mode 100644
index 2e1366736a..0000000000
Binary files a/dev-support/sandbox/images/image-20211102105446286.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102110553460.png
b/dev-support/sandbox/images/image-20211102110553460.png
deleted file mode 100644
index fc41b84ef6..0000000000
Binary files a/dev-support/sandbox/images/image-20211102110553460.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102111037483.png
b/dev-support/sandbox/images/image-20211102111037483.png
deleted file mode 100644
index a9f115306a..0000000000
Binary files a/dev-support/sandbox/images/image-20211102111037483.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102111253713.png
b/dev-support/sandbox/images/image-20211102111253713.png
deleted file mode 100644
index a902f4aba2..0000000000
Binary files a/dev-support/sandbox/images/image-20211102111253713.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102111745414.png
b/dev-support/sandbox/images/image-20211102111745414.png
deleted file mode 100644
index b4e462c22d..0000000000
Binary files a/dev-support/sandbox/images/image-20211102111745414.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102111806627.png
b/dev-support/sandbox/images/image-20211102111806627.png
deleted file mode 100644
index acb714b800..0000000000
Binary files a/dev-support/sandbox/images/image-20211102111806627.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102112221619.png
b/dev-support/sandbox/images/image-20211102112221619.png
deleted file mode 100644
index 6e8addc04c..0000000000
Binary files a/dev-support/sandbox/images/image-20211102112221619.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102112351003.png
b/dev-support/sandbox/images/image-20211102112351003.png
deleted file mode 100644
index 280548a425..0000000000
Binary files a/dev-support/sandbox/images/image-20211102112351003.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102113005133.png
b/dev-support/sandbox/images/image-20211102113005133.png
deleted file mode 100644
index f43d93bb72..0000000000
Binary files a/dev-support/sandbox/images/image-20211102113005133.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102113055804.png
b/dev-support/sandbox/images/image-20211102113055804.png
deleted file mode 100644
index db9ca8f344..0000000000
Binary files a/dev-support/sandbox/images/image-20211102113055804.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/image-20211102113605007.png
b/dev-support/sandbox/images/image-20211102113605007.png
deleted file mode 100644
index af4389ce38..0000000000
Binary files a/dev-support/sandbox/images/image-20211102113605007.png and
/dev/null differ
diff --git a/dev-support/sandbox/images/img.png
b/dev-support/sandbox/images/img.png
deleted file mode 100644
index 35dd277269..0000000000
Binary files a/dev-support/sandbox/images/img.png and /dev/null differ
diff --git a/dev-support/sandbox/images/img_1.png
b/dev-support/sandbox/images/img_1.png
deleted file mode 100644
index 11a1912c31..0000000000
Binary files a/dev-support/sandbox/images/img_1.png and /dev/null differ
diff --git
a/dev-support/sandbox/images/img_8835f30f-7805-4951-808d-45469370fa7g.png
b/dev-support/sandbox/images/img_8835f30f-7805-4951-808d-45469370fa7g.png
deleted file mode 100644
index a10ad96a30..0000000000
Binary files
a/dev-support/sandbox/images/img_8835f30f-7805-4951-808d-45469370fa7g.png and
/dev/null differ
diff --git a/scalastyle-config.xml b/dev-support/scalastyle-config.xml
similarity index 100%
rename from scalastyle-config.xml
rename to dev-support/scalastyle-config.xml
diff --git a/pom.xml b/pom.xml
index 1c68857e51..e15e2e4ad2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,7 +126,7 @@
<!-- Spark versions -->
<delta.version>2.0.2</delta.version>
- <spark.version>3.2.0-kylin-4.6.10.0</spark.version>
+ <spark.version>3.2.0-kylin-4.6.9.0</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
@@ -506,6 +506,20 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kap-second-storage-core</artifactId>
<version>5.0.0-beta</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-job</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-spark-common</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.kylin</groupId>-->
@@ -3161,7 +3175,7 @@
<includeTestSourceDirectory>false</includeTestSourceDirectory>
<failOnWarning>false</failOnWarning>
<sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
- <configLocation>scalastyle-config.xml</configLocation>
+
<configLocation>dev-support/scalastyle-config.xml</configLocation>
<outputFile>${basedir}/target/scalastyle-output.xml</outputFile>
<inputEncoding>${project.build.sourceEncoding}</inputEncoding>
<outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index 9a93d3f596..e700fb0801 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -61,7 +61,7 @@ public class KylinPrepareEnvListener implements
EnvironmentPostProcessor, Ordere
if (env.acceptsProfiles(Profiles.of("sandbox"))) {
if (env.acceptsProfiles(Profiles.of("docker"))) {
- setSandboxEnvs("../../dev-support/sandbox/conf");
+ setSandboxEnvs("../../dev-support/contributor/sandbox/conf");
} else {
setSandboxEnvs("../examples/test_case_data/sandbox");
}
diff --git a/src/spark-project/engine-spark/pom.xml
b/src/spark-project/engine-spark/pom.xml
index 692eca1919..188e19a32d 100644
--- a/src/spark-project/engine-spark/pom.xml
+++ b/src/spark-project/engine-spark/pom.xml
@@ -44,6 +44,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-job</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-sparder</artifactId>
