This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b0c7326856cc6ef8e55212f3a383503119da15c4 Author: qianhao.zhou <z.qian...@gmail.com> AuthorDate: Tue Nov 29 12:08:03 2022 +0800 KYLIN-5428 Module refactor * refactor sys-tools & kylin-tool * fix core-storage UT * fix kylin-tool pom * refactor core-job * refactor core-job * refactor core-job * fix code smell * fix UT Co-authored-by: qhzhou <qianhao.z...@kyligence.io> --- pom.xml | 16 +- src/common-server/pom.xml | 5 + src/common-service/pom.xml | 215 ++++----------------- .../org/apache/kylin/rest/cache/KylinEhCache.java | 0 .../apache/kylin/rest/security/LoginLogFilter.java | 0 .../apache/kylin/rest/cache/KylinEhCacheTest.java | 5 +- .../apache/kylin/storage/cache/EhcacheTest.java | 0 src/common-service/src/test/resources/ehcache.xml | 2 +- .../src/test/resources/user-defined-ehcache.xml | 0 .../org/apache/kylin/rest/cache/KylinCache.java | 0 src/core-job/pom.xml | 40 ++-- .../kylin/common/util/SystemInfoCollector.java | 6 +- .../kylin/job/execution/AbstractExecutable.java | 4 +- src/core-metadata/pom.xml | 33 ++-- .../cube/storage/GarbageStorageCollector.java | 0 .../cube/storage/ProjectStorageInfoCollector.java | 6 +- .../cube/storage/StorageInfoCollector.java | 0 .../metadata/cube/storage/StorageInfoEnum.java | 0 .../cube/storage/StorageQuotaCollector.java | 0 .../metadata/cube/storage/StorageVolumeInfo.java | 0 .../cube/storage/TotalStorageCollector.java | 0 .../metadata/query/CuboidLayoutQueryTimes.java | 6 - .../kylin/metadata/query/QueryStatistics.java | 12 +- .../kylin/metadata/query/QueryTimesResponse.java | 3 - .../apache/kylin/metrics/HdfsCapacityMetrics.java | 0 .../storage/ProjectStorageInfoCollectorTest.java | 0 .../kylin/metrics/HdfsCapacityMetricsTest.java | 0 src/core-metrics/pom.xml | 5 - src/core-storage/pom.xml | 12 +- src/data-loading-server/pom.xml | 4 + src/datasource-service/pom.xml | 9 +- src/kylin-it/pom.xml | 7 +- .../apache/kylin/newten/CalciteDynamicTest.java | 5 +- .../kylin/newten/NBuildAndQuerySnapshotTest.java | 5 +- .../org/apache/kylin/newten/TimeZoneQueryTest.java | 17 +- src/metadata-server/pom.xml | 4 + src/query-server/pom.xml | 11 ++ src/query-service/pom.xml | 10 - .../rest/health/SparkContextHealthIndicator.java | 3 +- .../health/SparkSqlContextHealthIndicator.java | 3 +- .../kylin/rest/service/AysncQueryServiceTest.java | 3 +- src/second-storage/core-ui/pom.xml | 5 + src/second-storage/core/pom.xml | 6 - .../kap/secondstorage/metadata/NodeGroupTest.java | 2 +- src/server/pom.xml | 20 +- .../rest/config/KylinRequestHandlerSelectors.java | 0 .../apache/kylin/rest/config/SecurityConfig.java | 0 .../apache/kylin/rest/config/SwaggerConfig.java | 0 .../rest/health/FileSystemHealthIndicator.java | 6 +- .../rest/health/MetaStoreHealthIndicator.java | 34 +++- .../rest/security/UnauthorisedEntryPoint.java | 0 src/source-hive/pom.xml | 4 - .../kylin/engine/spark/job/NSparkExecutable.java | 4 +- .../engine/spark/job/NSparkSnapshotJobTest.java | 5 +- src/systools/pom.xml | 153 ++------------- .../rest/health/AbstractKylinHealthIndicator.java | 47 ----- .../kylin/rest/security/KerberosLoginManager.java | 4 +- .../apache/kylin/rest/util/AclPermissionUtil.java | 2 +- .../rest/security/KerberosLoginManagerTest.java | 10 +- .../kylin/rest/security/UserAclManagerTest.java | 2 +- src/systools/src/test/resources/ehcache.xml | 19 -- src/tool/pom.xml | 66 +++---- .../org/apache/kylin/tool/RecCandidateTool.java | 2 +- .../apache/kylin/tool/RecCandidateToolTest.java | 0 64 files changed, 234 insertions(+), 608 deletions(-) diff --git a/pom.xml b/pom.xml index 90135eb7ef..fcd1c2c27f 100644 --- a/pom.xml +++ b/pom.xml @@ -594,12 +594,6 @@ <artifactId>kylin-core-metrics</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-streaming</artifactId> - <version>${project.version}</version> - <type>test-jar</type> - </dependency> <dependency> <groupId>org.freemarker</groupId> @@ -1195,11 +1189,6 @@ <version>${commons-cli.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>${commons-lang.version}</version> - </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> @@ -2047,6 +2036,7 @@ <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <version>${lombok.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.owasp.esapi</groupId> @@ -2930,6 +2920,10 @@ <groupId>org.scala-lang</groupId> <artifactId>scala-compiler</artifactId> </dependency> + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + </dependency> </dependencies> <distributionManagement> diff --git a/src/common-server/pom.xml b/src/common-server/pom.xml index 7ac148c88f..0e23257e0a 100644 --- a/src/common-server/pom.xml +++ b/src/common-server/pom.xml @@ -113,6 +113,11 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-log4j2</artifactId> </dependency> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml index e74994f577..66c8a53bfd 100644 --- a/src/common-service/pom.xml +++ b/src/common-service/pom.xml @@ -35,66 +35,63 @@ <artifactId>kylin-tool</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kap-second-storage-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-streaming</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> + <groupId>org.springframework</groupId> + <artifactId>spring-aop</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-tool</artifactId> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-web</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-spark-ddl</artifactId> + <groupId>commons-fileupload</groupId> + <artifactId>commons-fileupload</artifactId> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-config</artifactId> </dependency> <dependency> - <groupId>commons-fileupload</groupId> - <artifactId>commons-fileupload</artifactId> + <groupId>redis.clients</groupId> + <artifactId>jedis</artifactId> </dependency> - <dependency> <groupId>net.sf.ehcache</groupId> <artifactId>ehcache</artifactId> </dependency> - <dependency> - <groupId>cglib</groupId> - <artifactId>cglib</artifactId> - </dependency> - <dependency> - <groupId>com.jayway.jsonpath</groupId> - <artifactId>json-path</artifactId> - </dependency> + <!-- Spring Boot --> <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-jdbc</artifactId> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-actuator</artifactId> + <exclusions> + <exclusion> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-logging</artifactId> + </exclusion> + <exclusion> + <artifactId>spring-context</artifactId> + <groupId>org.springframework</groupId> + </exclusion> + </exclusions> </dependency> <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-aop</artifactId> + <groupId>org.springframework.session</groupId> + <artifactId>spring-session-core</artifactId> </dependency> + <!-- hadoop --> <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-context-support</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> <scope>provided</scope> </dependency> + + <dependency> <groupId>org.springframework</groupId> <artifactId>spring-test</artifactId> @@ -127,142 +124,6 @@ <groupId>org.mockito</groupId> <artifactId>mockito-junit-jupiter</artifactId> <scope>test</scope> - <version>${mockito.version}</version> - </dependency> - - <!-- spring aop --> - <dependency> - <groupId>org.aspectj</groupId> - <artifactId>aspectjrt</artifactId> - </dependency> - <dependency> - <groupId>org.aspectj</groupId> - <artifactId>aspectjweaver</artifactId> - </dependency> - <dependency> - <groupId>redis.clients</groupId> - <artifactId>jedis</artifactId> - </dependency> - - <!-- Spring Boot --> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-autoconfigure</artifactId> - <version>${spring.boot.version}</version> - <exclusions> - <exclusion> - <artifactId>spring-context</artifactId> - <groupId>org.springframework</groupId> - </exclusion> - <exclusion> - <artifactId>spring-core</artifactId> - <groupId>org.springframework</groupId> - </exclusion> - </exclusions> - </dependency> - - <!-- database --> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-dbcp2</artifactId> - </dependency> - - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - </dependency> - - <dependency> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - <version>${xerces.version}</version> - </dependency> - - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-common</artifactId> - <version>${jersey.version}</version> - </dependency> - - <!-- Apache POI --> - <dependency> - <groupId>org.apache.poi</groupId> - <artifactId>poi</artifactId> - <version>${poi.version}</version> - </dependency> - <dependency> - <groupId>org.apache.poi</groupId> - <artifactId>poi-ooxml</artifactId> - <version>${poi.version}</version> - </dependency> - - <!-- spring security --> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-ext-jdk15on</artifactId> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.tomcat.embed</groupId> - <artifactId>tomcat-embed-core</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <version>4.4.5</version> - </dependency> - <dependency> - <groupId>org.apache.calcite.avatica</groupId> - <artifactId>avatica-core</artifactId> - <exclusions> - <exclusion> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-web</artifactId> - </dependency> - - <dependency> - <groupId>org.yaml</groupId> - <artifactId>snakeyaml</artifactId> - </dependency> - - <dependency> - <groupId>javax.validation</groupId> - <artifactId>validation-api</artifactId> - </dependency> - - <!-- hadoop --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - </dependency> - - <!-- test --> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-systools</artifactId> - <type>test-jar</type> - <scope>test</scope> </dependency> <!-- spark --> @@ -272,16 +133,6 @@ <scope>test</scope> </dependency> - <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-api</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-junit-jupiter</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> diff --git a/src/systools/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java b/src/common-service/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java rename to src/common-service/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java b/src/common-service/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java rename to src/common-service/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java diff --git a/src/systools/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java similarity index 91% rename from src/systools/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java rename to src/common-service/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java index f71faaac54..9d6f4d3108 100644 --- a/src/systools/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java +++ b/src/common-service/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java @@ -47,14 +47,15 @@ public class KylinEhCacheTest extends NLocalFileMetadataTestCase { overwriteSystemProp("kylin.cache.config", "file://" + url + "not-exists-ehcache.xml"); KylinCache cache = KylinEhCache.getInstance(); CacheManager cacheManager = (CacheManager) FieldUtils.getFieldValue(cache, "cacheManager"); - Assert.assertEquals("DefaultCacheName", cacheManager.getName()); + Assert.assertEquals("DEFAULT_CACHE", cacheManager.getName()); cacheManager.shutdown(); } @Test public void testRightCacheConfig() throws IllegalAccessException { String url = Thread.currentThread().getContextClassLoader().getResource("").getPath(); - overwriteSystemProp("kylin.cache.config", "file://" + url + "user-defined-ehcache.xml"); + String location = "file://" + url + "user-defined-ehcache.xml"; + overwriteSystemProp("kylin.cache.config", location); KylinCache cache = KylinEhCache.getInstance(); CacheManager cacheManager = (CacheManager) FieldUtils.getFieldValue(cache, "cacheManager"); Assert.assertEquals("UserDefinedCache", cacheManager.getName()); diff --git a/src/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/src/common-service/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java similarity index 100% rename from src/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java rename to src/common-service/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java diff --git a/src/common-service/src/test/resources/ehcache.xml b/src/common-service/src/test/resources/ehcache.xml index f7000111a5..c39e3687ea 100644 --- a/src/common-service/src/test/resources/ehcache.xml +++ b/src/common-service/src/test/resources/ehcache.xml @@ -15,7 +15,7 @@ limitations under the License. --> -<ehcache maxBytesLocalHeap="512M"> +<ehcache maxBytesLocalHeap="512M" name="DEFAULT_CACHE"> <cache name="StorageCache" eternal="false" timeToIdleSeconds="86400" diff --git a/src/systools/src/test/resources/user-defined-ehcache.xml b/src/common-service/src/test/resources/user-defined-ehcache.xml similarity index 100% rename from src/systools/src/test/resources/user-defined-ehcache.xml rename to src/common-service/src/test/resources/user-defined-ehcache.xml diff --git a/src/systools/src/main/java/org/apache/kylin/rest/cache/KylinCache.java b/src/core-common/src/main/java/org/apache/kylin/rest/cache/KylinCache.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/cache/KylinCache.java rename to src/core-common/src/main/java/org/apache/kylin/rest/cache/KylinCache.java diff --git a/src/core-job/pom.xml b/src/core-job/pom.xml index 298bc6298e..d41f67cfd2 100644 --- a/src/core-job/pom.xml +++ b/src/core-job/pom.xml @@ -41,40 +41,19 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> </dependency> - <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-curator-test</artifactId> - </dependency> - <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-curator</artifactId> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metrics</artifactId> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metrics</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-metadata</artifactId> + <groupId>com.github.oshi</groupId> + <artifactId>oshi-core</artifactId> </dependency> - + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>com.github.oshi</groupId> - <artifactId>oshi-core</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.zookeeper</groupId> @@ -89,6 +68,17 @@ </dependency> <!--Env & Test--> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.kyligence.ke</groupId> + <artifactId>kap-external-curator-test</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> diff --git a/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java index d8440cff37..b330a327e2 100644 --- a/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java +++ b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java @@ -24,7 +24,6 @@ import org.apache.kylin.common.KylinConfig; import lombok.val; import oshi.SystemInfo; import oshi.hardware.HardwareAbstractionLayer; -import oshi.software.os.OperatingSystem; public final class SystemInfoCollector { @@ -37,16 +36,13 @@ public final class SystemInfoCollector { private static HardwareAbstractionLayer hal = null; - private static OperatingSystem os = null; - static { init(); } - public static void init() { + private static void init() { SystemInfo si = new SystemInfo(); hal = si.getHardware(); - os = si.getOperatingSystem(); } public static Integer getAvailableMemoryInfo() { diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java index a8c8857a75..db308fc8f5 100644 --- a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java +++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java @@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Collectors; +import io.kyligence.kap.guava20.shaded.common.base.Throwables; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.ArrayUtils; @@ -88,7 +89,6 @@ import com.google.common.collect.Sets; import io.kyligence.kap.guava20.shaded.common.annotations.VisibleForTesting; import io.kyligence.kap.guava20.shaded.common.base.MoreObjects; -import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.base.Throwables; import lombok.Getter; import lombok.Setter; import lombok.val; @@ -264,7 +264,7 @@ public abstract class AbstractExecutable implements Executable { }, project, UnitOfWork.DEFAULT_MAX_RETRY, getEpochId(), getTempLockName()); } catch (Exception e) { if (Throwables.getCausalChain(e).stream().anyMatch(x -> x instanceof JobStoppedException)) { - // "in this short period user might changed job state" happens + // "in this short period user might change job state" happens logger.info("[LESS_LIKELY_THINGS_HAPPENED] JobStoppedException thrown from in a UnitOfWork", e); tryAgain = true; } else { diff --git a/src/core-metadata/pom.xml b/src/core-metadata/pom.xml index 80a87ca8c0..323788989a 100644 --- a/src/core-metadata/pom.xml +++ b/src/core-metadata/pom.xml @@ -36,18 +36,10 @@ </properties> <dependencies> - <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> </dependency> - <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-influxdb</artifactId> - </dependency> <dependency> <groupId>com.tdunning</groupId> <artifactId>t-digest</artifactId> @@ -76,10 +68,6 @@ <groupId>org.apache.calcite</groupId> <artifactId>calcite-core</artifactId> </dependency> - <dependency> - <groupId>net.sf.ehcache</groupId> - <artifactId>ehcache</artifactId> - </dependency> <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-core</artifactId> @@ -93,16 +81,27 @@ <artifactId>mybatis-dynamic-sql</artifactId> </dependency> - <!--Env & Test--> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - <scope>test</scope> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>net.sf.ehcache</groupId> + <artifactId>ehcache</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> + + <!--Env & Test--> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> + <type>test-jar</type> <scope>test</scope> </dependency> <dependency> diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java similarity index 91% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java index 6a5f4d7c56..ea864bf15f 100644 --- a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java @@ -31,14 +31,14 @@ public class ProjectStorageInfoCollector { private List<StorageInfoCollector> collectors = Lists.newArrayList(); - private static final ImmutableMap<Class, StorageInfoEnum> collectorType = ImmutableMap - .<Class, StorageInfoEnum> builder().put(GarbageStorageCollector.class, StorageInfoEnum.GARBAGE_STORAGE) + private static final ImmutableMap<Class<?>, StorageInfoEnum> collectorType = ImmutableMap + .<Class<?>, StorageInfoEnum> builder().put(GarbageStorageCollector.class, StorageInfoEnum.GARBAGE_STORAGE) .put(TotalStorageCollector.class, StorageInfoEnum.TOTAL_STORAGE) .put(StorageQuotaCollector.class, StorageInfoEnum.STORAGE_QUOTA).build(); public ProjectStorageInfoCollector(List<StorageInfoEnum> storageInfoList) { if (CollectionUtils.isNotEmpty(storageInfoList)) { - storageInfoList.forEach(si -> addCollectors(si)); + storageInfoList.forEach(this::addCollectors); } } diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java index 22b1077dfb..3605935abb 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java @@ -18,20 +18,14 @@ package org.apache.kylin.metadata.query; -import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Column; -import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Measurement; import lombok.Getter; import lombok.Setter; @SuppressWarnings("serial") -@Measurement(name = "realization_metric") @Getter @Setter public class CuboidLayoutQueryTimes { - @Column(name = "model") private String modelId; - @Column(name = "layout_id") private String layoutId; - @Column(name = "query_times") private int queryTimes; } diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java index b819aa4905..231de32242 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java @@ -18,44 +18,36 @@ package org.apache.kylin.metadata.query; -import java.time.Instant; - import com.fasterxml.jackson.annotation.JsonProperty; - -import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Column; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import java.time.Instant; + @Getter @Setter @NoArgsConstructor public class QueryStatistics { @JsonProperty("engine_type") - @Column(name = "engine_type", tag = true) private String engineType; @JsonProperty("count") - @Column(name = "count") private long count; @JsonProperty("ratio") private double ratio; @JsonProperty("mean") - @Column(name = "mean") private double meanDuration; @JsonProperty("model") - @Column(name = "model", tag = true) private String model; @JsonProperty("time") - @Column(name = "time") private Instant time; @JsonProperty("month") - @Column(name = "month", tag = true) private String month; public QueryStatistics(String engineType) { diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java index e3973b2f91..333feb66ff 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java @@ -18,15 +18,12 @@ package org.apache.kylin.metadata.query; -import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Column; import lombok.Data; @Data public class QueryTimesResponse { - @Column(name = "model") private String model; - @Column(name = "query_times") private int queryTimes; } diff --git a/src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java b/src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java similarity index 100% rename from src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java rename to src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java diff --git a/src/core-job/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java similarity index 100% rename from src/core-job/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java rename to src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java diff --git a/src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java similarity index 100% rename from src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java rename to src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java diff --git a/src/core-metrics/pom.xml b/src/core-metrics/pom.xml index fb2e75ff70..8902aae48f 100644 --- a/src/core-metrics/pom.xml +++ b/src/core-metrics/pom.xml @@ -67,11 +67,6 @@ <artifactId>commons-collections</artifactId> <scope>provided</scope> </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <scope>provided</scope> - </dependency> <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-core</artifactId> diff --git a/src/core-storage/pom.xml b/src/core-storage/pom.xml index 4de9e7315d..8846566e70 100644 --- a/src/core-storage/pom.xml +++ b/src/core-storage/pom.xml @@ -41,25 +41,15 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> </dependency> - - <!-- Provided --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> <scope>test</scope> </dependency> - </dependencies> </project> diff --git a/src/data-loading-server/pom.xml b/src/data-loading-server/pom.xml index b49dc21c87..7ce2077b10 100644 --- a/src/data-loading-server/pom.xml +++ b/src/data-loading-server/pom.xml @@ -62,6 +62,10 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> </dependency> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + </dependency> <!--Env & Test--> <dependency> diff --git a/src/datasource-service/pom.xml b/src/datasource-service/pom.xml index f0b8f9eb84..efdb2d3061 100644 --- a/src/datasource-service/pom.xml +++ b/src/datasource-service/pom.xml @@ -35,10 +35,6 @@ </properties> <dependencies> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-systools</artifactId> - </dependency> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-common-service</artifactId> @@ -60,6 +56,10 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-spark-ddl</artifactId> + </dependency> <dependency> <groupId>org.apache.tomcat.embed</groupId> <artifactId>tomcat-embed-core</artifactId> @@ -100,6 +100,7 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> + <scope>test</scope> </dependency> <dependency> <groupId>org.awaitility</groupId> diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml index 84db80f22f..ebeb329508 100644 --- a/src/kylin-it/pom.xml +++ b/src/kylin-it/pom.xml @@ -50,12 +50,7 @@ <artifactId>commons-io</artifactId> <scope>compile</scope> </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <scope>compile</scope> - </dependency> - + <!--yinglong--> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-query-server</artifactId> diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java index ceacaea174..98403fa32e 100644 --- a/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java +++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java @@ -30,7 +30,6 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest; import org.apache.kylin.job.impl.threadpool.NDefaultScheduler; import org.apache.kylin.util.ExecAndComp; -import org.apache.parquet.Strings; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparderEnv; import org.junit.After; @@ -68,8 +67,8 @@ public class CalciteDynamicTest extends NLocalWithSparkSessionTest { List<List<String>> results = transformToString(rows); for (int i = 0; i < benchmark.size(); i++) { if (!ListUtils.isEqualList(benchmark.get(i), results.get(i))) { - String expected = Strings.join(benchmark.get(i), ","); - String actual1 = Strings.join(results.get(i), ","); + String expected = String.join(",", benchmark.get(i)); + String actual1 = String.join(",", results.get(i)); fail("expected: " + expected + ", results: " + actual1); } } diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java index 43867cc5f5..4f3afa6375 100644 --- a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java +++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java @@ -42,7 +42,6 @@ import org.apache.kylin.metadata.model.SegmentRange; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.query.engine.QueryExec; import org.apache.kylin.util.ExecAndComp; -import org.apache.parquet.Strings; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparderEnv; import org.junit.Assert; @@ -140,8 +139,8 @@ public class NBuildAndQuerySnapshotTest extends NLocalWithSparkSessionTest { } private void setPartitions(NSparkSnapshotJob job, Set<String> partitions) { - job.setParam("partitions", Strings.join(partitions, ",")); - job.getSnapshotBuildingStep().setParam("partitions", Strings.join(partitions, ",")); + job.setParam("partitions", String.join(",", partitions)); + job.getSnapshotBuildingStep().setParam("partitions", String.join(",", partitions)); } } diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java index 3f0f5dab4e..26eca957d8 100644 --- a/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java +++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java @@ -46,7 +46,6 @@ import org.apache.kylin.query.engine.PrepareSqlStateParam; import org.apache.kylin.query.pushdown.SparkSqlClient; import org.apache.kylin.rest.util.PrepareSQLUtils; import org.apache.kylin.util.ExecAndComp; -import org.apache.parquet.Strings; import org.apache.spark.SparkConf; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparderEnv; @@ -128,9 +127,9 @@ public class TimeZoneQueryTest extends NLocalWithSparkSessionTest { for (int i = 0; i < jdbc.size(); i++) { if (!ListUtils.isEqualList(calciteDf.get(i), pushDown.get(i)) && !ListUtils.isEqualList(calciteDf.get(i), jdbc.get(i))) { - String expected = Strings.join(pushDown.get(i), ","); - String actual = Strings.join(jdbc.get(i), ","); - String actual2 = Strings.join(calciteDf.get(i), ","); + String expected = String.join(",", pushDown.get(i)); + String actual = String.join(",", jdbc.get(i)); + String actual2 = String.join(",", calciteDf.get(i)); fail("expected: " + expected + ", actual: " + actual + ", actual2: " + actual2); } } @@ -188,11 +187,11 @@ public class TimeZoneQueryTest extends NLocalWithSparkSessionTest { && !ListUtils.isEqualList(benchmark.get(i), setTimestampPushdownResults.get(i)) && !ListUtils.isEqualList(benchmark.get(i), setStringResults.get(i)) && !ListUtils.isEqualList(benchmark.get(i), setStringPushdownResults.get(i))) { - String expected = Strings.join(benchmark.get(i), ","); - String actual1 = Strings.join(setTimestampResults.get(i), ","); - String actual2 = Strings.join(setTimestampPushdownResults.get(i), ","); - String actual3 = Strings.join(setStringResults.get(i), ","); - String actual4 = Strings.join(setStringPushdownResults.get(i), ","); + String expected = String.join(",", benchmark.get(i)); + String actual1 = String.join(",", setTimestampResults.get(i)); + String actual2 = String.join(",", setTimestampPushdownResults.get(i)); + String actual3 = String.join(",", setStringResults.get(i)); + String actual4 = String.join(",", setStringPushdownResults.get(i)); fail("expected: " + expected + ", setTimestampResults: " + actual1 + ", setTimestampPushdownResults: " + actual2 + ", setStringResults: " + actual3 + ", setStringPushdownResults: " + actual4); } diff --git a/src/metadata-server/pom.xml b/src/metadata-server/pom.xml index 5f16c8cc7d..32b8984ea3 100644 --- a/src/metadata-server/pom.xml +++ b/src/metadata-server/pom.xml @@ -50,6 +50,10 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-common-service</artifactId> </dependency> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + </dependency> <dependency> <groupId>org.springframework</groupId> diff --git a/src/query-server/pom.xml b/src/query-server/pom.xml index d770732813..3ed9aecaab 100644 --- a/src/query-server/pom.xml +++ b/src/query-server/pom.xml @@ -74,6 +74,11 @@ <artifactId>kylin-integration-service</artifactId> </dependency> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + </dependency> + <!--Env & Test--> <dependency> <groupId>org.junit.jupiter</groupId> @@ -222,6 +227,12 @@ <artifactId>awaitility</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>com.jayway.jsonpath</groupId> + <artifactId>json-path</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-metastore</artifactId> diff --git a/src/query-service/pom.xml b/src/query-service/pom.xml index d4ab4fe0d8..26fbb531f6 100644 --- a/src/query-service/pom.xml +++ b/src/query-service/pom.xml @@ -108,11 +108,6 @@ <artifactId>hive-metastore</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <scope>test</scope> - </dependency> <!-- Spring Security --> <dependency> <groupId>org.springframework.security</groupId> @@ -173,11 +168,6 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - <scope>test</scope> - </dependency> <dependency> <groupId>org.awaitility</groupId> <artifactId>awaitility</artifactId> diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java index 08cc86c105..267acfca3e 100644 --- a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java +++ b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java @@ -20,10 +20,11 @@ package org.apache.kylin.rest.health; import org.apache.spark.sql.SparderEnv; import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.stereotype.Component; @Component -public class SparkContextHealthIndicator extends AbstractKylinHealthIndicator { +public class SparkContextHealthIndicator implements HealthIndicator { @Override public Health health() { diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java index ae245dc540..e6ca11cdf2 100644 --- a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java +++ b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java @@ -19,10 +19,11 @@ package org.apache.kylin.rest.health; import org.apache.kylin.rest.monitor.SparkContextCanary; import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.stereotype.Component; @Component -public class SparkSqlContextHealthIndicator extends AbstractKylinHealthIndicator { +public class SparkSqlContextHealthIndicator implements HealthIndicator { @Override public Health health() { diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java index edb498fce2..282824ac65 100644 --- a/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java +++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java @@ -64,7 +64,6 @@ import org.apache.kylin.query.exception.NAsyncQueryIllegalParamException; import org.apache.kylin.query.pushdown.SparkSqlClient; import org.apache.kylin.query.util.AsyncQueryUtil; import org.apache.kylin.rest.response.SQLResponse; -import org.apache.parquet.Strings; import org.apache.poi.ss.usermodel.CellType; import org.apache.poi.xssf.usermodel.XSSFCell; import org.apache.poi.xssf.usermodel.XSSFRow; @@ -984,7 +983,7 @@ public class AysncQueryServiceTest extends ServiceTestBase { try (FSDataOutputStream os = fileSystem .create(new Path(asyncQueryResultDir, AsyncQueryUtil.getMetaDataFileName())); // OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8)) { // - String metaString = Strings.join(columnNames, ",") + "\n" + Strings.join(dataTypes, ","); + String metaString = String.join(",", columnNames) + "\n" + String.join(",", dataTypes); osw.write(metaString); if (needMeta) { fileSystem.createNewFile(new Path(asyncQueryResultDir, AsyncQueryUtil.getMetaDataFileName())); diff --git a/src/second-storage/core-ui/pom.xml b/src/second-storage/core-ui/pom.xml index 7e3f843756..dea9225ee0 100644 --- a/src/second-storage/core-ui/pom.xml +++ b/src/second-storage/core-ui/pom.xml @@ -58,9 +58,14 @@ <groupId>org.apache.kylin</groupId> <artifactId>kap-second-storage-core</artifactId> </dependency> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/src/second-storage/core/pom.xml b/src/second-storage/core/pom.xml index a1b066c359..f5102b72d9 100644 --- a/src/second-storage/core/pom.xml +++ b/src/second-storage/core/pom.xml @@ -51,12 +51,6 @@ <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> - <exclusions> - <exclusion> - <groupId>org.projectlombok</groupId> - <artifactId>lombok</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.kylin</groupId> diff --git a/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java b/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java index 48e939cab6..e9e6db633e 100644 --- a/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java +++ b/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java @@ -18,7 +18,7 @@ package io.kyligence.kap.secondstorage.metadata; -import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists; +import com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Test; diff --git a/src/server/pom.xml b/src/server/pom.xml index 6c4d716f10..d26f87b24d 100644 --- a/src/server/pom.xml +++ b/src/server/pom.xml @@ -139,6 +139,14 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>io.springfox</groupId> + <artifactId>springfox-boot-starter</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-config</artifactId> + </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-jdbc</artifactId> @@ -181,13 +189,21 @@ <groupId>com.taobao.arthas</groupId> <artifactId>arthas-spring-boot-starter</artifactId> </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + </dependency> <dependency> <groupId>io.micrometer</groupId> <artifactId>micrometer-registry-prometheus</artifactId> </dependency> <dependency> - <groupId>org.postgresql</groupId> - <artifactId>postgresql</artifactId> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-common</artifactId> + </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> </dependency> <dependency> <groupId>com.lmax</groupId> diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java b/src/server/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java rename to src/server/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java rename to src/server/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java rename to src/server/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java diff --git a/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java b/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java index bba9e10dbf..ed23640a1b 100644 --- a/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java +++ b/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java @@ -32,15 +32,15 @@ import org.apache.kylin.rest.config.initialize.AfterMetadataReadyEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; import com.google.common.annotations.VisibleForTesting; @Component -public class FileSystemHealthIndicator extends AbstractKylinHealthIndicator - implements ApplicationListener<AfterMetadataReadyEvent> { - public static final Logger logger = LoggerFactory.getLogger(FileSystemHealthIndicator.class); +public class FileSystemHealthIndicator implements HealthIndicator, ApplicationListener<AfterMetadataReadyEvent> { + private static final Logger logger = LoggerFactory.getLogger(FileSystemHealthIndicator.class); private static final ScheduledExecutorService FILE_SYSTEM_HEALTH_EXECUTOR = Executors.newScheduledThreadPool(1, new NamedThreadFactory("FileSystemHealthChecker")); private volatile boolean isHealth = false; diff --git a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java index 42a5b29210..ca20cc13a3 100644 --- a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java +++ b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java @@ -26,23 +26,23 @@ import org.apache.kylin.common.KapConfig; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.StringEntity; -import org.apache.kylin.common.util.NamedThreadFactory; -import org.apache.kylin.common.util.RandomUtil; import org.apache.kylin.common.persistence.transaction.UnitOfWork; import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams; +import org.apache.kylin.common.util.NamedThreadFactory; +import org.apache.kylin.common.util.RandomUtil; import org.apache.kylin.rest.config.initialize.AfterMetadataReadyEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; import com.google.common.annotations.VisibleForTesting; @Component -public class MetaStoreHealthIndicator extends AbstractKylinHealthIndicator - implements ApplicationListener<AfterMetadataReadyEvent> { - public static final Logger logger = LoggerFactory.getLogger(MetaStoreHealthIndicator.class); +public class MetaStoreHealthIndicator implements HealthIndicator, ApplicationListener<AfterMetadataReadyEvent> { + private static final Logger logger = LoggerFactory.getLogger(MetaStoreHealthIndicator.class); private static final String UNIT_NAME = "_health"; private static final String HEALTH_ROOT_PATH = "/" + UNIT_NAME; @@ -52,10 +52,28 @@ public class MetaStoreHealthIndicator extends AbstractKylinHealthIndicator new NamedThreadFactory("MetaStoreHealthChecker")); private volatile boolean isHealth = false; + private final int warningResponseMs; + private final int errorResponseMs; + public MetaStoreHealthIndicator() { - this.config = KylinConfig.getInstanceFromEnv(); - this.warningResponseMs = KapConfig.wrap(config).getMetaStoreHealthWarningResponseMs(); - this.errorResponseMs = KapConfig.wrap(config).getMetaStoreHealthErrorResponseMs(); + KapConfig wrappedConfig = KapConfig.wrap(KylinConfig.getInstanceFromEnv()); + this.warningResponseMs = wrappedConfig.getMetaStoreHealthWarningResponseMs(); + this.errorResponseMs = wrappedConfig.getMetaStoreHealthErrorResponseMs(); + } + + private void checkTime(long start, String operation) throws InterruptedException { + // in case canary was timeout + if (Thread.interrupted()) + throw new InterruptedException(); + + long response = System.currentTimeMillis() - start; + logger.trace("{} took {} ms", operation, response); + + if (response > errorResponseMs) { + throw new IllegalStateException("check time is time out"); + } else if (response > warningResponseMs) { + logger.warn("found warning, {} took {} ms", operation, response); + } } @Override diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java b/src/server/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java similarity index 100% rename from src/systools/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java rename to src/server/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java diff --git a/src/source-hive/pom.xml b/src/source-hive/pom.xml index 69d1bb28d6..41e9f9d1c3 100644 --- a/src/source-hive/pom.xml +++ b/src/source-hive/pom.xml @@ -42,10 +42,6 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-job</artifactId> - </dependency> <!-- Env & Test --> <dependency> diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java index 308654a884..5b02c09393 100644 --- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java +++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java @@ -37,7 +37,7 @@ import java.util.stream.Stream; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.kylin.common.KapConfig; @@ -367,7 +367,7 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage if (getParent() != null) { String yarnQueue = getParent().getSparkYarnQueue(); // TODO double check if valid yarn queue - if (!Strings.isNullOrEmpty(yarnQueue)) { + if (!StringUtils.isEmpty(yarnQueue)) { jobOverrides.put("kylin.engine.spark-conf." + SPARK_YARN_QUEUE, yarnQueue); } } diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java index f360876711..d7bd1f3f15 100644 --- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java +++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java @@ -39,7 +39,6 @@ import org.apache.kylin.job.execution.NExecutableManager; import org.apache.kylin.job.impl.threadpool.NDefaultScheduler; import org.apache.kylin.metadata.model.NTableMetadataManager; import org.apache.kylin.metadata.model.TableDesc; -import org.apache.parquet.Strings; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -213,8 +212,8 @@ public class NSparkSnapshotJobTest extends NLocalWithSparkSessionTest { } private void setPartitions(NSparkSnapshotJob job, Set<String> partitions) { - job.setParam("partitions", Strings.join(partitions, ",")); - job.getSnapshotBuildingStep().setParam("partitions", Strings.join(partitions, ",")); + job.setParam("partitions", String.join(",", partitions)); + job.getSnapshotBuildingStep().setParam("partitions", String.join(",", partitions)); } @Test diff --git a/src/systools/pom.xml b/src/systools/pom.xml index ea156c7cb4..e64ff36e53 100644 --- a/src/systools/pom.xml +++ b/src/systools/pom.xml @@ -34,36 +34,6 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-metadata</artifactId> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-datasource-sdk</artifactId> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-engine-spark</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-swagger</artifactId> - </dependency> - <dependency> - <groupId>io.kyligence.ke</groupId> - <artifactId>kap-external-guava20</artifactId> - </dependency> - - <!-- spring aop --> <dependency> <groupId>org.aspectj</groupId> <artifactId>aspectjrt</artifactId> @@ -72,144 +42,50 @@ <groupId>org.aspectj</groupId> <artifactId>aspectjweaver</artifactId> </dependency> - - <!-- spring --> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-aop</artifactId> - </dependency> <dependency> <groupId>org.springframework</groupId> <artifactId>spring-context-support</artifactId> </dependency> - <dependency> - <groupId>org.springframework</groupId> - <artifactId>spring-web</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.session</groupId> - <artifactId>spring-session-core</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-autoconfigure</artifactId> - <exclusions> - <exclusion> - <artifactId>spring-context</artifactId> - <groupId>org.springframework</groupId> - </exclusion> - <exclusion> - <artifactId>spring-core</artifactId> - <groupId>org.springframework</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-actuator</artifactId> - <exclusions> - <exclusion> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-logging</artifactId> - </exclusion> - <exclusion> - <artifactId>spring-context</artifactId> - <groupId>org.springframework</groupId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.springframework</groupId> <artifactId>spring-webmvc</artifactId> </dependency> - - <!-- Spring Security --> <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-ldap</artifactId> </dependency> - <dependency> - <groupId>org.springframework.security.extensions</groupId> - <artifactId>spring-security-saml2-core</artifactId> - <exclusions> - <exclusion> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - </exclusion> - <exclusion> - <groupId>commons-httpclient</groupId> - <artifactId>commons-httpclient</artifactId> - </exclusion> - <exclusion> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - </exclusion> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-ext-jdk15on</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-test</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.springframework.security</groupId> - <artifactId>spring-security-web</artifactId> - </dependency> <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-acl</artifactId> </dependency> - <!-- hadoop --> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> - - <!-- Swagger --> - <dependency> - <groupId>io.springfox</groupId> - <artifactId>springfox-boot-starter</artifactId> - </dependency> - - <dependency> - <groupId>org.mapstruct</groupId> - <artifactId>mapstruct</artifactId> - </dependency> - - <!-- util --> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <scope>provided</scope> </dependency> - <dependency> - <groupId>redis.clients</groupId> - <artifactId>jedis</artifactId> - </dependency> <dependency> <groupId>org.apache.tomcat.embed</groupId> <artifactId>tomcat-embed-core</artifactId> <scope>provided</scope> </dependency> + <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - <scope>compile</scope> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-test</artifactId> + <scope>test</scope> </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> @@ -240,5 +116,12 @@ <artifactId>junit-vintage-engine</artifactId> <scope>test</scope> </dependency> + + <!--FIX ME--> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-engine-spark</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/src/systools/src/main/java/org/apache/kylin/rest/health/AbstractKylinHealthIndicator.java b/src/systools/src/main/java/org/apache/kylin/rest/health/AbstractKylinHealthIndicator.java deleted file mode 100644 index f2e24f38d0..0000000000 --- a/src/systools/src/main/java/org/apache/kylin/rest/health/AbstractKylinHealthIndicator.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.rest.health; - -import org.apache.kylin.common.KylinConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.actuate.health.HealthIndicator; - -public abstract class AbstractKylinHealthIndicator implements HealthIndicator { - public static final Logger logger = LoggerFactory.getLogger(AbstractKylinHealthIndicator.class); - - protected KylinConfig config; - - protected int warningResponseMs; - protected int errorResponseMs; - - protected void checkTime(long start, String operation) throws InterruptedException { - // in case canary was timeout - if (Thread.interrupted()) - throw new InterruptedException(); - - long response = System.currentTimeMillis() - start; - logger.trace("{} took {} ms", operation, response); - - if (response > errorResponseMs) { - throw new RuntimeException("check time is time out"); - } else if (response > warningResponseMs) { - logger.warn("found warning, {} took {} ms", operation, response); - } - } -} diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java b/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java index f20e658b88..6cfe74dbda 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java +++ b/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java @@ -25,6 +25,7 @@ import java.security.PrivilegedAction; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.kylin.common.KapConfig; @@ -117,7 +118,8 @@ public class KerberosLoginManager { } } - private boolean checkExistsTablesAccess(UserGroupInformation ugi, String project) { + @VisibleForTesting + boolean checkExistsTablesAccess(UserGroupInformation ugi, String project) { val kapConfig = KapConfig.getInstanceFromEnv(); val projectManager = NProjectManager.getInstance(kapConfig.getKylinConfig()); return ugi.doAs((PrivilegedAction<Boolean>) () -> { diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java b/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java index 6a70456aba..c948ab3e53 100644 --- a/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java +++ b/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java @@ -27,7 +27,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.QueryContext; import org.apache.kylin.common.exception.KylinException; diff --git a/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java b/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java index 0405f03871..e5422160c2 100644 --- a/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java +++ b/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java @@ -18,15 +18,12 @@ package org.apache.kylin.rest.security; -import java.lang.reflect.Method; - import org.apache.hadoop.security.UserGroupInformation; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.springframework.util.ReflectionUtils; public class KerberosLoginManagerTest extends NLocalFileMetadataTestCase { @Before @@ -41,11 +38,8 @@ public class KerberosLoginManagerTest extends NLocalFileMetadataTestCase { @Test public void testCheckExistsTablesAccess() throws Exception { - Method method = KerberosLoginManager.class.getDeclaredMethod("checkExistsTablesAccess", - new Class[] { UserGroupInformation.class, String.class }); - method.setAccessible(true); UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser(); - Assert.assertTrue((boolean) ReflectionUtils.invokeMethod(method, KerberosLoginManager.getInstance(), - userGroupInformation, "ssb")); + KerberosLoginManager kerberosLoginManager = KerberosLoginManager.getInstance(); + Assert.assertTrue(kerberosLoginManager.checkExistsTablesAccess(userGroupInformation, "ssb")); } } diff --git a/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java b/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java index 951dad6970..67000f1e1e 100644 --- a/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java +++ b/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java @@ -18,7 +18,7 @@ package org.apache.kylin.rest.security; -import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.kylin.common.util.NLocalFileMetadataTestCase; import org.junit.After; import org.junit.Assert; diff --git a/src/systools/src/test/resources/ehcache.xml b/src/systools/src/test/resources/ehcache.xml index 7f997b56df..e69de29bb2 100644 --- a/src/systools/src/test/resources/ehcache.xml +++ b/src/systools/src/test/resources/ehcache.xml @@ -1,19 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<ehcache name="DefaultCacheName" maxBytesLocalHeap="1M"> -</ehcache> diff --git a/src/tool/pom.xml b/src/tool/pom.xml index d5c563a0eb..24dbd4ce9c 100644 --- a/src/tool/pom.xml +++ b/src/tool/pom.xml @@ -42,38 +42,11 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-streaming</artifactId> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> - <type>test-jar</type> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-dbcp2</artifactId> - </dependency> <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-core</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-xml</artifactId> - </dependency> - - <!-- Spark dependency --> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.12</artifactId> - <version>${spark.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-framework</artifactId> @@ -97,6 +70,13 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> + + <!-- Spark dependency --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.12</artifactId> + <scope>provided</scope> + </dependency> <!-- mysql --> <dependency> <groupId>mysql</groupId> @@ -118,7 +98,16 @@ </exclusions> <scope>provided</scope> </dependency> - + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-jdbc</artifactId> @@ -131,17 +120,13 @@ </exclusions> </dependency> + <!-- Test --> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - <version>2.10.2</version> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-common</artifactId> + <type>test-jar</type> + <scope>test</scope> </dependency> - - <!-- Test --> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> @@ -160,7 +145,6 @@ <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> - <version>${mockito.version}</version> <scope>test</scope> </dependency> <dependency> @@ -181,7 +165,6 @@ <dependency> <groupId>org.awaitility</groupId> <artifactId>awaitility</artifactId> - <version>${awaitility.version}</version> <scope>test</scope> </dependency> <dependency> @@ -194,11 +177,6 @@ <artifactId>powermock-api-mockito2</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-client</artifactId> - <scope>provided</scope> - </dependency> <dependency> <groupId>com.github.stefanbirkner</groupId> <artifactId>system-lambda</artifactId> diff --git a/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java b/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java index 953b399cb4..0e553458ad 100644 --- a/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java +++ b/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java @@ -251,7 +251,7 @@ public class RecCandidateTool extends ExecutableApplication { } } - private String getProjectByModelId(String modelId) { + String getProjectByModelId(String modelId) { if (StringUtils.isEmpty(modelId)) { throw new KylinException(PARAMETER_EMPTY, "model"); } diff --git a/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java new file mode 100644 index 0000000000..e69de29bb2