KYLIN-2195 All code changes, ready for test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/826f23f1 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/826f23f1 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/826f23f1 Branch: refs/heads/master Commit: 826f23f14f60ba068b8594ba312c14e66c4807f2 Parents: 1ec1ffc Author: Li Yang <liy...@apache.org> Authored: Wed Nov 23 11:49:20 2016 +0800 Committer: Li Yang <liy...@apache.org> Committed: Wed Nov 23 17:08:29 2016 +0800 ---------------------------------------------------------------------- build/bin/check-env.sh | 2 +- build/bin/find-hive-dependency.sh | 4 +- build/bin/sample.sh | 8 +- build/conf/kylin.properties | 94 +- .../common/BackwardCompatibilityConfig.java | 21 +- .../apache/kylin/common/KylinConfigBase.java | 272 ++--- .../common/persistence/JsonSerializer.java | 94 +- .../kylin/common/persistence/Serializer.java | 70 +- .../common/persistence/StorageException.java | 76 +- .../kylin/common/restclient/RestClient.java | 222 ++-- .../org/apache/kylin/common/util/Array.java | 164 +-- .../org/apache/kylin/common/util/ByteArray.java | 492 ++++----- .../kylin/common/util/BytesSerializer.java | 70 +- .../apache/kylin/common/util/BytesSplitter.java | 250 ++--- .../kylin/common/util/CliCommandExecutor.java | 298 ++--- .../apache/kylin/common/util/MailService.java | 2 +- .../apache/kylin/common/util/ShardingHash.java | 134 +-- .../kylin-backward-compatibility.properties | 358 +++--- .../apache/kylin/common/KylinConfigTest.java | 2 +- .../kylin/common/util/MailServiceTest.java | 2 +- .../cube/common/FuzzyValueCombination.java | 260 ++--- .../org/apache/kylin/cube/cuboid/CuboidCLI.java | 286 ++--- .../org/apache/kylin/cube/kv/RowConstants.java | 94 +- .../apache/kylin/cube/kv/RowKeyColumnIO.java | 126 +-- .../apache/kylin/cube/kv/RowKeyColumnOrder.java | 216 ++-- .../org/apache/kylin/cube/model/CubeDesc.java | 2 +- .../apache/kylin/cube/model/RowKeyColDesc.java | 322 +++--- .../kylin/cube/model/v1_4_0/RowKeyColDesc.java | 184 ++-- .../cube/model/validation/IValidatorRule.java | 54 +- .../cube/model/validation/ResultLevel.java | 76 +- .../cube/model/validation/ValidateContext.java | 200 ++-- .../validation/rule/AggregationGroupRule.java | 400 +++---- .../model/validation/rule/FunctionRule.java | 404 +++---- .../model/validation/rule/RowKeyAttrRule.java | 110 +- .../validation/rule/StreamingCubeRule.java | 158 +-- .../kylin/cube/AggregationGroupRuleTest.java | 244 ++--- .../org/apache/kylin/cube/CubeDescTest.java | 6 +- .../apache/kylin/cube/CubeManagerCacheTest.java | 156 +-- .../apache/kylin/cube/RowKeyAttrRuleTest.java | 120 +-- .../metadata/measure/MeasureCodecTest.java | 182 ++-- .../kylin/dict/DictionaryInfoSerializer.java | 160 +-- .../kylin/dict/TrieDictionaryBuilder.java | 1020 +++++++++--------- .../kylin/dict/lookup/LookupStringTable.java | 220 ++-- .../apache/kylin/dict/lookup/LookupTable.java | 356 +++--- .../kylin/dict/lookup/SnapshotManager.java | 370 +++---- .../apache/kylin/dict/lookup/SnapshotTable.java | 570 +++++----- .../dict/lookup/SnapshotTableSerializer.java | 158 +-- .../kylin/dict/AppendTrieDictionaryTest.java | 2 +- .../impl/threadpool/DistributedScheduler.java | 6 +- .../kylin/metadata/MetadataConstants.java | 104 +- .../kylin/metadata/cachesync/Broadcaster.java | 2 +- .../kylin/metadata/model/JoinTableDesc.java | 124 +-- .../org/apache/kylin/metadata/tuple/ITuple.java | 86 +- .../apache/kylin/storage/StorageContext.java | 2 +- .../translate/DerivedFilterTranslator.java | 400 +++---- .../translate/FuzzyValueCombination.java | 264 ++--- .../apache/kylin/engine/mr/DFSFileTable.java | 198 ++-- .../kylin/engine/mr/DFSFileTableReader.java | 504 ++++----- .../engine/mr/steps/FactDistinctColumnsJob.java | 318 +++--- .../apache/kylin/engine/mr/TableReaderTest.java | 92 +- .../test_case_data/localmeta/kylin.properties | 52 +- .../test_case_data/localmeta/kylin_job_conf.xml | 134 +-- examples/test_case_data/sandbox/hive-site.xml | 6 +- .../test_case_data/sandbox/kylin.properties | 64 +- .../test_case_data/sandbox/kylin_job_conf.xml | 156 +-- .../sandbox/kylin_job_conf_inmem.xml | 182 ++-- .../resources/org-apache-kylin-jdbc.properties | 54 +- .../kylin/job/BaseTestDistributedScheduler.java | 4 +- .../kylin/provision/BuildCubeWithEngine.java | 2 +- .../apache/kylin/query/ITKylinQueryTest.java | 4 +- kylin-it/src/test/resources/logging.properties | 44 +- .../aggregate/DimCountDistinctAggFuncTest.java | 4 +- .../src/main/webapp/WEB-INF/kylin-servlet.xml | 26 +- server/src/main/webapp/WEB-INF/web.xml | 180 ++-- .../kylin/rest/metrics/QueryMetricsTest.java | 4 +- .../kylin/rest/service/CacheServiceTest.java | 4 +- .../apache/kylin/source/hive/HiveMRInput.java | 2 +- .../kylin/source/hive/HiveCmdBuilderTest.java | 10 +- .../source/kafka/hadoop/KafkaFlatTableJob.java | 328 +++--- .../hbase/cube/v1/RegionScannerAdapter.java | 194 ++-- .../hbase/cube/v1/ResultScannerAdapter.java | 200 ++-- .../v1/coprocessor/observer/ObserverTuple.java | 142 +-- .../kylin/storage/hbase/steps/CubeHFileJob.java | 2 +- .../kylin/storage/hbase/util/PingHBaseCLI.java | 158 +-- .../hbase/common/FuzzyValueCombinationTest.java | 260 ++--- .../common/coprocessor/RowProjectorTest.java | 176 +-- .../observer/RowAggregatorsTest.java | 124 +-- .../apache/kylin/tool/MrJobInfoExtractor.java | 2 +- .../apache/kylin/tool/StorageCleanupJob.java | 4 +- webapp/app/js/controllers/admin.js | 4 +- webapp/app/js/services/kylinProperties.js | 10 +- 91 files changed, 6869 insertions(+), 6858 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/build/bin/check-env.sh ---------------------------------------------------------------------- diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh index 9cb7636..9cd8a64 100644 --- a/build/bin/check-env.sh +++ b/build/bin/check-env.sh @@ -41,7 +41,7 @@ then quit "Please make sure the user has the privilege to run hadoop shell" fi -WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir` +WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.env.hdfs-working-dir` hadoop fs -mkdir -p $WORKING_DIR if [ $? != 0 ] then http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/build/bin/find-hive-dependency.sh ---------------------------------------------------------------------- diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh index 69b63a4..fe9e0a3 100644 --- a/build/bin/find-hive-dependency.sh +++ b/build/bin/find-hive-dependency.sh @@ -19,13 +19,13 @@ source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh -client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.client` +client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client` hive_env= echo Retrieving hive dependency... if [ "${client_mode}" == "beeline" ] then - beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.beeline.params` + beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params` hive_env=`beeline ${beeline_params} --outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' ` else hive_env=`hive -e set 2>&1 | grep 'env:CLASSPATH'` http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/build/bin/sample.sh ---------------------------------------------------------------------- diff --git a/build/bin/sample.sh b/build/bin/sample.sh index c1a7bcc..1397034 100644 --- a/build/bin/sample.sh +++ b/build/bin/sample.sh @@ -29,10 +29,10 @@ echo "Loading sample data into HDFS tmp path: /tmp/kylin/sample_cube/data" hadoop fs -mkdir -p /tmp/kylin/sample_cube/data hadoop fs -put * /tmp/kylin/sample_cube/data/ -hive_client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.client` +hive_client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.client` if [ "${hive_client_mode}" == "beeline" ] then - beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.beeline.params` + beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params` beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } else hive -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } @@ -42,8 +42,8 @@ echo "Sample hive tables are created successfully; Going to create sample cube.. hadoop fs -rm -r /tmp/kylin/sample_cube # set engine type and storage type to cube desc -default_engine_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.default.cube.engine` -default_storage_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.default.storage.engine` +default_engine_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.engine.default` +default_storage_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.storage.default` if [ -z "$default_engine_type" ]; then default_engine_type=2 default_storage_type=2 http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 3b50c12..7cdc1c5 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -22,23 +22,23 @@ kyin.server.mode=all # Optional information for the owner of kylin platform, it can be your team's email # Currently it will be attached to each kylin's htable attribute -kylin.owner=who...@kylin.apache.org +kylin.storage.hbase.owner-tag=who...@kylin.apache.org # List of web servers in use, this enables one web server instance to sync up with other servers. -kylin.rest.servers=localhost:7070 +kylin.server.cluster-servers=localhost:7070 # Display timezone on UI,format like[GMT+N or GMT-N] -kylin.rest.timezone=GMT+8 +kylin.web.timezone=GMT+8 ### SOURCE ### # Hive client, valid value [cli, beeline] -kylin.hive.client=cli +kylin.source.hive.client=cli # Parameters for beeline client, only necessary if hive client is beeline -#kylin.hive.beeline.params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000' +#kylin.source.hive.beeline-params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://localhost:10000' -kylin.hive.keep.flat.table=false +kylin.source.hive.keep-flat-table=false ### STORAGE ### @@ -49,24 +49,24 @@ kylin.metadata.url=kylin_metadata@hbase kylin.storage.url=hbase # Working folder in HDFS, make sure user has the right access to the hdfs directory -kylin.hdfs.working.dir=/kylin +kylin.env.hdfs-working-dir=/kylin # Compression codec for htable, valid value [none, snappy, lzo, gzip, lz4] -kylin.hbase.default.compression.codec=none +kylin.storage.hbase.compression-codec=none # HBase Cluster FileSystem, which serving hbase, format as hdfs://hbase-cluster:8020 # Leave empty if hbase running on same cluster with hive and mapreduce -#kylin.hbase.cluster.fs= +#kylin.storage.hbase.cluster-fs= # The cut size for hbase region, in GB. -kylin.hbase.region.cut=5 +kylin.storage.hbase.region-cut-gb=5 # The hfile size of GB, smaller hfile leading to the converting hfile MR has more reducers and be faster. # Set 0 to disable this optimization. -kylin.hbase.hfile.size.gb=2 +kylin.storage.hbase.hfile-size-gb=2 -kylin.hbase.region.count.min=1 -kylin.hbase.region.count.max=500 +kylin.storage.hbase.min-region-count=1 +kylin.storage.hbase.max-region-count=500 ### JOB ### @@ -74,47 +74,47 @@ kylin.hbase.region.count.max=500 kylin.job.retry=0 # If true, job engine will not assume that hadoop CLI reside on the same server as it self -# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password +# you will have to specify kylin.job.remote-cli-hostname, kylin.job.remote-cli-username and kylin.job.remote-cli-password # It should not be set to "true" unless you're NOT running Kylin.sh on a hadoop client machine # (Thus kylin instance has to ssh to another real hadoop client machine to execute hbase,hive,hadoop commands) -kylin.job.run.as.remote.cmd=false +kylin.job.use-remote-cli=false -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.hostname= -kylin.job.remote.cli.port=22 +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-hostname= +kylin.job.remote-cli-port=22 -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.username= +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-username= -# Only necessary when kylin.job.run.as.remote.cmd=true -kylin.job.remote.cli.password= +# Only necessary when kylin.job.use-remote-cli=true +kylin.job.remote-cli-password= # Used by test cases to prepare synthetic data for sample cube -kylin.job.remote.cli.working.dir=/tmp/kylin +kylin.job.remote-cli-working-dir=/tmp/kylin # Max count of concurrent jobs running -kylin.job.concurrent.max.limit=10 +kylin.job.max-concurrent-jobs=10 # Time interval to check hadoop job status -kylin.job.yarn.app.rest.check.interval.seconds=10 +kylin.engine.mr.yarn-check-interval-seconds=10 # Hive database name for putting the intermediate flat tables -kylin.job.hive.database.for.intermediatetable=default +kylin.source.hive.database-for-flat-table=default # Whether redistribute the intermediate flat table before building -kylin.job.hive.intermediatetable.redistribute.enabled=true +kylin.source.hive.redistribute-flat-table=true # The percentage of the sampling, default 100% -kylin.job.cubing.inmem.sampling.percent=100 +kylin.job.sampling-percentage=100 # Whether get job status from resource manager with kerberos authentication kylin.job.status.with.kerberos=false -kylin.job.mapreduce.default.reduce.input.mb=500 +kylin.engine.mr.reduce-input-mb=500 -kylin.job.mapreduce.max.reducer.number=500 +kylin.engine.mr.max-reducer-number=500 -kylin.job.mapreduce.mapper.input.rows=1000000 +kylin.engine.mr.mapper-input-rows=1000000 kylin.job.step.timeout=7200 @@ -124,35 +124,35 @@ kylin.job.step.timeout=7200 # 'auto', 'inmem', 'layer' or 'random' for testing kylin.cube.algorithm=auto -kylin.cube.algorithm.auto.threshold=8 +kylin.cube.algorithm.layer-or-inmem-threshold=8 -kylin.cube.aggrgroup.max.combination=4096 +kylin.cube.aggrgroup.max-combination=4096 kylin.dictionary.max.cardinality=5000000 -kylin.table.snapshot.max_mb=300 +kylin.snapshot.max-mb=300 ### QUERY ### -kylin.query.scan.threshold=10000000 +kylin.query.scan-threshold=10000000 kylin.query.udf.version=org.apache.kylin.query.udf.VersionUDF # 3G -kylin.query.mem.budget=3221225472 +kylin.query.memory-budget=3221225472 -kylin.query.coprocessor.mem.gb=3 +kylin.storage.hbase.coprocessor-mem-gb=3 # the default coprocessor timeout is (hbase.rpc.timeout * 0.9) / 1000 seconds, # you can set it to a smaller value. 0 means use default. -# kylin.query.coprocessor.timeout.seconds=0 +# kylin.storage.hbase.coprocessor-timeout-seconds=0 # Enable/disable ACL check for cube query -kylin.query.security.enabled=true +kylin.query.security-enabled=true -kylin.query.cache.enabled=true +kylin.query.cache-enabled=true ### SECURITY ### @@ -192,11 +192,11 @@ saml.context.contextPath=/kylin ### MAIL ### # If true, will send email notification; -mail.enabled=false -mail.host= -mail.username= -mail.password= -mail.sender= +kylin.job.notification-enabled=false +kylin.job.notification-mail-host= +kylin.job.notification-mail-username= +kylin.job.notification-mail-password= +kylin.job.notification-mail-sender= ### WEB ### # Help info, format{name|displayName|link}, optional @@ -216,9 +216,9 @@ kylin.web.diagnostic= #contact mail on web page, optional kylin.web.contact_mail= -crossdomain.enable=true +kylin.web.cross-domain-enabled=true ### OTHER ### # Env DEV|QA|PROD -deploy.env=QA +kylin.env=QA http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java index 0239af0..cbeb73a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java @@ -118,7 +118,7 @@ public class BackwardCompatibilityConfig { public static void main(String[] args) throws IOException { String kylinRepoDir = args.length > 0 ? args[0] : "."; - String outputDir = args.length > 1 ? args[1] : "."; + String outputDir = args.length > 1 ? args[1] : kylinRepoDir; generateFindAndReplaceScript(kylinRepoDir, outputDir); } @@ -154,9 +154,10 @@ public class BackwardCompatibilityConfig { for (File f : dir.listFiles()) { if (f.getName().startsWith(".")) continue; - if (f.isDirectory()) - stack.push(f); - else if (isSourceFile(f)) + if (f.isDirectory()) { + if (acceptSourceDir(f)) + stack.push(f); + } else if (acceptSourceFile(f)) out.println("sed -i -f upgrade-old-config.sed " + f.getAbsolutePath()); } } @@ -173,7 +174,17 @@ public class BackwardCompatibilityConfig { return key.replace(".", "[.]"); } - private static boolean isSourceFile(File f) { + private static boolean acceptSourceDir(File f) { + // exclude webapp/app/components + if (f.getName().equals("components") && f.getParentFile().getName().equals("app")) + return false; + else if (f.getName().equals("target")) + return false; + else + return true; + } + + private static boolean acceptSourceFile(File f) { String name = f.getName(); if (name.startsWith(KYLIN_BACKWARD_COMPATIBILITY)) return false; http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index a84ed62..db4f365 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -141,7 +141,7 @@ abstract public class KylinConfigBase implements Serializable { // ============================================================================ public boolean isDevEnv() { - return "DEV".equals(getOptional("deploy.env", "DEV")); + return "DEV".equals(getOptional("kylin.env", "DEV")); } public String getMetadataUrl() { @@ -182,7 +182,7 @@ abstract public class KylinConfigBase implements Serializable { */ @Deprecated public String getHiveUrl() { - return getOptional("hive.url", ""); + return getOptional("kylin.source.hive.url", ""); } /** @@ -190,7 +190,7 @@ abstract public class KylinConfigBase implements Serializable { */ @Deprecated public String getHiveUser() { - return getOptional("hive.user", ""); + return getOptional("kylin.source.hive.user", ""); } /** @@ -198,11 +198,11 @@ abstract public class KylinConfigBase implements Serializable { */ @Deprecated public String getHivePassword() { - return getOptional("hive.password", ""); + return getOptional("kylin.source.hive.password", ""); } public String getHdfsWorkingDirectory() { - String root = getRequired("kylin.hdfs.working.dir"); + String root = getRequired("kylin.env.hdfs-working-dir"); if (!root.endsWith("/")) { root += "/"; } @@ -210,7 +210,7 @@ abstract public class KylinConfigBase implements Serializable { } public String[] getRealizationProviders() { - return getOptionalStringArray("kylin.realization.providers", // + return getOptionalStringArray("kylin.metadata.realization-providers", // new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); } @@ -223,19 +223,19 @@ abstract public class KylinConfigBase implements Serializable { } public String getHBaseClusterFs() { - return getOptional("kylin.hbase.cluster.fs", ""); + return getOptional("kylin.storage.hbase.cluster-fs", ""); } public String getHBaseClusterHDFSConfigFile() { - return getOptional("kylin.hbase.cluster.hdfs.config.file", ""); + return getOptional("kylin.storage.hbase.cluster-hdfs-config-file", ""); } public String getKylinJobLogDir() { - return getOptional("kylin.job.log.dir", "/tmp/kylin/logs"); + return getOptional("kylin.job.log-dir", "/tmp/kylin/logs"); } public String getKylinJobJarPath() { - final String jobJar = getOptional("kylin.job.jar"); + final String jobJar = getOptional("kylin.engine.mr.job-jar"); if (StringUtils.isNotEmpty(jobJar)) { return jobJar; } @@ -247,24 +247,24 @@ abstract public class KylinConfigBase implements Serializable { } public void overrideMRJobJarPath(String path) { - logger.info("override " + "kylin.job.jar" + " to " + path); - System.setProperty("kylin.job.jar", path); + logger.info("override " + "kylin.engine.mr.job-jar" + " to " + path); + System.setProperty("kylin.engine.mr.job-jar", path); } public String getKylinJobMRLibDir() { - return getOptional("kylin.job.mr.lib.dir", ""); + return getOptional("kylin.engine.mr.lib-dir", ""); } public Map<String, String> getMRConfigOverride() { - return getPropertiesByPrefix("kylin.job.mr.config.override."); + return getPropertiesByPrefix("kylin.engine.mr.config-override."); } public Map<String, String> getHiveConfigOverride() { - return getPropertiesByPrefix("kylin.hive.config.override."); + return getPropertiesByPrefix("kylin.source.hive.config-override."); } public String getKylinSparkJobJarPath() { - final String jobJar = getOptional("kylin.job.jar.spark"); + final String jobJar = getOptional("kylin.engine.mr.job-jar.spark"); if (StringUtils.isNotEmpty(jobJar)) { return jobJar; } @@ -276,8 +276,8 @@ abstract public class KylinConfigBase implements Serializable { } public void overrideSparkJobJarPath(String path) { - logger.info("override " + "kylin.job.jar.spark" + " to " + path); - System.setProperty("kylin.job.jar.spark", path); + logger.info("override " + "kylin.engine.mr.job-jar.spark" + " to " + path); + System.setProperty("kylin.engine.mr.job-jar.spark", path); } private static final Pattern COPROCESSOR_JAR_NAME_PATTERN = Pattern.compile("kylin-coprocessor-(.+)\\.jar"); @@ -285,7 +285,7 @@ abstract public class KylinConfigBase implements Serializable { private static final Pattern SPARK_JOB_JAR_NAME_PATTERN = Pattern.compile("kylin-engine-spark-(.+)\\.jar"); public String getCoprocessorLocalJar() { - final String coprocessorJar = getOptional("kylin.coprocessor.local.jar"); + final String coprocessorJar = getOptional("kylin.storage.hbase.coprocessor-local-jar"); if (StringUtils.isNotEmpty(coprocessorJar)) { return coprocessorJar; } @@ -297,8 +297,8 @@ abstract public class KylinConfigBase implements Serializable { } public void overrideCoprocessorLocalJar(String path) { - logger.info("override " + "kylin.coprocessor.local.jar" + " to " + path); - System.setProperty("kylin.coprocessor.local.jar", path); + logger.info("override " + "kylin.storage.hbase.coprocessor-local-jar" + " to " + path); + System.setProperty("kylin.storage.hbase.coprocessor-local-jar", path); } private static String getFileName(String homePath, Pattern pattern) { @@ -320,117 +320,117 @@ abstract public class KylinConfigBase implements Serializable { } public double getDefaultHadoopJobReducerInputMB() { - return Double.parseDouble(getOptional("kylin.job.mapreduce.default.reduce.input.mb", "500")); + return Double.parseDouble(getOptional("kylin.engine.mr.reduce-input-mb", "500")); } public double getDefaultHadoopJobReducerCountRatio() { - return Double.parseDouble(getOptional("kylin.job.mapreduce.default.reduce.count.ratio", "1.0")); + return Double.parseDouble(getOptional("kylin.engine.mr.reduce-count-ratio", "1.0")); } public int getHadoopJobMinReducerNumber() { - return Integer.parseInt(getOptional("kylin.job.mapreduce.min.reducer.number", "1")); + return Integer.parseInt(getOptional("kylin.engine.mr.min-reducer-number", "1")); } public int getHadoopJobMaxReducerNumber() { - return Integer.parseInt(getOptional("kylin.job.mapreduce.max.reducer.number", "500")); + return Integer.parseInt(getOptional("kylin.engine.mr.max-reducer-number", "500")); } public int getHadoopJobMapperInputRows() { - return Integer.parseInt(getOptional("kylin.job.mapreduce.mapper.input.rows", "1000000")); + return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000")); } public boolean getRunAsRemoteCommand() { - return Boolean.parseBoolean(getOptional("kylin.job.run.as.remote.cmd")); + return Boolean.parseBoolean(getOptional("kylin.job.use-remote-cli")); } public void setRunAsRemoteCommand(String v) { - setProperty("kylin.job.run.as.remote.cmd", v); + setProperty("kylin.job.use-remote-cli", v); } public int getRemoteHadoopCliPort() { - return Integer.parseInt(getOptional("kylin.job.remote.cli.port", "22")); + return Integer.parseInt(getOptional("kylin.job.remote-cli-port", "22")); } public String getRemoteHadoopCliHostname() { - return getOptional("kylin.job.remote.cli.hostname"); + return getOptional("kylin.job.remote-cli-hostname"); } public void setRemoteHadoopCliHostname(String v) { - setProperty("kylin.job.remote.cli.hostname", v); + setProperty("kylin.job.remote-cli-hostname", v); } public String getRemoteHadoopCliUsername() { - return getOptional("kylin.job.remote.cli.username"); + return getOptional("kylin.job.remote-cli-username"); } public void setRemoteHadoopCliUsername(String v) { - setProperty("kylin.job.remote.cli.username", v); + setProperty("kylin.job.remote-cli-username", v); } public String getRemoteHadoopCliPassword() { - return getOptional("kylin.job.remote.cli.password"); + return getOptional("kylin.job.remote-cli-password"); } public void setRemoteHadoopCliPassword(String v) { - setProperty("kylin.job.remote.cli.password", v); + setProperty("kylin.job.remote-cli-password", v); } public String getCliWorkingDir() { - return getOptional("kylin.job.remote.cli.working.dir"); + return getOptional("kylin.job.remote-cli-working-dir"); } public boolean isEmptySegmentAllowed() { - return Boolean.parseBoolean(getOptional("kylin.job.allow.empty.segment", "true")); + return Boolean.parseBoolean(getOptional("kylin.job.allow-empty-segment", "true")); } //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns public int getUHCReducerCount() { - return Integer.parseInt(getOptional("kylin.job.uhc.reducer.count", "3")); + return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3")); } public String getOverrideHiveTableLocation(String table) { - return getOptional("hive.table.location." + table.toUpperCase()); + return getOptional("kylin.source.hive.table-location." + table.toUpperCase()); } public String getYarnStatusCheckUrl() { - return getOptional("kylin.job.yarn.app.rest.check.status.url", null); + return getOptional("kylin.engine.mr.yarn-check-status-url", null); } public int getYarnStatusCheckIntervalSeconds() { - return Integer.parseInt(getOptional("kylin.job.yarn.app.rest.check.interval.seconds", "60")); + return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "60")); } public int getMaxConcurrentJobLimit() { - return Integer.parseInt(getOptional("kylin.job.concurrent.max.limit", "10")); + return Integer.parseInt(getOptional("kylin.job.max-concurrent-jobs", "10")); } public String getTimeZone() { - return getOptional("kylin.rest.timezone", "PST"); + return getOptional("kylin.web.timezone", "PST"); } public String[] getRestServers() { - return getOptionalStringArray("kylin.rest.servers", new String[0]); + return getOptionalStringArray("kylin.server.cluster-servers", new String[0]); } public String getClusterName() { - return this.getOptional("kylin.cluster.name", getMetadataUrlPrefix()); + return this.getOptional("kylin.server.cluster-name", getMetadataUrlPrefix()); } public int getWorkersPerServer() { //for sequence sql use - return Integer.parseInt(getOptional("kylin.rest.workers.per.server", "1")); + return Integer.parseInt(getOptional("kylin.server.sequence-sql.workers-per-server", "1")); } public String[] getAdminDls() { - return getOptionalStringArray("kylin.job.admin.dls", null); + return getOptionalStringArray("kylin.job.notification-admin-emails", null); } public double getJobCuboidSizeRatio() { - return Double.parseDouble(getOptional("kylin.job.cuboid.size.ratio", "0.25")); + return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25")); } public double getJobCuboidSizeMemHungryRatio() { - return Double.parseDouble(getOptional("kylin.job.cuboid.size.memhungry.ratio", "0.05")); + return Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio", "0.05")); } public String getCubeAlgorithm() { @@ -438,57 +438,57 @@ abstract public class KylinConfigBase implements Serializable { } public double getCubeAlgorithmAutoThreshold() { - return Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "7")); + return Double.parseDouble(getOptional("kylin.cube.algorithm.layer-or-inmem-threshold", "7")); } public int getCubeAlgorithmAutoMapperLimit() { - return Integer.parseInt(getOptional("kylin.cube.algorithm.auto.mapper.limit", "500")); + return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500")); } @Deprecated public int getCubeAggrGroupMaxSize() { - return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12")); + return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12")); } public int getCubeAggrGroupMaxCombination() { - return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.combination", "4096")); + return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-combination", "4096")); } public boolean getCubeAggrGroupIsMandatoryOnlyValid() { - return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.isMandatoryOnlyValid", "false")); + return Boolean.parseBoolean(getOptional("kylin.cube.aggrgroup.is-mandatory-only-valid", "false")); } public String[] getCubeDimensionCustomEncodingFactories() { - return getOptionalStringArray("kylin.cube.dimension.customEncodingFactories", new String[0]); + return getOptionalStringArray("kylin.metadata.custom-dimension-encodings", new String[0]); } public Map<String, String> getCubeCustomMeasureTypes() { - return getPropertiesByPrefix("kylin.cube.measure.customMeasureType."); + return getPropertiesByPrefix("kylin.metadata.custom-measure-types."); } public int getTableSnapshotMaxMB() { - return Integer.parseInt(getOptional("kylin.table.snapshot.max_mb", "300")); + return Integer.parseInt(getOptional("kylin.snapshot.max-mb", "300")); } public int getTrieDictionaryForestMaxTrieSizeMB() { - return Integer.parseInt(getOptional("kylin.dictionary.forest.trie.size.max_mb", "500")); + return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500")); } public int getHBaseRegionCountMin() { - return Integer.parseInt(getOptional("kylin.hbase.region.count.min", "1")); + return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1")); } public int getHBaseRegionCountMax() { - return Integer.parseInt(getOptional("kylin.hbase.region.count.max", "500")); + return Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500")); } // for test only public void setHBaseHFileSizeGB(float size) { - setProperty("kylin.hbase.hfile.size.gb", String.valueOf(size)); + setProperty("kylin.storage.hbase.hfile-size-gb", String.valueOf(size)); } public float getHBaseHFileSizeGB() { - return Float.parseFloat(getOptional("kylin.hbase.hfile.size.gb", "2.0")); + return Float.parseFloat(getOptional("kylin.storage.hbase.hfile-size-gb", "2.0")); } //check KYLIN-1684, in most cases keep the default value @@ -497,35 +497,35 @@ abstract public class KylinConfigBase implements Serializable { } public int getStoragePushDownLimitMax() { - return Integer.parseInt(getOptional("kylin.query.pushdown.limit.max", "10000")); + return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000")); } public int getScanThreshold() { - return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000")); + return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000")); } public int getDerivedInThreshold() { - return Integer.parseInt(getOptional("kylin.query.filter.derived_in.max", "20")); + return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20")); } public int getBadQueryStackTraceDepth() { - return Integer.parseInt(getOptional("kylin.query.badquery.stacktrace.depth", "10")); + return Integer.parseInt(getOptional("kylin.query.badquery-stacktrace-depth", "10")); } public int getBadQueryHistoryNum() { - return Integer.parseInt(getOptional("kylin.query.badquery.history.num", "10")); + return Integer.parseInt(getOptional("kylin.query.badquery-history-number", "10")); } public int getBadQueryDefaultAlertingSeconds() { - return Integer.parseInt(getOptional("kylin.query.badquery.alerting.seconds", "90")); + return Integer.parseInt(getOptional("kylin.query.badquery-alerting-seconds", "90")); } public int getBadQueryDefaultDetectIntervalSeconds() { - return Integer.parseInt(getOptional("kylin.query.badquery.detect.interval.seconds", "60")); + return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60")); } public boolean getBadQueryPersistentEnabled() { - return Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", "true")); + return Boolean.parseBoolean(getOptional("kylin.query.badquery-persistent-enabled", "true")); } public String[] getQueryTransformers() { @@ -533,84 +533,84 @@ abstract public class KylinConfigBase implements Serializable { } public int getCachedDictMaxEntrySize() { - return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", "3000")); + return Integer.parseInt(getOptional("kylin.dictionary.max-cache-entry", "3000")); } public int getCachedSnapshotMaxEntrySize() { - return Integer.parseInt(getOptional("kylin.snapshot.cache.max.entry", "500")); + return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500")); } public boolean getQueryRunLocalCoprocessor() { - return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false")); + return Boolean.parseBoolean(getOptional("kylin.storage.hbase.run-local-coprocessor", "false")); } public long getQueryDurationCacheThreshold() { - return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000))); + return Long.parseLong(this.getOptional("kylin.query.cache-threshold-duration", String.valueOf(2000))); } public long getQueryScanCountCacheThreshold() { - return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024))); + return Long.parseLong(this.getOptional("kylin.query.cache-threshold-scan-count", String.valueOf(10 * 1024))); } public long getQueryMemBudget() { - return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024))); + return Long.parseLong(this.getOptional("kylin.query.memory-budget", String.valueOf(3L * 1024 * 1024 * 1024))); } public double getQueryCoprocessorMemGB() { - return Double.parseDouble(this.getOptional("kylin.query.coprocessor.mem.gb", "3.0")); + return Double.parseDouble(this.getOptional("kylin.storage.hbase.coprocessor-mem-gb", "3.0")); } public int getQueryCoprocessorTimeoutSeconds() { - return Integer.parseInt(this.getOptional("kylin.query.coprocessor.timeout.seconds", "0")); + return Integer.parseInt(this.getOptional("kylin.storage.hbase.coprocessor-timeout-seconds", "0")); } public boolean isQuerySecureEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "true")); + return Boolean.parseBoolean(this.getOptional("kylin.query.security-enabled", "true")); } public boolean isQueryCacheEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true")); + return Boolean.parseBoolean(this.getOptional("kylin.query.cache-enabled", "true")); } public boolean isQueryIgnoreUnknownFunction() { - return Boolean.parseBoolean(this.getOptional("kylin.query.ignore_unknown_function", "false")); + return Boolean.parseBoolean(this.getOptional("kylin.query.ignore-unknown-function", "false")); } public int getQueryScanFuzzyKeyMax() { - return Integer.parseInt(this.getOptional("kylin.query.scan.fuzzykey.max", "200")); + return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-fuzzykey-scan", "200")); } public int getQueryStorageVisitScanRangeMax() { - return Integer.valueOf(this.getOptional("kylin.query.storage.visit.scanrange.max", "1000000")); + return Integer.valueOf(this.getOptional("kylin.storage.hbase.max-visit-scanrange", "1000000")); } public String getQueryAccessController() { - return getOptional("kylin.query.access.controller", null); + return getOptional("kylin.query.access-controller", null); } public long getSequenceExpireTime() { - return Long.valueOf(this.getOptional("kylin.query.sequence.expire.time", "86400000"));//default a day + return Long.valueOf(this.getOptional("kylin.server.sequence-sql.expire-time", "86400000"));//default a day } public boolean getQueryMetricsEnabled() { - return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", "false")); + return Boolean.parseBoolean(getOptional("kylin.server.query-metrics-enabled", "false")); } public int[] getQueryMetricsPercentilesIntervals() { String[] dft = { "60", "300", "3600" }; - return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft); + return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft); } public String getDefaultIGTStorage() { - return getOptional("kylin.query.storage.default.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC"); + return getOptional("kylin.storage.hbase.gtstorage", "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC"); } public int getHBaseScanCacheRows() { - return Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024")); + return Integer.parseInt(this.getOptional("kylin.storage.hbase.scan-cache-rows", "1024")); } public boolean isGrowingDictEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false")); + return Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled", "false")); } /** @@ -619,90 +619,90 @@ abstract public class KylinConfigBase implements Serializable { * @return */ public float getKylinHBaseRegionCut() { - return Float.valueOf(getOptional("kylin.hbase.region.cut", "5.0")); + return Float.valueOf(getOptional("kylin.storage.hbase.region-cut-gb", "5.0")); } public int getHBaseScanMaxResultSize() { - return Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 1024 * 1024))); // 5 MB + return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-scan-result-bytes", "" + (5 * 1024 * 1024))); // 5 MB } public int getCubingInMemSamplingPercent() { - int percent = Integer.parseInt(this.getOptional("kylin.job.cubing.inmem.sampling.percent", "100")); + int percent = Integer.parseInt(this.getOptional("kylin.job.sampling-percentage", "100")); percent = Math.max(percent, 1); percent = Math.min(percent, 100); return percent; } public String getHbaseDefaultCompressionCodec() { - return getOptional("kylin.hbase.default.compression.codec", "none"); + return getOptional("kylin.storage.hbase.compression-codec", "none"); } public String getHbaseDefaultEncoding() { - return getOptional("kylin.hbase.default.encoding", "FAST_DIFF"); + return getOptional("kylin.storage.hbase.rowkey-encoding", "FAST_DIFF"); } public int getHbaseDefaultBlockSize() { - return Integer.valueOf(getOptional("kylin.hbase.default.block.size", "1048576")); + return Integer.valueOf(getOptional("kylin.storage.hbase.block-size-bytes", "1048576")); } public int getHbaseSmallFamilyBlockSize() { - return Integer.valueOf(getOptional("kylin.hbase.small.family.block.size", "65536")); + return Integer.valueOf(getOptional("kylin.storage.hbase.small-family-block-size-bytes", "65536")); } public boolean isHiveKeepFlatTable() { - return Boolean.parseBoolean(this.getOptional("kylin.hive.keep.flat.table", "false")); + return Boolean.parseBoolean(this.getOptional("kylin.source.hive.keep-flat-table", "false")); } public String getHiveDatabaseForIntermediateTable() { - return this.getOptional("kylin.job.hive.database.for.intermediatetable", "default"); + return this.getOptional("kylin.source.hive.database-for-flat-table", "default"); } public boolean isHiveRedistributeEnabled() { - return Boolean.parseBoolean(this.getOptional("kylin.job.hive.intermediatetable.redistribute.enabled", "true")); + return Boolean.parseBoolean(this.getOptional("kylin.source.hive.redistribute-flat-table", "true")); } public String getHiveDependencyFilterList() { - return this.getOptional("kylin.job.dependency.filterlist", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar"); + return this.getOptional("kylin.job.dependency-filter-list", "[^,]*hive-exec[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-metastore[0-9.-]+[^,]*?\\.jar" + "|" + "[^,]*hive-hcatalog-core[0-9.-]+[^,]*?\\.jar"); } public String getKylinOwner() { - return this.getOptional("kylin.owner", ""); + return this.getOptional("kylin.storage.hbase.owner-tag", ""); } public String getSparkHome() { - return getRequired("kylin.spark.home"); + return getRequired("kylin.engine.spark.spark-home"); } public String getSparkMaster() { - return getRequired("kylin.spark.master"); + return getRequired("kylin.engine.spark.spark-master"); } public boolean isMailEnabled() { - return Boolean.parseBoolean(getOptional("mail.enabled", "false")); + return Boolean.parseBoolean(getOptional("kylin.job.notification-enabled", "false")); } public void setMailEnabled(boolean enable) { - setProperty("mail.enabled", "" + enable); + setProperty("kylin.job.notification-enabled", "" + enable); } public String getMailHost() { - return getOptional("mail.host", ""); + return getOptional("kylin.job.notification-mail-host", ""); } public String getMailUsername() { - return getOptional("mail.username", ""); + return getOptional("kylin.job.notification-mail-username", ""); } public String getMailPassword() { - return getOptional("mail.password", ""); + return getOptional("kylin.job.notification-mail-password", ""); } public String getMailSender() { - return getOptional("mail.sender", ""); + return getOptional("kylin.job.notification-mail-sender", ""); } public boolean isWebCrossDomainEnabled() { - return Boolean.parseBoolean(getOptional("crossdomain.enable", "true")); + return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } public int getJobRetry() { @@ -714,35 +714,35 @@ abstract public class KylinConfigBase implements Serializable { } public String getHiveClientMode() { - return getOptional("kylin.hive.client", "cli"); + return getOptional("kylin.source.hive.client", "cli"); } public String getHiveBeelineParams() { - return getOptional("kylin.hive.beeline.params", ""); + return getOptional("kylin.source.hive.beeline-params", ""); } public String getDeployEnv() { - return getOptional("deploy.env", "DEV"); + return getOptional("kylin.env", "DEV"); } public String getInitTasks() { - return getOptional("kylin.init.tasks"); + return getOptional("kylin.server.init-tasks"); } public int getDimCountDistinctMaxCardinality() { - return Integer.parseInt(getOptional("kylin.query.dim.distinct.max", "5000000")); + return Integer.parseInt(getOptional("kylin.query.max-dimension-count-distinct", "5000000")); } public int getCubeStatsHLLPrecision() { - return Integer.parseInt(getOptional("kylin.job.cubing.inmem.sampling.hll.precision", "14")); + return Integer.parseInt(getOptional("kylin.job.sampling-hll-precision", "14")); } public String getJobControllerLock() { - return getOptional("kylin.job.controller.lock", "org.apache.kylin.storage.hbase.util.ZookeeperJobLock"); + return getOptional("kylin.job.lock", "org.apache.kylin.storage.hbase.util.ZookeeperJobLock"); } public Map<Integer, String> getJobEngines() { - Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.cube.engine.")); + Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider.")); // ref constants in IEngineAware r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); @@ -750,7 +750,7 @@ abstract public class KylinConfigBase implements Serializable { } public Map<Integer, String> getSourceEngines() { - Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.source.engine.")); + Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider.")); // ref constants in ISourceAware r.put(0, "org.apache.kylin.source.hive.HiveSource"); r.put(1, "org.apache.kylin.source.kafka.KafkaSource"); @@ -758,7 +758,7 @@ abstract public class KylinConfigBase implements Serializable { } public Map<Integer, String> getStorageEngines() { - Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.storage.engine.")); + Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider.")); // ref constants in IStorageAware r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage"); r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage"); @@ -767,22 +767,22 @@ abstract public class KylinConfigBase implements Serializable { } public int getDefaultStorageEngine() { - return Integer.parseInt(getOptional("kylin.default.storage.engine", "2")); + return Integer.parseInt(getOptional("kylin.storage.default", "2")); } public int getDefaultCubeEngine() { - return Integer.parseInt(getOptional("kylin.default.cube.engine", "2")); + return Integer.parseInt(getOptional("kylin.engine.default", "2")); } public Map<Integer, String> getSchedulers() { - Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.scheduler.")); + Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.job.scheduler.provider.")); r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler"); r.put(2, "org.apache.kylin.job.impl.threadpool.DistributedScheduler"); return r; } public Integer getSchedulerType() { - return Integer.parseInt(getOptional("kylin.enable.scheduler", "0")); + return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0")); } private Map<Integer, String> convertKeyToInteger(Map<String, String> map) { @@ -794,7 +794,7 @@ abstract public class KylinConfigBase implements Serializable { } public boolean getCompressionResult() { - return Boolean.parseBoolean(getOptional("kylin.query.endpoint.compression.result", "true")); + return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", "true")); } public Map<String, String> getUDFs() { @@ -803,45 +803,45 @@ abstract public class KylinConfigBase implements Serializable { } public int getHBaseMaxConnectionThreads() { - return Integer.parseInt(getOptional("kylin.query.hbase.hconnection.threads.max", "2048")); + return Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads", "2048")); } public int getHBaseCoreConnectionThreads() { - return Integer.parseInt(getOptional("kylin.query.hbase.hconnection.threads.core", "2048")); + return Integer.parseInt(getOptional("kylin.storage.hbase.core-hconnection-threads", "2048")); } public long getHBaseConnectionThreadPoolAliveSeconds() { - return Long.parseLong(getOptional("kylin.query.hbase.hconnection.threads.alive.seconds", "60")); + return Long.parseLong(getOptional("kylin.storage.hbase.hconnection-threads-alive-seconds", "60")); } public int getAppendDictEntrySize() { - return Integer.parseInt(getOptional("kylin.dict.append.entry.size", "10000000")); + return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000")); } // for test public void setAppendDictEntrySize(int entrySize) { - setProperty("kylin.dict.append.entry.size", String.valueOf(entrySize)); + setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize)); } public int getAppendDictCacheSize() { - return Integer.parseInt(getOptional("kylin.dict.append.cache.size", "20")); + return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20")); } // for test public void setAppendDictCacheSize(int cacheSize) { - setProperty("kylin.dict.append.cache.size", String.valueOf(cacheSize)); + setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize)); } @Deprecated public String getCreateFlatHiveTableMethod() { - return getOptional("kylin.hive.create.flat.table.method", "1"); + return getOptional("kylin.source.hive.create-flat-table-method", "1"); } public int getMaxBuildingSegments() { - return Integer.parseInt(getOptional("kylin.cube.building.segment.max", "10")); + return Integer.parseInt(getOptional("kylin.cube.max-building-segments", "10")); } public void setMaxBuildingSegments(int maxBuildingSegments) { - setProperty("kylin.cube.building.segment.max", String.valueOf(maxBuildingSegments)); + setProperty("kylin.cube.max-building-segments", String.valueOf(maxBuildingSegments)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java b/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java index 2e7768a..8f0cc51 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/JsonSerializer.java @@ -1,47 +1,47 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.persistence; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -import org.apache.kylin.common.util.JsonUtil; - -/** - * @author yangli9 - */ -public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> { - - Class<T> clz; - - public JsonSerializer(Class<T> clz) { - this.clz = clz; - } - - @Override - public T deserialize(DataInputStream in) throws IOException { - return JsonUtil.readValue(in, clz); - } - - @Override - public void serialize(T obj, DataOutputStream out) throws IOException { - JsonUtil.writeValueIndent(out, obj); - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.persistence; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.kylin.common.util.JsonUtil; + +/** + * @author yangli9 + */ +public class JsonSerializer<T extends RootPersistentEntity> implements Serializer<T> { + + Class<T> clz; + + public JsonSerializer(Class<T> clz) { + this.clz = clz; + } + + @Override + public T deserialize(DataInputStream in) throws IOException { + return JsonUtil.readValue(in, clz); + } + + @Override + public void serialize(T obj, DataOutputStream out) throws IOException { + JsonUtil.writeValueIndent(out, obj); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/persistence/Serializer.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/Serializer.java b/core-common/src/main/java/org/apache/kylin/common/persistence/Serializer.java index af32ec0..6b8b230 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/Serializer.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/Serializer.java @@ -1,35 +1,35 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.persistence; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -/** - * @author yangli9 - * - */ -public interface Serializer<T extends RootPersistentEntity> { - - public void serialize(T obj, DataOutputStream out) throws IOException; - - public T deserialize(DataInputStream in) throws IOException; - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.persistence; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** + * @author yangli9 + * + */ +public interface Serializer<T extends RootPersistentEntity> { + + public void serialize(T obj, DataOutputStream out) throws IOException; + + public T deserialize(DataInputStream in) throws IOException; + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/persistence/StorageException.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/StorageException.java b/core-common/src/main/java/org/apache/kylin/common/persistence/StorageException.java index 0517866..604941a 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/StorageException.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/StorageException.java @@ -1,38 +1,38 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.persistence; - -/** - * - * @author xjiang - * - */ -public class StorageException extends RuntimeException { - - private static final long serialVersionUID = -3748712888242406257L; - - public StorageException(String msg, Throwable t) { - super(msg, t); - } - - public StorageException(String msg) { - super(msg); - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.persistence; + +/** + * + * @author xjiang + * + */ +public class StorageException extends RuntimeException { + + private static final long serialVersionUID = -3748712888242406257L; + + public StorageException(String msg, Throwable t) { + super(msg, t); + } + + public StorageException(String msg) { + super(msg); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java index 363cc54..269e935 100644 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java @@ -1,125 +1,125 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.restclient; - -import java.io.IOException; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.util.EntityUtils; -import org.apache.kylin.common.util.JsonUtil; - -/** - * @author yangli9 - */ -public class RestClient { - - protected String host; - protected int port; - protected String baseUrl; - protected String userName; - protected String password; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.restclient; + +import java.io.IOException; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; +import org.apache.kylin.common.util.JsonUtil; + +/** + * @author yangli9 + */ +public class RestClient { + + protected String host; + protected int port; + protected String baseUrl; + protected String userName; + protected String password; protected DefaultHttpClient client; - - protected static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?"); - - public static boolean matchFullRestPattern(String uri) { - Matcher m = fullRestPattern.matcher(uri); - return m.matches(); - } - - /** - * @param uri - * "user:pwd@host:port" - */ - public RestClient(String uri) { - Matcher m = fullRestPattern.matcher(uri); - if (!m.matches()) - throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern); - - String user = m.group(1); - String pwd = m.group(2); - String host = m.group(3); - String portStr = m.group(4); - int port = Integer.parseInt(portStr == null ? "7070" : portStr); - - init(host, port, user, pwd); - } - - private void init(String host, int port, String userName, String password) { - this.host = host; - this.port = port; - this.userName = userName; - this.password = password; - this.baseUrl = "http://" + host + ":" + port + "/kylin/api"; - + + protected static Pattern fullRestPattern = Pattern.compile("(?:([^:]+)[:]([^@]+)[@])?([^:]+)(?:[:](\\d+))?"); + + public static boolean matchFullRestPattern(String uri) { + Matcher m = fullRestPattern.matcher(uri); + return m.matches(); + } + + /** + * @param uri + * "user:pwd@host:port" + */ + public RestClient(String uri) { + Matcher m = fullRestPattern.matcher(uri); + if (!m.matches()) + throw new IllegalArgumentException("URI: " + uri + " -- does not match pattern " + fullRestPattern); + + String user = m.group(1); + String pwd = m.group(2); + String host = m.group(3); + String portStr = m.group(4); + int port = Integer.parseInt(portStr == null ? "7070" : portStr); + + init(host, port, user, pwd); + } + + private void init(String host, int port, String userName, String password) { + this.host = host; + this.port = port; + this.userName = userName; + this.password = password; + this.baseUrl = "http://" + host + ":" + port + "/kylin/api"; + client = new DefaultHttpClient(); - - if (userName != null && password != null) { + + if (userName != null && password != null) { CredentialsProvider provider = new BasicCredentialsProvider(); UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password); - provider.setCredentials(AuthScope.ANY, credentials); - client.setCredentialsProvider(provider); - } - } - - public void wipeCache(String entity, String event, String cacheKey) throws IOException { - String url = baseUrl + "/cache/" + entity + "/" + cacheKey + "/" + event; + provider.setCredentials(AuthScope.ANY, credentials); + client.setCredentialsProvider(provider); + } + } + + public void wipeCache(String entity, String event, String cacheKey) throws IOException { + String url = baseUrl + "/cache/" + entity + "/" + cacheKey + "/" + event; HttpPut request = new HttpPut(url); - - try { + + try { HttpResponse response = client.execute(request); String msg = EntityUtils.toString(response.getEntity()); - + if (response.getStatusLine().getStatusCode() != 200) - throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); + throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); } catch (Exception ex) { - throw new IOException(ex); - } finally { - request.releaseConnection(); - } - } - - public String getKylinProperties() throws IOException { - String url = baseUrl + "/admin/config"; + throw new IOException(ex); + } finally { + request.releaseConnection(); + } + } + + public String getKylinProperties() throws IOException { + String url = baseUrl + "/admin/config"; HttpGet request = new HttpGet(url); - try { + try { HttpResponse response = client.execute(request); String msg = EntityUtils.toString(response.getEntity()); - Map<String, String> map = JsonUtil.readValueAsMap(msg); - msg = map.get("config"); - + Map<String, String> map = JsonUtil.readValueAsMap(msg); + msg = map.get("config"); + if (response.getStatusLine().getStatusCode() != 200) throw new IOException("Invalid response " + response.getStatusLine().getStatusCode() + " with cache wipe url " + url + "\n" + msg); - return msg; - } finally { - request.releaseConnection(); - } - } - -} + return msg; + } finally { + request.releaseConnection(); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/826f23f1/core-common/src/main/java/org/apache/kylin/common/util/Array.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Array.java b/core-common/src/main/java/org/apache/kylin/common/util/Array.java index c48bb3d..7447b46 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/Array.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/Array.java @@ -1,82 +1,82 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import java.util.Arrays; - -/* - * An array with correct equals(), hashCode(), compareTo() and toString() - */ -public class Array<T> implements Comparable<Array<T>> { - public T[] data; - - public Array(T[] data) { - this.data = data; - } - - public String toString() { - return Arrays.toString(data); - } - - @Override - public boolean equals(Object o) { - if (o != null && o instanceof Array) { - return Arrays.equals(this.data, ((Array<?>) o).data); - } - return false; - } - - @Override - public int hashCode() { - return Arrays.hashCode(data); - } - - @Override - public int compareTo(Array<T> other) { - return compare(this.data, other.data, null); - } - - public static boolean isEmpty(Object[] array) { - if (array == null || array.length == 0) - return true; - - for (Object o : array) { - if (o != null) - return false; - } - return true; - } - - @SuppressWarnings("unchecked") - public static <T> int compare(T[] a, T[] b, boolean[] ascending) { - int r = 0; - int n = Math.min(a.length, b.length); - boolean asc = true; - - for (int i = 0; i < n; i++) { - r = ((Comparable<T>) a[i]).compareTo(b[i]); - if (r != 0) { - asc = (ascending != null && ascending.length > i) ? ascending[i] : true; - return asc ? r : -r; - } - } - return a.length - b.length; - } - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.util; + +import java.util.Arrays; + +/* + * An array with correct equals(), hashCode(), compareTo() and toString() + */ +public class Array<T> implements Comparable<Array<T>> { + public T[] data; + + public Array(T[] data) { + this.data = data; + } + + public String toString() { + return Arrays.toString(data); + } + + @Override + public boolean equals(Object o) { + if (o != null && o instanceof Array) { + return Arrays.equals(this.data, ((Array<?>) o).data); + } + return false; + } + + @Override + public int hashCode() { + return Arrays.hashCode(data); + } + + @Override + public int compareTo(Array<T> other) { + return compare(this.data, other.data, null); + } + + public static boolean isEmpty(Object[] array) { + if (array == null || array.length == 0) + return true; + + for (Object o : array) { + if (o != null) + return false; + } + return true; + } + + @SuppressWarnings("unchecked") + public static <T> int compare(T[] a, T[] b, boolean[] ascending) { + int r = 0; + int n = Math.min(a.length, b.length); + boolean asc = true; + + for (int i = 0; i < n; i++) { + r = ((Comparable<T>) a[i]).compareTo(b[i]); + if (r != 0) { + asc = (ascending != null && ascending.length > i) ? ascending[i] : true; + return asc ? r : -r; + } + } + return a.length - b.length; + } + +}