This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master-hadoop3.1
in repository https://gitbox.apache.org/repos/asf/kylin.git.
omit fa4d339 Upgrade tomcat vertion to 8.5.51
omit 47b4e3c Backport HBASE-22887 to Kylin HFileOutputFormat3
omit 16f7293 fix kylin jdbc driver guava class not found problem
omit 5242ba6 [maven-release-plugin] prepare release kylin-3.0.0
omit 8444b88 KYLIN-3973: InvalidProtocolBufferException: Protocol message
was too large.
omit b8a2881 Package kafka and curator
omit 3162f9b KYLIN-4086 Upgrade avatica to 1.12
omit d4d2c12 remove htable toUpperCase, for HBase tableName is case
sensitive
omit 9160906 KYLIN-4072 find-hbase-dependency.sh don't work at CDH 6.x
omit 5473106 fix add jacoco.version in pom.xml (#699)
omit 849c253 Rebase master
omit 9fd8fed fix beeline meta data parser on partition information
omit 6d48885 KYLIN-3537
omit 8275caa KYLIN-2565 upgrade to hadoop 3.0 hbase 2.0, pass UT
add c75242a [maven-release-plugin] prepare release kylin-3.0.0
add 62fe6f9 [maven-release-plugin] prepare for next development iteration
add 46001fc KYLIN-4292 Use HFileOutputFormat3 in all places to replace
HFileOutputFormat2
add d30ac84 Minor, fix typo in KylinHealthCheckJob
add f888372 #4238 Fix kylin_streaming_model broken when changing
kylin.source.hive.database-for-flattable to non-default value
add 4300fa5 KYLIN-4280 SegmenPruner support 'OR' and 'NOT'
add c581826 KYLIN-4287 segmenPruner satisfy "IN" filter bug
add a233418 KYLIN-4166 Case when return null when SQL no GROUP BY
add 6635c04 KYLIN-4225 close hive session manually
add 753e1a8 KYLIN-4197 Set deadline for ClientEnvExtractor (#994)
add a2e3bcc KYLIN-4304 Project list cannot be correctly sorted by 'Create
Time'
add 14bad16 KYLIN-4252 Fix the error "Cannot read property 'index' of
null" in visualization page
add cb6e44c KYLIN-4235 Failed to load table metadata from JDBC data source
add a9945ca KYLIN-4237 Return error when execute "explain plan for SQL"
to get the execution plan of SQL
add b764400 KYLIN-4250 FechRunnner should skip the job to process other
jobs instead of throwing exception when the job section metadata is not found
add 8e2bc1f KYLIN-3609 KYLIN-3409 KYLIN-4161 CubeSegmentStatisticsMap
exist only on OLAPQuery.EnumeratorTypeEnum.OLAP query type. (#942)
add e8e1a20 KYLIN-4260 When using server side PreparedStatement cache,
the query result are not match on TopN scenario
add 21241e8 fix bug KYLIN-4300.
add 892d68b KYLIN-4297 Build cube throw NPE when partition column is not
set in JDBC Data Source
add b8c5252 fix bug in KYLIN-4243
add a877fee KYLIN-4295 Instances displayed on Query Node are inconsistent
with Job Node
add 466a7a3 KYLIN-4080 Project schema update event casues error reload
NEW DataModelDesc
add 00559cd KYLIN-4306: Delete data model desc when encountering
WriteConflictException
add 393bc1f KYLIN-3956 Segments of not only streaming cube but also batch
cube need to show their status
add 988b428 Set repository sites to https
add ebfc745 KYLIN-4333 Build Server OOM
add e373c64 Fix sql injection issue
add 2ee8e24 Prevent uncontrolled data used in path expression
add a1bd108 Encrept response output for BroadcasterReceiveServlet
add 17777e9 Fix "equals" on incomparable types
add 0d83874 Fix type mismatch on container access
add e683fff Fix not thread-safe double-checked locking
add 228f848 Fix potential possibility of array index out of bounds error
add abd29e4 Remove container never accessed
add 7395b46 Ensure the validity of http header from concated string
add bb03e27 Fix synchronization on boxed types or strings
add 7ea101d KYLIN-4327 TOPN Comparator may violate its general contract
add ac69061 Prevent uncontrolled data used in path expression
add ceefc82 Fix equals method does not inspect argument type
add 2e0d58a Fix queries built from user-controlled sources
add 173d88b Prevent uncontrolled data used in path expression
add ec7558e Prevent uncontrolled data used in path expression
add 5016842 Minor, throw exception in case of illegal project name
add 3b55848 Minor, add class TableDesc.TableProject to avoid the use of
Pair
add cff4593 Fix sql starts with "with" cannot be queried on web
add ef1e63c statement cache eviction invalidation base on time
add a91ae52 upgrade tomcat to 7.0.100
add ce04820 KYLIN-4390 Update tomcat to 7.0.100, update server.xml and
web.xml
add 18230a9 KYLIN-4370 Fix "Table or view not found" error when build
cube with spark for JDBC source at step "Build cube by Layer"
add 9a86c37 KYLIN-4132 Kylin needn't use
"org.apache.directory.api.util.Strings" to import extra dependency
add 35bd44b KYLIN-4400: use client in config to submit sql in
system-cube.sh
add f81c2f9 KYLIN-4396 Close FileReader in SaveDictStep
add c5dda34 KYLIN-4426 Refine CliCommandExecutor
add 76ece25 use system level KylinConfig for migration
add 25ce513 use cube config when uhc step enabled
add 9bd42fc KYLIN-4379: use config overwrite for all MR job
add e2aaec4 KYLIN-4425 Refactor Diagnosis Module
add a236bdd use newLinkedHashMap for deterministic order
add cc05aa5 Check partitionDesc for null value before use it
add 54d9eea KYLIN-3628 Fix the silently swallow legit exceptions and
reduce code duplication
add 556be3b KYLIN-4119 Project ADMIN can not operate the Hybrid model
add f4de7ab KYLIN-4438 fix bug: null password may cause RuntimeException
when starting up
add f242808 KYLIN-4470 The user cannot log in kylin normally after being
assigned to a group
add 7378d49 KYLIN-4437: repalce deprecated mapred.job.name & set cube in
hive mr name
add ece7506 KYLIN-4354 Prune segment not using given filter when using
jdbc preparestatement
add 58fad56 KYLIN-4426 CliCommandExecutor
add 619e05e KYLIN-4385 Fix HiveProducer can not write to Hive Table[AWS
Azure]
add 36bd70a KYLIN-4385 Only hdfs is appendable
add 8100701 KYLIN-4481 Project-level ACL lookups not working for
non-admin SAML-federated users
add 90f0946 KYLIN-4298 KYLIN-4299
add 918b8a1 Fix for KYLIN-4206
add 04acd09 KYLIN-4340 Fix bug of get value of isSparkFactDistinctEnable
for cube not correct
add ee15808 KYLIN-4353 Add regular check for cube state convert.
add 8cf0a73 KYLIN-4372 Fix bug of delete file too later cause ZK started
by HBase crash
add 543f4de KYLIN-4383 Fix reflection error
add 31cca88 KYLIN-4251 Add livy to docker
add 9fcd39f Minor, change resource URL in Dockerfile
add e96075a KYLIN-4272: optimize docker build scripts
add c474815 KYLIN-4388 Refine the Dockerfile
new f92c07d KYLIN-2565 upgrade to hadoop 3.0 hbase 2.0, pass UT
new 22c3dff KYLIN-3537
new 68424a4 fix beeline meta data parser on partition information
new 4af7e1a Rebase master
new 22da7c1 fix add jacoco.version in pom.xml (#699)
new 6ff4580 KYLIN-4072 find-hbase-dependency.sh don't work at CDH 6.x
new 9af7fb2 remove htable toUpperCase, for HBase tableName is case
sensitive
new 77a07f4 KYLIN-4086 Upgrade avatica to 1.12
new 34153d3 Package kafka and curator
new b672d05 KYLIN-3973: InvalidProtocolBufferException: Protocol message
was too large.
new a2742f3 [maven-release-plugin] prepare release kylin-3.0.0
new e91c080 fix kylin jdbc driver guava class not found problem
new 30355da Backport HBASE-22887 to Kylin HFileOutputFormat3
new 98ba5e8 Upgrade tomcat vertion to 8.5.51
new 35b4ef1 [maven-release-plugin] prepare release kylin-3.0.2
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fa4d339)
\
N -- N -- N refs/heads/master-hadoop3.1 (35b4ef1)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
assembly/pom.xml | 2 +-
build/bin/diag.sh | 17 ++
build/bin/kylin-port-replace-util.sh | 2 +-
build/bin/sample.sh | 13 +-
build/bin/system-cube.sh | 40 +++-
build/deploy/server.xml | 6 +-
build/script/download-tomcat.sh | 22 +-
cache/pom.xml | 2 +-
core-common/pom.xml | 2 +-
.../org/apache/kylin/common/KylinConfigBase.java | 37 +--
.../org/apache/kylin/common/KylinConfigExt.java | 2 +
.../common/metrics/metrics2/CodahaleMetrics.java | 13 +-
.../common/persistence/JDBCResourceStore.java | 47 ++--
.../kylin/common/persistence/ResourceStore.java | 3 +-
.../kylin/common/util/CliCommandExecutor.java | 45 +++-
.../org/apache/kylin/common/util/EncryptUtil.java | 6 +
.../org/apache/kylin/common/util/HadoopUtil.java | 1 -
.../apache/kylin/common/util/HiveCmdBuilder.java | 2 +-
.../src/main/resources/kylin-defaults.properties | 4 +-
.../kylin/common/util/CliCommandExecutorTest.java | 51 +++++
.../apache/kylin/common/util/EncryptUtilTest.java | 6 +
.../kylin/common/util/HiveCmdBuilderTest.java | 2 +-
core-cube/pom.xml | 2 +-
.../java/org/apache/kylin/cube/CubeManager.java | 32 +--
.../apache/kylin/cube/common/SegmentPruner.java | 63 +++---
.../apache/kylin/cube/common/TupleFilterNode.java | 76 +++++++
.../model/validation/rule/StreamingCubeRule.java | 3 +-
.../kylin/cube/common/SegmentPrunerTest.java | 202 +++++++++++++++++
core-dictionary/pom.xml | 2 +-
.../apache/kylin/dict/lookup/SnapshotManager.java | 44 ++--
core-job/pom.xml | 2 +-
.../kylin/job/execution/ExecutableManager.java | 4 +
.../job/impl/threadpool/DefaultFetcherRunner.java | 14 +-
.../job/impl/threadpool/PriorityFetcherRunner.java | 14 +-
core-metadata/pom.xml | 2 +-
.../measure/percentile/PercentileSerializer.java | 5 +-
.../kylin/measure/topn/DoubleDeltaSerializer.java | 6 +-
.../org/apache/kylin/measure/topn/TopNCounter.java | 8 +-
.../apache/kylin/measure/topn/TopNMeasureType.java | 7 +-
.../kylin/metadata/TableMetadataManager.java | 15 +-
.../metadata/badquery/BadQueryHistoryManager.java | 2 +
.../metadata/datatype/DataTypeSerializer.java | 5 +-
.../apache/kylin/metadata/model/DataModelDesc.java | 12 +
.../kylin/metadata/model/DataModelManager.java | 38 ++--
.../org/apache/kylin/metadata/model/TableDesc.java | 31 ++-
.../apache/kylin/metadata/model/TableExtDesc.java | 137 ++++++-----
.../kylin/metadata/realization/SQLDigest.java | 4 +-
.../kylin/measure/topn/TopNCounterBasicTest.java | 47 ++++
core-metrics/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
.../kylin/storage/hybrid/HybridInstanceTest.java | 1 +
datasource-sdk/pom.xml | 2 +-
docker/Dockerfile | 103 ++-------
docker/Dockerfile_dev | 43 ++++
docker/{Dockerfile => Dockerfile_hadoop} | 27 ++-
docker/README.md | 145 ++++++++++++
docker/build_image.sh | 22 +-
docker/entrypoint.sh | 19 +-
docker/run_container.sh | 3 +-
docker/use-guide.md | 136 -----------
engine-mr/pom.xml | 2 +-
.../engine/mr/common/MapReduceExecutable.java | 26 ++-
.../kylin/engine/mr/steps/CreateDictionaryJob.java | 4 +-
.../mr/streaming/ColumnarSplitDictReader.java | 2 +-
.../kylin/engine/mr/streaming/DictsReader.java | 2 +
.../kylin/engine/mr/streaming/SaveDictStep.java | 46 ++--
engine-spark/pom.xml | 2 +-
.../engine/spark/SparkBatchCubingJobBuilder2.java | 3 +-
.../org/apache/kylin/engine/spark/SparkUtil.java | 4 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
.../apache/kylin/storage/hbase/ITStorageTest.java | 2 +-
metrics-reporter-hive/pom.xml | 7 +-
.../kylin/metrics/lib/impl/hive/HiveProducer.java | 111 ++++++---
metrics-reporter-kafka/pom.xml | 2 +-
pom.xml | 8 +-
query/pom.xml | 2 +-
.../kylin/query/enumerator/OLAPEnumerator.java | 5 +-
.../apache/kylin/query/relnode/OLAPContext.java | 3 +-
.../apache/kylin/query/relnode/OLAPProjectRel.java | 32 +--
.../org/apache/kylin/query/util/QueryUtil.java | 55 ++---
.../org/apache/kylin/query/util/QueryUtilTest.java | 35 +++
server-base/pom.xml | 2 +-
.../broadcaster/BroadcasterReceiveServlet.java | 24 +-
.../kylin/rest/controller/CubeController.java | 3 +-
.../kylin/rest/controller/DashboardController.java | 17 +-
.../rest/controller/ExternalFilterController.java | 12 +-
.../kylin/rest/controller/ModelController.java | 1 +
.../kylin/rest/controller/QueryController.java | 10 +-
.../ServiceDiscoveryStateController.java | 29 ++-
.../rest/controller/StreamingV2Controller.java | 11 +-
.../kylin/rest/controller/UserController.java | 2 +
.../apache/kylin/rest/job/KylinHealthCheckJob.java | 4 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 47 ++--
.../rest/security/SAMLUserDetailsService.java | 11 +
.../org/apache/kylin/rest/service/CubeService.java | 94 +++++---
.../kylin/rest/service/DashboardService.java | 250 +++++++++++----------
.../kylin/rest/service/DiagnosisService.java | 5 +-
.../org/apache/kylin/rest/service/JobService.java | 9 +-
.../apache/kylin/rest/service/ModelService.java | 17 +-
.../apache/kylin/rest/service/ProjectService.java | 4 +-
.../apache/kylin/rest/service/QueryService.java | 12 +
.../org/apache/kylin/rest/util/ValidateUtil.java | 8 +-
server/pom.xml | 2 +-
server/src/main/webapp/WEB-INF/web.xml | 1 +
source-hive/pom.xml | 2 +-
.../kylin/source/hive/BeelineHiveClient.java | 2 +-
.../apache/kylin/source/hive/CLIHiveClient.java | 24 +-
.../kylin/source/hive/CreateFlatHiveTableStep.java | 2 +-
.../source/hive/HiveMetaStoreClientFactory.java | 62 +++++
.../source/hive/RedistributeFlatHiveTableStep.java | 2 +-
source-jdbc/pom.xml | 2 +-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 21 +-
.../kylin/source/jdbc/JdbcHiveInputBase.java | 8 +-
.../org/apache/kylin/source/jdbc/JdbcSource.java | 6 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 5 +-
storage-stream/pom.xml | 2 +-
stream-coordinator/pom.xml | 2 +-
.../stream/coordinator/assign/AssignmentUtil.java | 2 +-
.../assign/CubePartitionRoundRobinAssigner.java | 4 +-
.../coordinate/ReceiverClusterManager.java | 6 +-
.../coordinate/StreamingCoordinator.java | 2 +-
stream-core/pom.xml | 2 +-
.../core/storage/StreamingSegmentManager.java | 2 +-
.../storage/columnar/ColumnarStoreDimDesc.java | 5 +-
.../storage/columnar/ColumnarStoreMetricsDesc.java | 6 +-
stream-receiver/pom.xml | 2 +-
.../kylin/stream/server/StreamingServer.java | 9 +
stream-source-kafka/pom.xml | 2 +-
tomcat-ext/pom.xml | 2 +-
tool-assembly/pom.xml | 12 +-
tool/pom.xml | 8 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 4 +-
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 53 +++--
.../org/apache/kylin/tool/JobDiagnosisInfoCLI.java | 112 ++++-----
.../{ => extractor}/AbstractInfoExtractor.java | 51 ++++-
.../tool/{ => extractor}/ClientEnvExtractor.java | 70 ++++--
.../tool/{ => extractor}/CubeMetaExtractor.java | 69 ++++--
.../tool/{ => extractor}/HBaseUsageExtractor.java | 56 +++--
.../{common => extractor}/HadoopConfExtractor.java | 18 +-
.../kylin/tool/{ => extractor}/HiveConfigCLI.java | 2 +-
.../kylin/tool/extractor/JStackExtractor.java | 59 +++++
.../tool/{ => extractor}/JobInstanceExtractor.java | 4 +-
.../tool/{ => extractor}/KylinLogExtractor.java | 10 +-
.../tool/{ => extractor}/MrJobInfoExtractor.java | 14 +-
.../tool/extractor/SparkEnvInfoExtractor.java | 91 ++++++++
.../kylin/tool/extractor/YarnLogExtractor.java | 194 ++++++++++++++++
.../java/org/apache/kylin/tool/util/ToolUtil.java | 13 --
.../apache/kylin/tool/ClientEnvExtractorTest.java | 68 ++++++
.../apache/kylin/tool/CubeMetaExtractorTest.java | 1 +
webapp/app/js/controllers/cubeSchema.js | 4 -
webapp/app/js/controllers/instances.js | 2 +-
webapp/app/js/controllers/query.js | 4 +-
webapp/app/js/model/projectConfig.js | 2 +-
webapp/app/partials/cubes/cube_detail.html | 2 +-
157 files changed, 2521 insertions(+), 1129 deletions(-)
create mode 100644
core-common/src/test/java/org/apache/kylin/common/util/CliCommandExecutorTest.java
create mode 100644
core-cube/src/main/java/org/apache/kylin/cube/common/TupleFilterNode.java
create mode 100644 docker/Dockerfile_dev
copy docker/{Dockerfile => Dockerfile_hadoop} (82%)
create mode 100644 docker/README.md
mode change 100644 => 100755 docker/run_container.sh
delete mode 100644 docker/use-guide.md
create mode 100644
source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetaStoreClientFactory.java
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/AbstractInfoExtractor.java (81%)
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/ClientEnvExtractor.java (57%)
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/CubeMetaExtractor.java (92%)
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/HBaseUsageExtractor.java (80%)
rename tool/src/main/java/org/apache/kylin/tool/{common =>
extractor}/HadoopConfExtractor.java (82%)
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/HiveConfigCLI.java (98%)
create mode 100644
tool/src/main/java/org/apache/kylin/tool/extractor/JStackExtractor.java
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/JobInstanceExtractor.java (98%)
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/KylinLogExtractor.java (91%)
rename tool/src/main/java/org/apache/kylin/tool/{ =>
extractor}/MrJobInfoExtractor.java (97%)
create mode 100644
tool/src/main/java/org/apache/kylin/tool/extractor/SparkEnvInfoExtractor.java
create mode 100644
tool/src/main/java/org/apache/kylin/tool/extractor/YarnLogExtractor.java
create mode 100644
tool/src/test/java/org/apache/kylin/tool/ClientEnvExtractorTest.java
