Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d50b269 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d50b269 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d50b269 Branch: refs/heads/master Commit: 4d50b26972bb7bbaff852172990e0f189f987673 Parents: 7889cec 5f2eff6 Author: Li Yang <liy...@apache.org> Authored: Sat Dec 30 21:57:51 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Sun Dec 31 19:57:05 2017 +0800 ---------------------------------------------------------------------- README.md | 2 +- build/bin/load-hive-conf.sh | 4 +- build/script/download-spark.sh | 18 +- build/script/download-tomcat.sh | 4 +- .../apache/kylin/common/KylinConfigBase.java | 149 ++++- .../org/apache/kylin/common/QueryContext.java | 459 ++++++++++++- .../org/apache/kylin/common/util/BytesUtil.java | 2 +- .../apache/kylin/common/util/HadoopUtil.java | 19 + .../org/apache/kylin/common/util/JsonUtil.java | 11 +- .../java/org/apache/kylin/common/util/Pair.java | 16 + .../kylin-backward-compatibility.properties | 1 + .../main/resources/kylin-defaults.properties | 13 + core-cube/pom.xml | 4 + .../org/apache/kylin/cube/CubeDescManager.java | 4 + .../org/apache/kylin/cube/CubeInstance.java | 90 ++- .../java/org/apache/kylin/cube/CubeManager.java | 158 +++-- .../java/org/apache/kylin/cube/CubeUpdate.java | 14 +- .../kylin/cube/common/RowKeySplitter.java | 2 +- .../org/apache/kylin/cube/cuboid/Cuboid.java | 10 +- .../apache/kylin/cube/cuboid/CuboidUtil.java | 85 +++ .../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +- .../cuboid/algorithm/CuboidRecommender.java | 6 + .../inmemcubing/AbstractInMemCubeBuilder.java | 13 +- .../ConsumeBlockingQueueController.java | 84 +++ .../cube/inmemcubing/DoggedCubeBuilder.java | 128 +--- .../cube/inmemcubing/InMemCubeBuilder.java | 102 +-- .../InMemCubeBuilderInputConverter.java | 149 ----- .../kylin/cube/inmemcubing/InputConverter.java | 69 ++ .../cube/inmemcubing/InputConverterUnit.java | 42 ++ .../InputConverterUnitForBaseCuboid.java | 64 ++ .../InputConverterUnitForRawData.java | 169 +++++ .../RecordConsumeBlockingQueueController.java | 80 +++ .../org/apache/kylin/cube/kv/RowKeyDecoder.java | 4 +- .../kylin/cube/model/AggregationGroup.java | 10 +- .../kylin/cube/model/CubeBuildTypeEnum.java | 7 +- .../org/apache/kylin/cube/model/CubeDesc.java | 111 +++- .../validation/rule/AggregationGroupRule.java | 2 +- .../kylin/gridtable/GTAggregateScanner.java | 8 +- .../apache/kylin/gridtable/GTFilterScanner.java | 6 + .../org/apache/kylin/gridtable/GTRecord.java | 5 + .../kylin/cube/AggregationGroupRuleTest.java | 6 +- .../org/apache/kylin/cube/CubeDescTest.java | 31 +- .../org/apache/kylin/cube/CubeInstanceTest.java | 14 +- .../org/apache/kylin/cube/CubeManagerTest.java | 109 +++ .../kylin/cube/cuboid/CuboidUtilTest.java | 57 ++ .../ConsumeBlockingQueueControllerTest.java | 124 ++++ .../apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +- .../apache/kylin/cube/kv/RowKeyEncoderTest.java | 6 +- .../apache/kylin/dict/DictionaryGenerator.java | 44 +- .../org/apache/kylin/dict/DictionaryInfo.java | 4 + .../apache/kylin/dict/DictionaryManager.java | 4 + .../kylin/dict/GlobalDictionaryBuilder.java | 16 +- .../apache/kylin/dict/IDictionaryBuilder.java | 2 +- .../dict/MultipleDictionaryValueEnumerator.java | 10 +- .../kylin/dict/Number2BytesConverter.java | 14 +- .../global/SegmentAppendTrieDictBuilder.java | 9 +- .../kylin/dict/DictionaryProviderTest.java | 2 +- .../apache/kylin/dict/TrieDictionaryTest.java | 30 +- core-job/pom.xml | 4 + .../org/apache/kylin/engine/EngineFactory.java | 4 + .../apache/kylin/engine/IBatchCubingEngine.java | 3 + .../org/apache/kylin/job/JoinedFlatTable.java | 18 +- .../kylin/job/common/ShellExecutable.java | 7 +- .../kylin/job/constant/ExecutableConstants.java | 6 + .../org/apache/kylin/job/dao/ExecutableDao.java | 18 +- .../org/apache/kylin/job/dao/ExecutablePO.java | 11 + .../kylin/job/engine/JobEngineConfig.java | 14 + .../kylin/job/exception/ShellException.java | 43 ++ .../kylin/job/execution/AbstractExecutable.java | 50 +- .../job/execution/CheckpointExecutable.java | 122 ++++ .../job/execution/DefaultChainedExecutable.java | 30 +- .../kylin/job/execution/ExecutableManager.java | 46 +- .../kylin/job/execution/ExecuteResult.java | 31 + .../job/impl/threadpool/DefaultScheduler.java | 170 ++++- .../kylin/job/metrics/JobMetricsFacade.java | 142 ++++ .../job/FiveSecondSucceedTestExecutable.java | 2 +- .../kylin/job/RetryableTestExecutable.java | 50 ++ .../apache/kylin/job/RunningTestExecutable.java | 2 +- .../apache/kylin/job/SelfStopExecutable.java | 2 +- .../apache/kylin/job/SucceedTestExecutable.java | 2 +- .../impl/threadpool/DefaultSchedulerTest.java | 16 + .../kylin/measure/hllc/HLLCSerializer.java | 6 +- .../kylin/metadata/TableMetadataManager.java | 2 + .../metadata/datatype/BigDecimalSerializer.java | 3 +- .../metadata/datatype/BooleanSerializer.java | 5 +- .../metadata/datatype/DataTypeSerializer.java | 1 + .../metadata/datatype/DateTimeSerializer.java | 5 +- .../metadata/datatype/DoubleSerializer.java | 1 + .../kylin/metadata/datatype/Int4Serializer.java | 5 +- .../metadata/datatype/Long8Serializer.java | 5 +- .../kylin/metadata/datatype/LongSerializer.java | 5 +- .../metadata/datatype/StringSerializer.java | 1 + .../kylin/metadata/model/DataModelDesc.java | 2 +- .../kylin/metadata/model/FunctionDesc.java | 13 +- .../kylin/metadata/model/ParameterDesc.java | 6 +- .../kylin/metadata/model/PartitionDesc.java | 2 +- .../apache/kylin/metadata/model/Segments.java | 26 +- .../kylin/metadata/project/ProjectInstance.java | 94 ++- .../hllc/NewHyperLogLogBenchmarkTest.java | 34 +- core-metrics/pom.xml | 51 ++ .../apache/kylin/metrics/MetricsManager.java | 148 +++++ .../kylin/metrics/lib/ActiveReservoir.java | 40 ++ .../metrics/lib/ActiveReservoirFilter.java | 44 ++ .../metrics/lib/ActiveReservoirListener.java | 30 + .../metrics/lib/ActiveReservoirReporter.java | 52 ++ .../org/apache/kylin/metrics/lib/Record.java | 51 ++ .../java/org/apache/kylin/metrics/lib/Sink.java | 23 + .../org/apache/kylin/metrics/lib/SinkTool.java | 32 + .../lib/impl/AbstractActiveReservoir.java | 68 ++ .../metrics/lib/impl/BaseScheduledReporter.java | 103 +++ .../metrics/lib/impl/BlockingReservoir.java | 167 +++++ .../metrics/lib/impl/InstantReservoir.java | 76 +++ .../kylin/metrics/lib/impl/MetricsSystem.java | 164 +++++ .../kylin/metrics/lib/impl/RecordEvent.java | 272 ++++++++ .../metrics/lib/impl/RecordEventTimeDetail.java | 77 +++ .../kylin/metrics/lib/impl/ReporterBuilder.java | 48 ++ .../kylin/metrics/lib/impl/StubReservoir.java | 54 ++ .../metrics/lib/impl/StubReservoirReporter.java | 51 ++ .../apache/kylin/metrics/lib/impl/StubSink.java | 30 + .../metrics/lib/impl/TimePropertyEnum.java | 49 ++ .../metrics/lib/impl/TimedRecordEvent.java | 47 ++ .../kylin/metrics/property/JobPropertyEnum.java | 56 ++ .../metrics/property/QueryCubePropertyEnum.java | 54 ++ .../metrics/property/QueryPropertyEnum.java | 52 ++ .../metrics/property/QueryRPCPropertyEnum.java | 51 ++ .../apache/kylin/storage/StorageContext.java | 34 +- .../storage/gtrecord/CubeScanRangePlanner.java | 49 +- .../gtrecord/GTCubeStorageQueryBase.java | 35 +- .../gtrecord/SequentialCubeTupleIterator.java | 6 +- dev-support/make_patch.sh | 156 +++++ dev-support/python-requirements.txt | 21 + dev-support/submit-patch.py | 311 +++++++++ engine-mr/pom.xml | 4 + .../kylin/engine/mr/BatchCubingJobBuilder2.java | 29 +- .../engine/mr/BatchOptimizeJobBuilder2.java | 228 +++++++ .../mr/BatchOptimizeJobCheckpointBuilder.java | 89 +++ .../org/apache/kylin/engine/mr/CubingJob.java | 104 ++- .../org/apache/kylin/engine/mr/IMROutput2.java | 29 +- .../kylin/engine/mr/JobBuilderSupport.java | 73 ++- .../kylin/engine/mr/MRBatchCubingEngine2.java | 5 + .../java/org/apache/kylin/engine/mr/MRUtil.java | 4 + .../engine/mr/common/AbstractHadoopJob.java | 51 +- .../kylin/engine/mr/common/BatchConstants.java | 8 + .../kylin/engine/mr/common/CubeStatsReader.java | 147 +++-- .../kylin/engine/mr/common/CubeStatsWriter.java | 25 +- .../engine/mr/common/CuboidRecommenderUtil.java | 22 +- .../engine/mr/common/CuboidSchedulerUtil.java | 60 ++ .../engine/mr/common/CuboidStatsReaderUtil.java | 15 +- .../kylin/engine/mr/common/HadoopCmdOutput.java | 9 +- .../engine/mr/common/HadoopShellExecutable.java | 8 +- .../engine/mr/common/JobInfoConverter.java | 67 +- .../engine/mr/common/MapReduceExecutable.java | 9 +- .../kylin/engine/mr/common/MapReduceUtil.java | 132 ++++ .../mr/common/SerializableConfiguration.java | 50 ++ .../mr/common/StatisticsDecisionUtil.java | 34 +- .../mr/exception/HadoopShellException.java | 44 ++ .../engine/mr/exception/MapReduceException.java | 43 ++ .../mr/exception/SegmentNotFoundException.java | 44 ++ .../engine/mr/steps/BaseCuboidMapperBase.java | 2 +- .../steps/CalculateStatsFromBaseCuboidJob.java | 120 ++++ .../CalculateStatsFromBaseCuboidMapper.java | 201 ++++++ ...CalculateStatsFromBaseCuboidPartitioner.java | 59 ++ .../CalculateStatsFromBaseCuboidReducer.java | 115 ++++ .../engine/mr/steps/CopyDictionaryStep.java | 70 ++ .../engine/mr/steps/CreateDictionaryJob.java | 16 +- .../engine/mr/steps/CubingExecutableUtil.java | 9 + .../apache/kylin/engine/mr/steps/CuboidJob.java | 28 +- .../mr/steps/FactDistinctColumnPartitioner.java | 51 +- .../engine/mr/steps/FactDistinctColumnsJob.java | 37 +- .../mr/steps/FactDistinctColumnsMapper.java | 332 ++++++---- .../mr/steps/FactDistinctColumnsMapperBase.java | 19 +- .../mr/steps/FactDistinctColumnsReducer.java | 54 +- .../FactDistinctColumnsReducerMapping.java | 156 +++++ .../mr/steps/FilterRecommendCuboidDataJob.java | 103 +++ .../steps/FilterRecommendCuboidDataMapper.java | 107 +++ .../mr/steps/InMemCuboidFromBaseCuboidJob.java | 160 +++++ .../steps/InMemCuboidFromBaseCuboidMapper.java | 102 +++ .../steps/InMemCuboidFromBaseCuboidReducer.java | 23 + .../kylin/engine/mr/steps/InMemCuboidJob.java | 4 +- .../engine/mr/steps/InMemCuboidMapper.java | 130 +--- .../engine/mr/steps/InMemCuboidMapperBase.java | 210 ++++++ .../kylin/engine/mr/steps/KVGTRecordWriter.java | 2 +- .../engine/mr/steps/MergeCuboidMapper.java | 9 +- .../engine/mr/steps/MergeDictionaryStep.java | 4 +- .../engine/mr/steps/MergeStatisticsStep.java | 4 +- .../mr/steps/MergeStatisticsWithOldStep.java | 155 +++++ .../kylin/engine/mr/steps/NDCuboidMapper.java | 15 +- .../kylin/engine/mr/steps/ReducerNumSizing.java | 106 --- .../engine/mr/steps/SaveStatisticsStep.java | 103 ++- .../kylin/engine/mr/steps/UHCDictionaryJob.java | 154 +++++ .../engine/mr/steps/UHCDictionaryMapper.java | 101 +++ .../mr/steps/UHCDictionaryPartitioner.java | 30 + .../engine/mr/steps/UHCDictionaryReducer.java | 113 ++++ .../mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 +- .../UpdateCubeInfoAfterCheckpointStep.java | 69 ++ .../mr/steps/UpdateCubeInfoAfterMergeStep.java | 8 +- .../steps/UpdateCubeInfoAfterOptimizeStep.java | 72 ++ .../mr/steps/UpdateOldCuboidShardJob.java | 105 +++ .../mr/steps/UpdateOldCuboidShardMapper.java | 142 ++++ .../FactDistinctColumnsReducerMappingTest.java | 93 +++ .../engine/spark/SparkBatchCubingEngine.java | 67 -- .../kylin/engine/spark/SparkCountDemo.java | 80 --- .../apache/kylin/engine/spark/SparkCubing.java | 590 ----------------- .../kylin/engine/spark/SparkCubingByLayer.java | 48 +- .../engine/spark/SparkCubingJobBuilder.java | 69 -- .../kylin/engine/spark/SparkCuboidWriter.java | 29 - .../kylin/engine/spark/SparkExecutable.java | 2 +- .../kylin/engine/spark/SparkHelloWorld.java | 37 -- .../kylin/engine/spark/SparkHiveDemo.java | 52 -- .../engine/spark/cube/BufferedCuboidWriter.java | 106 --- .../spark/cube/DefaultTupleConverter.java | 98 --- .../spark/cube/ListBackedCuboidWriter.java | 59 -- .../kylin/engine/spark/cube/TupleConverter.java | 29 - .../spark/cube/BufferedCuboidWriterTest.java | 69 -- .../test_case_data/localmeta/kylin.properties | 1 + .../test_case_data/sandbox/kylin.properties | 15 + .../ITDoggedCubeBuilderStressTest.java | 3 +- .../inmemcubing/ITDoggedCubeBuilderTest.java | 8 +- .../inmemcubing/ITInMemCubeBuilderTest.java | 27 +- .../dict/ITGlobalDictionaryBuilderTest.java | 4 +- .../apache/kylin/job/ContextTestExecutable.java | 2 +- .../hbase/ITAclTableMigrationToolTest.java | 2 + .../query/sql_intersect_count/query01.sql | 27 + .../query/sql_intersect_count/query03.sql | 23 + metrics-reporter-hive/pom.xml | 53 ++ .../metrics/lib/impl/hive/HiveProducer.java | 201 ++++++ .../lib/impl/hive/HiveProducerRecord.java | 196 ++++++ .../lib/impl/hive/HiveReservoirReporter.java | 139 ++++ .../kylin/metrics/lib/impl/hive/HiveSink.java | 30 + metrics-reporter-kafka/pom.xml | 46 ++ .../kafka/KafkaActiveReserviorListener.java | 115 ++++ .../lib/impl/kafka/KafkaReservoirReporter.java | 139 ++++ .../kylin/metrics/lib/impl/kafka/KafkaSink.java | 29 + pom.xml | 35 +- .../org/apache/kylin/query/QueryConnection.java | 2 +- .../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +- .../kylin/query/enumerator/OLAPQuery.java | 3 + .../kylin/query/relnode/OLAPAggregateRel.java | 1 + .../apache/kylin/query/relnode/OLAPContext.java | 2 +- .../kylin/query/relnode/OLAPTableScan.java | 2 +- .../kylin/query/security/QuerACLTestUtil.java | 43 -- .../kylin/query/security/QueryACLTestUtil.java | 55 ++ .../query/util/KeywordDefaultDirtyHack.java | 3 +- .../query/util/KeywordDefaultDirtyHackTest.java | 60 ++ server-base/pom.xml | 12 + .../kylin/rest/controller/CubeController.java | 277 +++++++- .../rest/controller/DashboardController.java | 129 ++++ .../kylin/rest/controller/JobController.java | 5 +- .../rest/controller/ProjectController.java | 16 +- .../rest/controller/StreamingController.java | 46 +- .../rest/job/StorageCleanJobHbaseUtil.java | 4 +- .../kylin/rest/job/StorageCleanupJob.java | 18 +- .../kylin/rest/metrics/QueryMetricsFacade.java | 165 ++++- .../java/org/apache/kylin/rest/msg/Message.java | 12 +- .../kylin/rest/request/JobOptimizeRequest.java | 34 + .../apache/kylin/rest/request/SQLRequest.java | 2 +- .../rest/response/CubeInstanceResponse.java | 26 +- .../kylin/rest/response/CuboidTreeResponse.java | 123 ++++ .../apache/kylin/rest/response/SQLResponse.java | 46 +- .../rest/security/AclPermissionFactory.java | 11 +- .../rest/security/AuthoritiesPopulator.java | 1 + .../rest/security/ExternalAclProvider.java | 16 +- .../apache/kylin/rest/service/BasicService.java | 4 + .../apache/kylin/rest/service/CubeService.java | 115 +++- .../kylin/rest/service/DashboardService.java | 333 ++++++++++ .../apache/kylin/rest/service/JobService.java | 434 +++++++++++- .../apache/kylin/rest/service/ModelService.java | 2 +- .../apache/kylin/rest/service/QueryService.java | 182 ++--- .../apache/kylin/rest/service/TableService.java | 1 - .../kylin/rest/util/QueryRequestLimits.java | 121 ++++ .../apache/kylin/rest/util/ValidateUtil.java | 1 + .../kylin/rest/util/QueryRequestLimitsTest.java | 69 ++ server/src/main/resources/kylinMetrics.xml | 79 +++ server/src/main/resources/kylinSecurity.xml | 5 +- server/src/main/webapp/WEB-INF/web.xml | 1 + .../rest/controller/CubeControllerTest.java | 9 - .../kylin/rest/metrics/QueryMetricsTest.java | 42 ++ .../rest/security/QueryWithTableACLTest.java | 22 +- .../kylin/rest/service/AdminServiceTest.java | 36 +- .../kylin/rest/service/CacheServiceTest.java | 2 +- .../kylin/rest/service/ServiceTestBase.java | 3 +- .../source/hive/CreateFlatHiveTableStep.java | 2 +- .../apache/kylin/source/hive/HiveMRInput.java | 9 +- .../cardinality/HiveColumnCardinalityJob.java | 2 +- .../hive/exception/SegmentEmptyException.java | 44 ++ .../org/apache/kylin/source/jdbc/CmdStep.java | 2 +- .../apache/kylin/source/jdbc/HiveCmdStep.java | 2 +- .../apache/kylin/source/jdbc/JdbcExplorer.java | 2 +- .../kylin/source/jdbc/JdbcHiveMRInput.java | 17 +- .../org/apache/kylin/source/jdbc/SqlUtil.java | 2 +- .../apache/kylin/source/jdbc/SqlUtilTest.java | 6 +- .../apache/kylin/source/kafka/KafkaMRInput.java | 2 +- .../source/kafka/config/KafkaClusterConfig.java | 19 - .../kylin/source/kafka/job/MergeOffsetStep.java | 4 +- .../source/kafka/job/UpdateTimeRangeStep.java | 2 +- .../kylin/storage/hbase/HBaseConnection.java | 8 + .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 22 +- .../storage/hbase/cube/v2/CubeHBaseRPC.java | 12 +- .../hbase/cube/v2/ExpectedSizeIterator.java | 4 +- .../coprocessor/endpoint/CubeVisitService.java | 40 +- .../endpoint/generated/CubeVisitProtos.java | 115 +++- .../endpoint/protobuf/CubeVisit.proto | 1 + .../kylin/storage/hbase/steps/BulkLoadJob.java | 2 +- .../storage/hbase/steps/CreateHTableJob.java | 30 +- .../kylin/storage/hbase/steps/CubeHFileJob.java | 30 +- .../storage/hbase/steps/CubeHFileMapper.java | 14 +- .../storage/hbase/steps/DeprecatedGCStep.java | 2 +- .../storage/hbase/steps/HBaseCuboidWriter.java | 5 +- .../hbase/steps/HBaseMROutput2Transition.java | 50 +- .../kylin/storage/hbase/steps/HBaseMRSteps.java | 81 ++- .../steps/HDFSPathGarbageCollectionStep.java | 2 +- .../kylin/storage/hbase/steps/MergeGCStep.java | 3 +- .../storage/hbase/steps/RowKeyWritable.java | 86 +++ .../hbase/util/DeployCoprocessorCLI.java | 150 +++-- .../hbase/util/HtableAlterMetadataCLI.java | 21 +- .../storage/hbase/util/ZookeeperAclBuilder.java | 119 ++++ .../hbase/util/ZookeeperDistributedLock.java | 3 +- .../hbase/steps/CubeHFileMapper2Test.java | 4 - .../hbase/steps/CubeHFileMapperTest.java | 8 +- .../hbase/util/ZookeeperAclBuilderTest.java | 97 +++ tool/pom.xml | 5 + .../apache/kylin/tool/job/CubeBuildingCLI.java | 138 ++++ .../metrics/systemcube/CubeDescCreator.java | 656 +++++++++++++++++++ .../metrics/systemcube/CubeInstanceCreator.java | 88 +++ .../metrics/systemcube/HiveTableCreator.java | 278 ++++++++ .../metrics/systemcube/KylinTableCreator.java | 114 ++++ .../tool/metrics/systemcube/ModelCreator.java | 267 ++++++++ .../tool/metrics/systemcube/ProjectCreator.java | 101 +++ .../tool/metrics/systemcube/SCCreator.java | 262 ++++++++ .../metrics/systemcube/util/HiveSinkTool.java | 61 ++ tool/src/main/resources/SCSinkTools.json | 15 + .../tool/metrics/systemcube/SCCreatorTest.java | 134 ++++ tool/src/test/resources/SCSinkTools.json | 1 + webapp/app/index.html | 15 +- webapp/app/js/app.js | 2 +- webapp/app/js/controllers/access.js | 10 +- webapp/app/js/controllers/admin.js | 5 + webapp/app/js/controllers/badQuery.js | 2 +- webapp/app/js/controllers/cube.js | 188 +++++- webapp/app/js/controllers/cubeAdvanceSetting.js | 60 ++ webapp/app/js/controllers/cubeEdit.js | 13 +- webapp/app/js/controllers/cubeMeasures.js | 10 +- webapp/app/js/controllers/cubeSchema.js | 48 +- webapp/app/js/controllers/dashboard.js | 296 +++++++++ webapp/app/js/controllers/job.js | 20 +- webapp/app/js/controllers/modelSchema.js | 4 +- webapp/app/js/controllers/page.js | 12 +- webapp/app/js/controllers/query.js | 119 +++- webapp/app/js/controllers/streamingConfig.js | 43 +- webapp/app/js/directives/directives.js | 97 +++ webapp/app/js/directives/ui-grid.js | 12 +- webapp/app/js/factories/graph.js | 77 --- webapp/app/js/filters/filter.js | 21 + webapp/app/js/model/cubeConfig.js | 80 ++- webapp/app/js/model/cubeDescModel.js | 5 +- webapp/app/js/model/dashboardConfig.js | 96 +++ webapp/app/js/model/jobListModel.js | 13 + webapp/app/js/model/queryConfig.js | 139 ++++ webapp/app/js/services/cubes.js | 47 +- webapp/app/js/services/dashboard.js | 95 +++ webapp/app/js/services/graph.js | 54 -- webapp/app/js/services/kylinProperties.js | 22 +- webapp/app/js/services/query.js | 13 +- webapp/app/js/utils/utils.js | 10 + webapp/app/less/app.less | 88 +++ webapp/app/less/component.less | 4 + webapp/app/partials/admin/admin.html | 44 +- .../cubeDesigner/advanced_settings.html | 91 ++- webapp/app/partials/cubeDesigner/measures.html | 6 +- .../partials/cubeDesigner/refresh_settings.html | 34 +- .../partials/cubeDesigner/streamingConfig.html | 27 +- webapp/app/partials/cubes/cube_detail.html | 83 ++- webapp/app/partials/cubes/cubes.html | 6 +- webapp/app/partials/dashboard/dashboard.html | 151 +++++ webapp/app/partials/header.html | 3 + webapp/app/partials/jobs/badQuery.html | 6 +- webapp/app/partials/jobs/jobList.html | 4 +- .../app/partials/modelDesigner/data_model.html | 2 +- webapp/app/partials/models/model_detail.html | 2 +- webapp/app/partials/models/models_tree.html | 4 +- .../partials/projects/project_table_tree.html | 2 +- webapp/app/partials/query/query_detail.html | 104 +-- .../app/partials/tables/source_table_tree.html | 2 +- webapp/app/partials/tables/table_detail.html | 2 +- webapp/app/partials/tables/table_load.html | 2 +- webapp/app/routes.json | 8 + webapp/bower.json | 14 +- webapp/grunt.json | 9 +- 388 files changed, 18016 insertions(+), 3356 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --cc core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 277ee31,7763457..2effdfb --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@@ -1043,11 -1035,33 +1091,33 @@@ abstract public class KylinConfigBase i return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000")); } + public int getCuboidStatsCalculatorMaxNumber() { + // set 1 to disable multi-thread statistics calculation + return Integer.parseInt(getOptional("kylin.engine.mr.max-cuboid-stats-calculator-number", "1")); + } + + public int getCuboidNumberPerStatsCalculator() { + return Integer.parseInt(getOptional("kylin.engine.mr.cuboid-number-per-stats-calculator", "100")); + } + + public int getHadoopJobPerReducerHLLCuboidNumber() { + return Integer.parseInt(getOptional("kylin.engine.mr.per-reducer-hll-cuboid-number", "100")); + } + + public int getHadoopJobHLLMaxReducerNumber() { + // by default multi-reducer hll calculation is disabled + return Integer.parseInt(getOptional("kylin.engine.mr.hll-max-reducer-number", "1")); + } + //UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns public int getUHCReducerCount() { - return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "5")); + return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1")); } + public boolean isBuildUHCDictWithMREnabled() { - return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-uhc-dict", "true")); ++ return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-uhc-dict-in-additional-step", "false")); + } + public boolean isBuildDictInReducerEnabled() { return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", "true")); } @@@ -1383,14 -1354,6 +1458,22 @@@ return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } + public boolean isAdminUserExportAllowed() { + return Boolean.parseBoolean(getOptional("kylin.web.export-allow-admin", "true")); + } + + public boolean isNoneAdminUserExportAllowed() { + return Boolean.parseBoolean(getOptional("kylin.web.export-allow-other", "true")); + } + ++ public String getPropertiesWhiteList() { ++ return getOptional("kylin.web.properties.whitelist", ++ "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic," ++ + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3," ++ + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile," ++ + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other"); ++ } ++ // ============================================================================ // RESTCLIENT // ============================================================================ @@@ -1403,9 -1366,9 +1486,10 @@@ return Integer.valueOf(this.getOptional("kylin.restclient.connection.max-total", "200")); } - /** - * metric - */ + // ============================================================================ + // Metrics + // ============================================================================ ++ public String getCoadhaleMetricsReportClassesNames() { return getOptional("kylin.metrics.reporter-classes", "org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter"); @@@ -1423,19 -1386,56 +1507,64 @@@ return getOptional("kylin.metrics.perflogger-class", "org.apache.kylin.common.metrics.perflog.PerfLogger"); } + public boolean isShowingGuiTraceToggle() { + return Boolean.valueOf(getOptional("kylin.htrace.show-gui-trace-toggle", "false")); + } + + public boolean isHtraceTracingEveryQuery() { + return Boolean.valueOf(getOptional("kylin.htrace.trace-every-query", "false")); + } + - public String getPropertiesWhiteList() { - return getOptional("kylin.web.properties.whitelist", - "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic," - + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3," - + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile," - + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other"); + public boolean isKylinMetricsMonitorEnabled() { + return Boolean.parseBoolean(getOptional("kylin.core.metrics.monitor-enabled", "false")); + } + + public boolean isKylinMetricsReporterForQueryEnabled() { + return Boolean.parseBoolean(getOptional("kylin.core.metrics.reporter-query-enabled", "false")); + } + + public boolean isKylinMetricsReporterForJobEnabled() { + return Boolean.parseBoolean(getOptional("kylin.core.metrics.reporter-job-enabled", "false")); + } + + public String getKylinMetricsPrefix() { + return getOptional("kylin.core.metrics.prefix", "KYLIN").toUpperCase(); + } + + public String getKylinMetricsActiveReservoirDefaultClass() { + return getOptional("kylin.core.metrics.active-reservoir-default-class", + "org.apache.kylin.metrics.lib.impl.StubReservoir"); + } + + public String getKylinSystemCubeSinkDefaultClass() { + return getOptional("kylin.core.metrics.system-cube-sink-default-class", + "org.apache.kylin.metrics.lib.impl.hive.HiveSink"); + } + + public String getKylinMetricsSubjectSuffix() { + return getOptional("kylin.core.metric.subject-suffix", getDeployEnv()); + } + + public String getKylinMetricsSubjectJob() { + return getOptional("kylin.core.metrics.subject-job", "METRICS_JOB") + "_" + getKylinMetricsSubjectSuffix(); + } + + public String getKylinMetricsSubjectJobException() { + return getOptional("kylin.core.metrics.subject-job-exception", "METRICS_JOB_EXCEPTION") + "_" + + getKylinMetricsSubjectSuffix(); + } + + public String getKylinMetricsSubjectQuery() { + return getOptional("kylin.core.metrics.subject-query", "METRICS_QUERY") + "_" + getKylinMetricsSubjectSuffix(); + } + + public String getKylinMetricsSubjectQueryCube() { + return getOptional("kylin.core.metrics.subject-query-cube", "METRICS_QUERY_CUBE") + "_" + + getKylinMetricsSubjectSuffix(); + } + + public String getKylinMetricsSubjectQueryRpcCall() { + return getOptional("kylin.core.metrics.subject-query-rpc", "METRICS_QUERY_RPC") + "_" + + getKylinMetricsSubjectSuffix(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/QueryContext.java ---------------------------------------------------------------------- diff --cc core-common/src/main/java/org/apache/kylin/common/QueryContext.java index f00a7cc,c7ded10..1af90f4 --- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java +++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java @@@ -18,7 -18,9 +18,10 @@@ package org.apache.kylin.common; + import java.io.Serializable; + import java.util.List; + import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; @@@ -34,9 -45,11 +46,12 @@@ public class QueryContext } }; + private long queryStartMillis; + private long deadline = Long.MAX_VALUE; + private String queryId; private String username; + private Set<String> groups; private AtomicLong scannedRows = new AtomicLong(); private AtomicLong scannedBytes = new AtomicLong(); http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/resources/kylin-defaults.properties ---------------------------------------------------------------------- diff --cc core-common/src/main/resources/kylin-defaults.properties index 51e5a2d,6ca516d..e87b31d --- a/core-common/src/main/resources/kylin-defaults.properties +++ b/core-common/src/main/resources/kylin-defaults.properties @@@ -43,12 -43,14 +43,18 @@@ kylin.server.cluster-servers=localhost: # Display timezone on UI,format like[GMT+N or GMT-N] kylin.web.timezone=GMT+8 + # Timeout value for the queries submitted through the Web UI, in milliseconds + kylin.web.query-timeout=300000 + kylin.web.cross-domain-enabled=true +#allow user to export query result +kylin.web.export-allow-admin=true +kylin.web.export-allow-other=true + + # Hide measures in measure list of cube designer, separate by comma + kylin.web.hide-measures=RAW + #max connections of one route kylin.restclient.connection.default-max-per-route=20 @@@ -179,6 -166,6 +188,9 @@@ kylin.engine.mr.build-dict-in-reducer=t # Number of reducers for fetching UHC column distinct values kylin.engine.mr.uhc-reducer-count=1 ++# Whether using an additional step to build UHC dictionary ++kylin.engine.mr.build-uhc-dict-in-additional-step=false ++ ### CUBE | DICTIONARY ### http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java ---------------------------------------------------------------------- diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java index a416499,d522263..cd43f15 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java @@@ -25,10 -25,12 +25,12 @@@ import java.util.Map import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.KylinConfig; + import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.ResourceStore; + import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.common.util.AutoReadWriteLock; +import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock; +import org.apache.kylin.cube.cuboid.CuboidManager; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.validation.CubeMetadataValidator; import org.apache.kylin.cube.model.validation.ValidateContext; @@@ -58,15 -61,39 +60,17 @@@ public class CubeDescManager private static final Logger logger = LoggerFactory.getLogger(CubeDescManager.class); + public static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER = new JsonSerializer<CubeDesc>(CubeDesc.class); - - // static cached instances - private static final ConcurrentMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>(); - ++ public static CubeDescManager getInstance(KylinConfig config) { - CubeDescManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (CubeDescManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new CubeDescManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init CubeDescManager from " + config, e); - } - } + return config.getManager(CubeDescManager.class); } - public static void clearCache() { - CACHE.clear(); + // called by reflection + static CubeDescManager newInstance(KylinConfig config) throws IOException { + return new CubeDescManager(config); } - + // ============================================================================ private KylinConfig config; http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 55e9325,70477eb..79341cb --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@@ -18,7 -18,8 +18,9 @@@ package org.apache.kylin.cube; +import static com.google.common.base.Preconditions.checkNotNull; + import static org.apache.kylin.cube.cuboid.CuboidModeEnum.CURRENT; + import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND; import java.io.IOException; import java.util.List; @@@ -343,6 -332,6 +356,14 @@@ public class CubeInstance extends RootP } return null; } ++ ++ public CubeSegment[] regetSegments(CubeSegment... segs) { ++ CubeSegment[] r = new CubeSegment[segs.length]; ++ for (int i = 0; i < segs.length; i++) { ++ r[i] = getSegmentById(segs[i].getUuid()); ++ } ++ return r; ++ } public long getCreateTimeUTC() { return createTimeUTC; @@@ -520,6 -546,6 +578,36 @@@ } } ++ public long getSizeKB() { ++ long sizeKb = 0L; ++ ++ for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { ++ sizeKb += cubeSegment.getSizeKB(); ++ } ++ ++ return sizeKb; ++ } ++ ++ public long getInputRecordCount() { ++ long sizeRecordCount = 0L; ++ ++ for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { ++ sizeRecordCount += cubeSegment.getInputRecords(); ++ } ++ ++ return sizeRecordCount; ++ } ++ ++ public long getInputRecordSizeMB() { ++ long sizeRecordSize = 0L; ++ ++ for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) { ++ sizeRecordSize += cubeSegment.getInputRecordsSize(); ++ } ++ ++ return sizeRecordSize; ++ } ++ public String getProject() { return getDescriptor().getProject(); }