This is an automated email from the ASF dual-hosted git repository. nic pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 19706b2 KYLIN-4010 Support date format "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" new 92273d4 KYLIN-4010 Auto adjust offset according to query server's timezone for time derived column new 4f3d8e7 KYLIN-4141 Build Global Dictionary in no time new a6fc3e7 KYLIN-4167 Clarify overall design for realtime OLAP new e8ad658 KYLIN-4167 Refactor streaming coordinator Phase1 new 80384da KYLIN-4167 Fix minor bug The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: build/conf/kylin-server-log4j.properties | 2 +- .../org/apache/kylin/common/KylinConfigBase.java | 18 + .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../kylin/job/execution/AbstractExecutable.java | 2 +- .../kylin/dimension}/TimeDerivedColumnType.java | 2 +- .../dimension}/TimeDerivedColumnTypeTest.java | 2 +- .../kylin/storage/gtrecord/CubeTupleConverter.java | 32 +- .../kylin/engine/mr/StreamingCubingEngine.java | 5 +- .../apache/kylin/query/relnode/OLAPFilterRel.java | 5 + .../query/relnode/visitor/TupleFilterVisitor.java | 24 +- .../relnode/visitor/TupleFilterVisitorTest.java | 9 +- .../controller/StreamingCoordinatorController.java | 8 +- .../rest/service/StreamingCoordinatorService.java | 27 +- .../kylin/rest/service/StreamingV2Service.java | 40 +- .../stream/rpc/HttpStreamDataSearchClient.java | 5 +- .../kylin/stream/coordinator/Coordinator.java | 3 +- .../stream/coordinator/StreamMetadataStore.java | 11 +- .../coordinator/ZookeeperStreamMetadataStore.java | 66 +- .../coordinator/client/CoordinatorClient.java | 15 + .../client/CoordinatorClientFactory.java | 17 +- .../coordinator/coordinate/BuildJobSubmitter.java | 464 +++++++++++++ .../coordinate/ReceiverClusterManager.java | 651 +++++++++++++++++++ .../coordinate/SegmentJobBuildInfo.java | 73 +++ .../coordinate/StreamingCoordinator.java | 644 ++++++++++++++++++ .../coordinate/annotations/NonSideEffect.java | 40 +- .../annotations/NotAtomicAndNotIdempotent.java | 45 ++ .../annotations/NotAtomicIdempotent.java | 44 +- .../stream/coordinator/doctor/ClusterDoctor.java | 16 +- .../coordinator/doctor/ClusterStateChecker.java | 23 +- .../exception/ClusterStateException.java | 6 +- .../coordinator/exception/CoordinateException.java | 3 + .../exception/NotLeadCoordinatorException.java | 2 + .../coordinator/exception/StoreException.java | 2 + .../src/main/resources/log4j.properties | 15 +- .../coordinate/BuildJobSubmitterTest.java | 194 ++++++ .../coordinator/coordinate/StreamingTestBase.java | 335 ++++++++++ stream-core/pom.xml | 5 + .../stream/core/client/ReceiverAdminClient.java | 76 +++ .../core/consumer/StreamingConsumerChannel.java | 12 + .../stream/core/dict/LocalDictionaryStore.java | 144 +++++ .../stream/core/dict/RemoteDictionaryStore.java | 215 ++++++ .../core/dict/StreamingDictionaryClient.java | 216 +++++++ .../core/dict/StreamingDistributedDictionary.java | 91 +++ .../kylin/stream/core/model/SegmentBuildState.java | 2 +- .../stream/core/model/stats/ClusterState.java | 8 + .../stream/core/model/stats/ReplicaSetState.java | 3 + .../core/query/StreamingDataQueryPlanner.java | 2 +- .../stream/core/query/StreamingTupleConverter.java | 29 +- .../core/storage/IStreamingSegmentStore.java | 5 + .../core/storage/StreamingSegmentManager.java | 61 +- .../storage/columnar/ColumnarSegmentStore.java | 14 +- .../core/storage/columnar/ColumnarStoreCache.java | 16 +- .../storage/columnar/ColumnarStoreDimDesc.java | 2 +- .../storage/columnar/FragmentFileSearcher.java | 718 ++++++++++----------- .../core/storage/columnar/SegmentMemoryStore.java | 12 +- .../columnar/TimeDerivedColumnEncoding.java | 2 +- .../core/util/CompareFilterTimeRangeChecker.java | 1 + .../kylin/stream/server/StreamingServer.java | 19 +- .../server/rest/controller/AdminController.java | 10 +- .../stream/source/kafka/TimedJsonStreamParser.java | 2 +- 60 files changed, 3957 insertions(+), 560 deletions(-) rename {stream-core/src/main/java/org/apache/kylin/stream/core/util => core-metadata/src/main/java/org/apache/kylin/dimension}/TimeDerivedColumnType.java (99%) rename {stream-core/src/test/java/org/apache/kylin/stream/core/util => core-metadata/src/test/java/org/apache/kylin/dimension}/TimeDerivedColumnTypeTest.java (99%) create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitter.java create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/ReceiverClusterManager.java create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/SegmentJobBuildInfo.java create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/StreamingCoordinator.java copy core-common/src/main/java/org/apache/kylin/common/util/ByteBufferOutputStream.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NonSideEffect.java (54%) create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NotAtomicAndNotIdempotent.java copy core-common/src/main/java/org/apache/kylin/common/util/SetThreadName.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/coordinate/annotations/NotAtomicIdempotent.java (50%) copy core-job/src/main/java/org/apache/kylin/job/lock/MockJobLock.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/doctor/ClusterDoctor.java (70%) copy core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryValueEnumerator.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/doctor/ClusterStateChecker.java (62%) copy dev-support/test_all_against_hdp_2_4_0_0_169.sh => stream-coordinator/src/main/resources/log4j.properties (69%) mode change 100755 => 100644 create mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/coordinate/BuildJobSubmitterTest.java create mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/coordinate/StreamingTestBase.java create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/LocalDictionaryStore.java create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/RemoteDictionaryStore.java create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/StreamingDictionaryClient.java create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/dict/StreamingDistributedDictionary.java