Merge branch 'master' into NewConsumer
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/c14557fb Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/c14557fb Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/c14557fb Branch: refs/heads/NewKafkaSystemConsumer Commit: c14557fb401f0d718e78163a159695f50bc82845 Parents: 89f7982 8ab04b2 Author: Boris S <[email protected]> Authored: Thu Aug 30 12:45:23 2018 -0700 Committer: Boris S <[email protected]> Committed: Thu Aug 30 12:45:23 2018 -0700 ---------------------------------------------------------------------- build.gradle | 5 +- .../samza/metadatastore/MetadataStore.java | 80 ++++ .../metadatastore/MetadataStoreFactory.java | 30 ++ .../apache/samza/operators/MessageStream.java | 12 +- .../org/apache/samza/operators/StreamGraph.java | 109 ++---- .../descriptors/GenericInputDescriptor.java | 43 +++ .../descriptors/GenericOutputDescriptor.java | 43 +++ .../descriptors/GenericSystemDescriptor.java | 67 ++++ .../base/stream/InputDescriptor.java | 187 +++++++++ .../base/stream/OutputDescriptor.java | 44 +++ .../base/stream/StreamDescriptor.java | 136 +++++++ .../ExpandingInputDescriptorProvider.java | 44 +++ .../base/system/OutputDescriptorProvider.java | 48 +++ .../system/SimpleInputDescriptorProvider.java | 43 +++ .../base/system/SystemDescriptor.java | 177 +++++++++ .../TransformingInputDescriptorProvider.java | 44 +++ .../operators/functions/InputTransformer.java | 45 +++ .../operators/functions/StreamExpander.java | 58 +++ .../org/apache/samza/table/TableProvider.java | 12 +- .../TestExpandingInputDescriptor.java | 61 +++ .../descriptors/TestGenericInputDescriptor.java | 123 ++++++ .../TestGenericSystemDescriptor.java | 63 +++ .../descriptors/TestSimpleInputDescriptor.java | 65 ++++ .../TestTransformingInputDescriptor.java | 66 ++++ .../ExampleExpandingInputDescriptor.java | 30 ++ .../ExampleExpandingOutputDescriptor.java | 29 ++ .../ExampleExpandingSystemDescriptor.java | 49 +++ .../serde/ExampleSimpleInputDescriptor.java | 30 ++ .../serde/ExampleSimpleOutputDescriptor.java | 29 ++ .../serde/ExampleSimpleSystemDescriptor.java | 43 +++ .../ExampleTransformingInputDescriptor.java | 30 ++ .../ExampleTransformingOutputDescriptor.java | 29 ++ .../ExampleTransformingSystemDescriptor.java | 43 +++ .../apache/samza/container/LocalityManager.java | 126 +++--- .../grouper/task/TaskAssignmentManager.java | 109 ++++-- .../CoordinatorStreamMetadataStoreFactory.java | 36 ++ .../metadatastore/CoordinatorStreamStore.java | 188 +++++++++ .../stream/CoordinatorStreamKeySerde.java | 52 +++ .../stream/CoordinatorStreamValueSerde.java | 80 ++++ .../samza/execution/JobGraphJsonGenerator.java | 4 +- .../org/apache/samza/execution/JobNode.java | 23 +- .../samza/operators/BaseTableDescriptor.java | 2 +- .../samza/operators/MessageStreamImpl.java | 2 +- .../apache/samza/operators/StreamGraphSpec.java | 152 +++++--- .../descriptors/DelegatingSystemDescriptor.java | 70 ++++ .../samza/operators/impl/InputOperatorImpl.java | 23 +- .../samza/operators/spec/InputOperatorSpec.java | 57 ++- .../samza/operators/spec/OperatorSpecs.java | 11 +- .../samza/operators/spec/OutputStreamImpl.java | 12 + .../stream/IntermediateMessageStreamImpl.java | 2 +- .../apache/samza/processor/StreamProcessor.java | 2 +- .../runtime/AbstractApplicationRunner.java | 12 + .../samza/table/TableConfigGenerator.java | 11 +- .../table/caching/CachingTableDescriptor.java | 3 +- .../table/caching/CachingTableProvider.java | 56 +-- .../guava/GuavaCacheTableDescriptor.java | 3 +- .../caching/guava/GuavaCacheTableProvider.java | 44 +-- .../samza/table/remote/RemoteReadableTable.java | 5 + .../table/remote/RemoteTableDescriptor.java | 3 +- .../samza/table/remote/RemoteTableProvider.java | 43 +-- .../samza/table/utils/BaseTableProvider.java | 76 ++++ .../apache/samza/task/StreamOperatorTask.java | 3 +- .../org/apache/samza/task/TaskCallbackImpl.java | 2 +- .../java/org/apache/samza/util/StreamUtil.java | 1 + .../org/apache/samza/config/JobConfig.scala | 4 + .../org/apache/samza/config/StorageConfig.scala | 1 + .../apache/samza/container/SamzaContainer.scala | 28 +- .../samza/coordinator/JobModelManager.scala | 20 +- .../TestClusterBasedJobCoordinator.java | 26 +- .../samza/container/TestLocalityManager.java | 106 ++--- .../grouper/task/TestTaskAssignmentManager.java | 132 ++----- .../coordinator/JobModelManagerTestUtil.java | 4 +- .../TestCoordinatorStreamStore.java | 129 +++++++ .../samza/execution/TestExecutionPlanner.java | 92 +++-- .../execution/TestJobGraphJsonGenerator.java | 38 +- .../org/apache/samza/execution/TestJobNode.java | 81 +++- .../samza/operators/TestJoinOperator.java | 50 ++- .../samza/operators/TestMessageStreamImpl.java | 4 +- .../samza/operators/TestOperatorSpecGraph.java | 4 +- .../samza/operators/TestStreamGraphSpec.java | 387 +++++++++---------- .../operators/impl/TestInputOperatorImpl.java | 80 ++++ .../operators/impl/TestOperatorImplGraph.java | 96 +++-- .../operators/impl/TestWindowOperator.java | 24 +- .../samza/operators/spec/TestOperatorSpec.java | 16 +- .../spec/TestPartitionByOperatorSpec.java | 63 ++- .../scala/org/apache/samza/util/TestUtil.scala | 2 - .../system/kafka/KafkaInputDescriptor.java | 108 ++++++ .../system/kafka/KafkaOutputDescriptor.java | 39 ++ .../system/kafka/KafkaSystemDescriptor.java | 251 ++++++++++++ .../apache/samza/system/kafka/GetOffset.scala | 3 +- .../system/kafka/TestKafkaInputDescriptor.java | 68 ++++ .../system/kafka/TestKafkaSystemDescriptor.java | 69 ++++ .../samza/system/kafka/TestGetOffset.scala | 31 +- .../kv/inmemory/InMemoryTableDescriptor.java | 3 + .../kv/inmemory/InMemoryTableProvider.java | 21 +- .../kv/inmemory/TestInMemoryTableProvider.java | 9 +- .../storage/kv/RocksDbTableDescriptor.java | 7 +- .../samza/storage/kv/RocksDbTableProvider.java | 21 +- .../storage/kv/TestRocksDbTableDescriptor.java | 15 + .../storage/kv/TestRocksDbTableProvider.java | 9 +- .../kv/BaseLocalStoreBackedTableDescriptor.java | 68 ++++ .../kv/BaseLocalStoreBackedTableProvider.java | 60 ++- .../kv/LocalStoreBackedReadableTable.java | 1 + .../TestBaseLocalStoreBackedTableProvider.java | 149 +++++++ .../TestLocalBaseStoreBackedTableProvider.java | 85 ---- .../samza/rest/proxy/task/SamzaTaskProxy.java | 4 +- .../apache/samza/sql/planner/QueryPlanner.java | 8 - .../sql/planner/SamzaSqlOperatorTable.java | 1 + .../samza/sql/translator/QueryTranslator.java | 10 +- .../samza/sql/translator/ScanTranslator.java | 13 +- .../samza/sql/translator/TranslatorContext.java | 34 +- .../sql/testutil/TestIOResolverFactory.java | 15 +- .../sql/translator/TestJoinTranslator.java | 2 +- .../sql/translator/TestQueryTranslator.java | 5 +- .../example/AppWithGlobalConfigExample.java | 17 +- .../apache/samza/example/BroadcastExample.java | 24 +- .../samza/example/KeyValueStoreExample.java | 27 +- .../org/apache/samza/example/MergeExample.java | 25 +- .../samza/example/OrderShipmentJoinExample.java | 29 +- .../samza/example/PageViewCounterExample.java | 17 +- .../samza/example/RepartitionExample.java | 24 +- .../org/apache/samza/example/WindowExample.java | 19 +- .../apache/samza/test/framework/TestRunner.java | 2 +- .../system/CollectionStreamSystemSpec.java | 25 +- .../TestStandaloneIntegrationApplication.java | 21 +- .../EndOfStreamIntegrationTest.java | 11 +- .../WatermarkIntegrationTest.java | 12 +- .../test/framework/BroadcastAssertApp.java | 11 +- .../StreamApplicationIntegrationTest.java | 29 +- .../samza/test/framework/TestTimerApp.java | 17 +- .../test/operator/RepartitionJoinWindowApp.java | 32 +- .../test/operator/RepartitionWindowApp.java | 19 +- .../samza/test/operator/SessionWindowApp.java | 19 +- .../operator/TestRepartitionJoinWindowApp.java | 12 +- .../test/operator/TestRepartitionWindowApp.java | 6 +- .../samza/test/operator/TumblingWindowApp.java | 19 +- .../test/processor/TestStreamApplication.java | 20 +- .../processor/TestZkLocalApplicationRunner.java | 60 ++- .../test/samzasql/TestSamzaSqlEndToEnd.java | 2 +- .../table/PageViewToProfileJoinFunction.java | 48 +++ .../apache/samza/test/table/TestLocalTable.java | 75 +--- .../table/TestLocalTableWithSideInputs.java | 47 ++- .../samza/test/table/TestRemoteTable.java | 50 +-- .../table/TestTableDescriptorsProvider.java | 10 +- .../benchmark/SystemConsumerWithSamzaBench.java | 11 +- 145 files changed, 4990 insertions(+), 1376 deletions(-) ----------------------------------------------------------------------
