Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/88f85595 Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/88f85595 Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/88f85595 Branch: refs/heads/NewKafkaSystemConsumer Commit: 88f855954e4d98aa3bdd3a3a778eb699a13ff659 Parents: 0edf343 8ce1bd5 Author: Boris S <[email protected]> Authored: Thu Aug 2 13:29:58 2018 -0700 Committer: Boris S <[email protected]> Committed: Thu Aug 2 13:29:58 2018 -0700 ---------------------------------------------------------------------- .travis.yml | 6 +- README.md | 8 +- RELEASE.md | 20 + bin/integration-tests.sh | 17 +- build.gradle | 2 +- docs/community/committers.md | 5 + .../versioned/jobs/configuration-table.html | 20 + .../org/apache/samza/metrics/ListGauge.java | 143 +++++++ .../apache/samza/metrics/MetricsRegistry.java | 9 + .../apache/samza/metrics/MetricsVisitor.java | 7 +- .../ReadableMetricsRegistryListener.java | 2 + .../samza/operators/functions/MapFunction.java | 2 +- .../apache/samza/runtime/ApplicationRunner.java | 22 -- .../org/apache/samza/sql/SamzaSqlRelRecord.java | 113 ++++++ .../samza/storage/SideInputsProcessor.java | 46 +++ .../storage/SideInputsProcessorFactory.java | 45 +++ .../samza/system/ExtendedSystemAdmin.java | 6 +- .../samza/system/IncomingMessageEnvelope.java | 9 + .../org/apache/samza/system/StreamSpec.java | 56 +-- .../org/apache/samza/system/SystemAdmin.java | 31 ++ .../samza/table/TableDescriptorsProvider.java | 100 +++++ .../org/apache/samza/table/TableProvider.java | 3 +- .../java/org/apache/samza/table/TableSpec.java | 44 ++- .../apache/samza/util/NoOpMetricsRegistry.java | 7 + .../org/apache/samza/util/TimestampedValue.java | 61 +++ .../org/apache/samza/metrics/TestListGauge.java | 99 +++++ .../org/apache/samza/metrics/TestTimer.java | 5 +- .../apache/samza/sql/TestSamzaSqlRelRecord.java | 43 ++ .../apache/samza/system/TestSystemAdmin.java | 116 ++++++ .../samza/system/eventhub/EventHubConfig.java | 40 ++ .../consumer/EventHubSystemConsumer.java | 129 ++++-- .../MockEventHubClientManagerFactory.java | 4 + .../system/eventhub/TestMetricsRegistry.java | 16 +- .../consumer/TestEventHubSystemConsumer.java | 94 +++++ .../AbstractContainerAllocator.java | 5 + .../HostAwareContainerAllocator.java | 10 + .../clustermanager/SamzaApplicationState.java | 8 + .../samza/config/InMemorySystemConfig.java | 52 +++ .../apache/samza/config/JavaStorageConfig.java | 47 +++ .../org/apache/samza/config/TaskConfigJava.java | 3 +- .../samza/container/SamzaContainerListener.java | 9 +- .../apache/samza/container/TaskContextImpl.java | 15 +- .../samza/execution/ExecutionPlanner.java | 22 +- .../org/apache/samza/execution/JobGraph.java | 18 +- .../samza/execution/JobGraphJsonGenerator.java | 12 +- .../org/apache/samza/execution/JobNode.java | 67 ++-- .../org/apache/samza/execution/StreamEdge.java | 34 +- .../apache/samza/execution/StreamManager.java | 19 +- .../executors/KeyBasedExecutorService.java | 174 +++++++++ .../org/apache/samza/metrics/MetricGroup.java | 4 + .../samza/operators/OperatorSpecGraph.java | 15 +- .../apache/samza/operators/StreamGraphSpec.java | 79 ++-- .../functions/PartialJoinFunction.java | 2 +- .../operators/impl/BroadcastOperatorImpl.java | 4 +- .../samza/operators/impl/OperatorImplGraph.java | 64 +-- .../operators/impl/OutputOperatorImpl.java | 4 +- .../operators/impl/PartialJoinOperatorImpl.java | 2 +- .../operators/impl/PartitionByOperatorImpl.java | 11 +- .../operators/impl/WindowOperatorImpl.java | 2 +- .../operators/impl/store/TimeSeriesStore.java | 2 + .../impl/store/TimeSeriesStoreImpl.java | 1 + .../operators/impl/store/TimestampedValue.java | 61 --- .../impl/store/TimestampedValueSerde.java | 1 + .../samza/operators/spec/InputOperatorSpec.java | 12 +- .../samza/operators/spec/JoinOperatorSpec.java | 2 +- .../samza/operators/spec/OperatorSpecs.java | 7 +- .../samza/operators/spec/OutputStreamImpl.java | 18 +- .../stream/IntermediateMessageStreamImpl.java | 7 +- .../apache/samza/processor/StreamProcessor.java | 281 ++++++++----- .../runtime/AbstractApplicationRunner.java | 111 ++---- .../samza/runtime/ApplicationRunnerMain.java | 10 - .../samza/runtime/LocalApplicationRunner.java | 36 +- .../samza/runtime/LocalContainerRunner.java | 150 +++---- .../samza/runtime/RemoteApplicationRunner.java | 34 +- .../standalone/PassthroughJobCoordinator.java | 4 + .../samza/storage/ChangelogStreamManager.java | 4 +- .../samza/storage/StorageManagerUtil.java | 142 +++++++ .../apache/samza/storage/StorageRecovery.java | 14 +- .../storage/TaskSideInputStorageManager.java | 375 ++++++++++++++++++ .../system/inmemory/InMemorySystemFactory.java | 15 +- .../samza/table/TableConfigGenerator.java | 143 +++++++ .../samza/table/caching/CachingTable.java | 39 +- .../table/caching/CachingTableProvider.java | 4 +- .../table/caching/guava/GuavaCacheTable.java | 12 +- .../caching/guava/GuavaCacheTableProvider.java | 1 + .../table/remote/RemoteReadWriteTable.java | 39 +- .../samza/table/remote/RemoteReadableTable.java | 20 +- .../table/utils/DefaultTableReadMetrics.java | 55 +++ .../table/utils/DefaultTableWriteMetrics.java | 63 +++ .../samza/table/utils/TableMetricsUtil.java | 101 +++++ .../apache/samza/task/SystemTimerScheduler.java | 1 + .../org/apache/samza/testUtils/TestClock.java | 45 +++ .../java/org/apache/samza/util/StreamUtil.java | 90 +++++ .../samza/zk/ZkBarrierForVersionUpgrade.java | 19 +- .../java/org/apache/samza/zk/ZkController.java | 39 -- .../org/apache/samza/zk/ZkControllerImpl.java | 163 -------- .../apache/samza/zk/ZkControllerListener.java | 37 -- .../org/apache/samza/zk/ZkJobCoordinator.java | 172 +++++--- .../org/apache/samza/zk/ZkLeaderElector.java | 17 +- .../main/java/org/apache/samza/zk/ZkUtils.java | 72 ++-- .../apache/samza/checkpoint/OffsetManager.scala | 6 +- .../org/apache/samza/config/JobConfig.scala | 13 + .../org/apache/samza/config/MetricsConfig.scala | 3 + .../org/apache/samza/config/StorageConfig.scala | 17 +- .../org/apache/samza/config/StreamConfig.scala | 2 +- .../org/apache/samza/config/TaskConfig.scala | 8 +- .../org/apache/samza/container/RunLoop.scala | 5 +- .../apache/samza/container/SamzaContainer.scala | 142 +++++-- .../samza/container/SamzaContainerMetrics.scala | 3 + .../apache/samza/container/TaskInstance.scala | 140 +++++-- .../diagnostics/DiagnosticsExceptionEvent.java | 90 +++++ .../samza/job/local/ThreadJobFactory.scala | 6 +- .../ContainerProcessManagerMetrics.scala | 22 +- .../apache/samza/metrics/MetricsHelper.scala | 6 +- .../samza/metrics/MetricsRegistryMap.scala | 15 + .../samza/metrics/reporter/JmxReporter.scala | 29 +- .../apache/samza/metrics/reporter/Metrics.scala | 9 +- .../samza/metrics/reporter/MetricsHeader.scala | 3 + .../reporter/MetricsSnapshotReporter.scala | 67 +++- .../MetricsSnapshotReporterFactory.scala | 11 +- .../serializers/MetricsSnapshotSerdeV2.java | 75 ++++ .../MetricsSnapshotSerdeV2Factory.java | 31 ++ .../samza/storage/TaskStorageManager.scala | 117 +----- .../apache/samza/system/SSPMetadataCache.java | 126 ++++++ .../system/chooser/BootstrappingChooser.scala | 35 +- .../scala/org/apache/samza/util/FileUtil.scala | 21 +- .../org/apache/samza/util/ScalaJavaUtil.scala | 12 + .../main/scala/org/apache/samza/util/Util.scala | 22 -- .../TestHostAwareContainerAllocator.java | 10 +- .../MockCoordinatorStreamSystemFactory.java | 8 +- .../samza/execution/TestExecutionPlanner.java | 42 +- .../apache/samza/execution/TestJobGraph.java | 28 +- .../execution/TestJobGraphJsonGenerator.java | 45 +-- .../org/apache/samza/execution/TestJobNode.java | 17 +- .../apache/samza/execution/TestStreamEdge.java | 16 +- .../executors/TestKeyBasedExecutorService.java | 84 ++++ .../metrics/TestMetricsSnapshotReporter.java | 115 ++++++ .../samza/operators/TestJoinOperator.java | 11 +- .../samza/operators/TestOperatorSpecGraph.java | 23 +- .../samza/operators/TestStreamGraphSpec.java | 336 ++++++---------- .../operators/impl/TestOperatorImplGraph.java | 274 +++++++------ .../operators/impl/TestWindowOperator.java | 16 +- .../impl/store/TestTimeSeriesStoreImpl.java | 1 + .../impl/store/TestTimestampedValueSerde.java | 1 + .../operators/spec/OperatorSpecTestUtils.java | 14 +- .../samza/operators/spec/TestOperatorSpec.java | 18 +- .../spec/TestPartitionByOperatorSpec.java | 14 +- .../samza/processor/TestStreamProcessor.java | 162 +++++++- .../runtime/TestAbstractApplicationRunner.java | 391 ------------------- .../runtime/TestApplicationRunnerMain.java | 11 +- .../runtime/TestLocalApplicationRunner.java | 22 +- .../serializers/TestMetricsSnapshotSerdeV2.java | 69 ++++ .../samza/system/TestSSPMetadataCache.java | 319 +++++++++++++++ .../samza/table/caching/TestCachingTable.java | 18 +- .../org/apache/samza/task/TestAsyncRunLoop.java | 2 +- .../apache/samza/task/TestTaskFactoryUtil.java | 5 +- .../apache/samza/testUtils/StreamTestUtils.java | 39 ++ .../org/apache/samza/testUtils/TestClock.java | 45 --- .../org/apache/samza/util/TestStreamUtil.java | 337 ++++++++++++++++ .../apache/samza/zk/TestZkJobCoordinator.java | 19 +- .../java/org/apache/samza/zk/TestZkUtils.java | 21 +- .../factories/TestPropertiesConfigFactory.scala | 3 +- .../samza/container/TestSamzaContainer.scala | 44 ++- .../samza/container/TestTaskInstance.scala | 5 +- .../org/apache/samza/job/TestJobRunner.scala | 11 +- .../serializers/TestMetricsSnapshotSerde.scala | 3 +- .../samza/storage/TestTaskStorageManager.scala | 154 ++++---- .../org/apache/samza/util/TestFileUtil.scala | 22 ++ .../hdfs/TestHdfsSystemProducerTestSuite.scala | 5 +- .../samza/system/kafka/KafkaStreamSpec.java | 15 +- .../org/apache/samza/config/KafkaConfig.scala | 4 +- .../samza/config/RegExTopicGenerator.scala | 8 +- .../apache/samza/system/kafka/BrokerProxy.scala | 8 +- .../samza/system/kafka/KafkaSystemAdmin.scala | 4 +- .../system/kafka/KafkaSystemConsumer.scala | 25 +- .../kafka/KafkaSystemConsumerMetrics.scala | 6 +- .../util/ClientUtilTopicMetadataStore.scala | 3 + .../scala/org/apache/samza/util/KafkaUtil.scala | 6 +- .../samza/system/kafka/TestKafkaStreamSpec.java | 6 +- .../kafka/TestKafkaCheckpointManager.scala | 4 +- .../system/kafka/TestKafkaSystemAdmin.scala | 2 +- .../system/kafka/TestKafkaSystemConsumer.scala | 2 +- .../kv/inmemory/InMemoryTableDescriptor.java | 3 +- .../storage/kv/RocksDbTableDescriptor.java | 3 +- .../samza/storage/kv/RocksDbKeyValueStore.scala | 9 +- .../kv/BaseLocalStoreBackedTableDescriptor.java | 21 + .../kv/BaseLocalStoreBackedTableProvider.java | 12 + .../kv/LocalStoreBackedReadWriteTable.java | 33 +- .../kv/LocalStoreBackedReadableTable.java | 33 +- .../TestLocalBaseStoreBackedTableProvider.java | 6 +- .../log4j/SimpleDiagnosticsAppender.java | 101 +++++ .../apache/samza/rest/SamzaRestApplication.java | 1 - .../org/apache/samza/rest/SamzaRestService.java | 15 +- .../apache/samza/sql/avro/AvroRelConverter.java | 17 +- .../samza/sql/data/SamzaSqlRelMessage.java | 85 +--- .../sql/runner/SamzaSqlApplicationRunner.java | 2 - .../SamzaSqlRelMessageSerdeFactory.java | 2 +- .../SamzaSqlRelRecordSerdeFactory.java | 18 +- .../samza/sql/TestSamzaSqlRelMessageSerde.java | 102 ----- .../samza/sql/TestSamzaSqlRelRecordSerde.java | 86 ---- .../samza/sql/avro/TestAvroRelConversion.java | 17 +- .../samza/sql/avro/schemas/ComplexRecord.java | 2 +- .../apache/samza/sql/avro/schemas/MyFixed.java | 5 +- .../samza/sql/data/TestSamzaSqlRelMessage.java | 18 + .../TestSamzaSqlRelMessageSerde.java | 102 +++++ .../serializers/TestSamzaSqlRelRecordSerde.java | 85 ++++ .../samza/sql/system/TestAvroSystemFactory.java | 9 + .../sql/translator/TestQueryTranslator.java | 235 ++++++----- .../config/standalone.failure.test.properties | 45 +++ .../test/framework/MessageStreamAssert.java | 192 +++++++++ .../samza/test/framework/StreamAssert.java | 220 ++++------- .../apache/samza/test/framework/TestRunner.java | 367 +++++++++++++++++ .../test/framework/stream/CollectionStream.java | 204 ++++++++++ .../system/CollectionStreamSystemSpec.java | 79 ++++ .../integration/LocalApplicationRunnerMain.java | 63 +++ .../test/integration/NegateNumberTask.java | 4 +- .../TestStandaloneIntegrationApplication.java | 42 ++ samza-test/src/main/python/configs/kafka.json | 2 +- samza-test/src/main/python/deployment.py | 4 +- samza-test/src/main/python/requirements.txt | 1 + .../src/main/python/standalone_deployment.py | 123 ++++++ .../main/python/standalone_integration_tests.py | 29 ++ samza-test/src/main/python/stream_processor.py | 121 ++++++ .../python/tests/standalone_failure_tests.py | 311 +++++++++++++++ samza-test/src/main/python/tests/zk_client.py | 129 ++++++ .../test/performance/TestPerformanceTask.scala | 4 +- .../samza/processor/TestZkStreamProcessor.java | 4 +- .../TestZkStreamProcessorFailures.java | 4 +- .../processor/TestZkStreamProcessorSession.java | 4 +- .../AsyncStreamTaskIntegrationTest.java | 144 +++++++ .../test/framework/BroadcastAssertApp.java | 58 +++ .../samza/test/framework/MyAsyncStreamTask.java | 67 ++++ .../samza/test/framework/MyStreamTestTask.java | 38 ++ .../StreamApplicationIntegrationTest.java | 132 +++++++ ...StreamApplicationIntegrationTestHarness.java | 302 ++++++++++++++ .../framework/StreamTaskIntegrationTest.java | 138 +++++++ .../samza/test/framework/TestTimerApp.java | 86 ++++ .../apache/samza/test/framework/TimerTest.java | 50 +++ .../samza/test/operator/BroadcastAssertApp.java | 59 --- .../test/operator/RepartitionJoinWindowApp.java | 13 +- ...StreamApplicationIntegrationTestHarness.java | 277 ------------- .../operator/TestRepartitionJoinWindowApp.java | 13 +- .../test/operator/TestRepartitionWindowApp.java | 1 + .../processor/TestZkLocalApplicationRunner.java | 145 +++++-- .../test/samzasql/TestSamzaSqlEndToEnd.java | 5 +- .../table/TestTableDescriptorsProvider.java | 164 ++++++++ .../apache/samza/test/timer/TestTimerApp.java | 87 ----- .../org/apache/samza/test/timer/TimerTest.java | 51 --- .../tools/json/JsonRelConverterFactory.java | 14 +- .../org/apache/samza/config/YarnConfig.java | 56 ++- .../job/yarn/YarnClusterResourceManager.java | 7 + .../apache/samza/job/yarn/ClientHelper.scala | 7 +- .../webapp/ApplicationMasterRestServlet.scala | 3 + 253 files changed, 9637 insertions(+), 3594 deletions(-) ----------------------------------------------------------------------
