Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/0edf343b Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/0edf343b Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/0edf343b Branch: refs/heads/NewKafkaSystemConsumer Commit: 0edf343b8c5360437fa10cc5a543696a4edcc319 Parents: 67e611e f249e71 Author: Boris S <[email protected]> Authored: Fri Jun 8 10:15:36 2018 -0700 Committer: Boris S <[email protected]> Committed: Fri Jun 8 10:15:36 2018 -0700 ---------------------------------------------------------------------- .travis.yml | 3 +- README.md | 4 +- bin/check-all.sh | 2 +- bin/integration-tests.sh | 3 +- build.gradle | 24 +- docs/Gemfile | 7 +- docs/Gemfile.lock | 95 ++- docs/README.md | 4 +- docs/community/committers.md | 8 +- .../documentation/versioned/azure/eventhubs.md | 6 +- .../versioned/container/metrics-table.html | 8 + .../versioned/jobs/configuration-table.html | 91 ++- .../documentation/versioned/jobs/logging.md | 2 +- .../documentation/versioned/rest/monitors.md | 8 + .../versioned/deploy-samza-job-from-hdfs.md | 2 +- .../tutorials/versioned/deploy-samza-to-CDH.md | 4 +- .../versioned/hello-samza-high-level-yarn.md | 2 +- .../versioned/hello-samza-high-level-zk.md | 2 +- .../versioned/remote-debugging-samza.md | 2 +- .../versioned/run-in-multi-node-yarn.md | 4 +- .../versioned/samza-rest-getting-started.md | 2 +- docs/startup/download/index.md | 19 +- docs/startup/hello-samza/versioned/index.md | 2 +- gradle.properties | 2 +- gradle/buildscript.gradle | 2 +- gradle/dependency-versions-scala-2.10.gradle | 2 +- gradle/dependency-versions-scala-2.11.gradle | 2 +- gradle/dependency-versions-scala-2.12.gradle | 2 +- gradle/dependency-versions.gradle | 7 +- gradle/release.gradle | 11 +- .../samza/application/StreamApplication.java | 7 +- .../samza/checkpoint/CheckpointManager.java | 12 +- .../java/org/apache/samza/config/MapConfig.java | 9 +- .../apache/samza/operators/MessageStream.java | 34 +- .../apache/samza/operators/TimerRegistry.java | 41 ++ .../operators/functions/ClosableFunction.java | 3 + .../operators/functions/FilterFunction.java | 3 +- .../operators/functions/FlatMapFunction.java | 3 +- .../operators/functions/FoldLeftFunction.java | 16 +- .../operators/functions/InitableFunction.java | 3 +- .../samza/operators/functions/JoinFunction.java | 3 +- .../samza/operators/functions/MapFunction.java | 3 +- .../samza/operators/functions/SinkFunction.java | 3 +- .../functions/StreamTableJoinFunction.java | 3 +- .../operators/functions/SupplierFunction.java | 38 ++ .../operators/functions/TimerFunction.java | 65 ++ .../samza/operators/triggers/AnyTrigger.java | 10 +- .../samza/operators/triggers/Trigger.java | 3 +- .../apache/samza/operators/windows/Window.java | 3 +- .../apache/samza/operators/windows/Windows.java | 53 +- .../windows/internal/WindowInternal.java | 32 +- .../apache/samza/runtime/ApplicationRunner.java | 19 + .../samza/serializers/SerializableSerde.java | 2 +- .../org/apache/samza/sql/udfs/ScalarUdf.java | 4 +- .../samza/storage/kv/KeyValueSnapshot.java | 42 ++ .../apache/samza/storage/kv/KeyValueStore.java | 14 + .../org/apache/samza/system/StreamSpec.java | 39 +- .../org/apache/samza/system/SystemAdmin.java | 23 +- .../samza/system/SystemStreamPartition.java | 4 +- .../table/LocalStoreBackedTableProvider.java | 37 -- .../org/apache/samza/table/ReadWriteTable.java | 9 +- .../org/apache/samza/table/ReadableTable.java | 11 +- .../org/apache/samza/table/TableProvider.java | 18 +- .../java/org/apache/samza/table/TableSpec.java | 12 +- .../java/org/apache/samza/task/TaskContext.java | 19 + .../org/apache/samza/task/TimerCallback.java | 34 ++ .../java/org/apache/samza/util/RateLimiter.java | 108 ++++ .../samza/operators/windows/TestWindowPane.java | 2 +- .../autoscaling/deployer/ConfigManager.java | 4 +- .../samza/system/kinesis/KinesisConfig.java | 28 +- .../samza/coordinator/AzureJobCoordinator.java | 93 +-- .../samza/coordinator/data/ProcessorEntity.java | 8 + .../scheduler/JMVersionUpgradeScheduler.java | 9 +- .../eventhub/EventHubClientManagerFactory.java | 3 +- .../samza/system/eventhub/EventHubConfig.java | 117 +++- .../eventhub/SamzaEventHubClientManager.java | 39 +- .../eventhub/admin/EventHubSystemAdmin.java | 138 +++-- .../consumer/EventHubSystemConsumer.java | 304 ++++++---- .../system/eventhub/metrics/SamzaHistogram.java | 69 ++- .../eventhub/producer/AsyncSystemProducer.java | 218 +++++++ .../producer/EventHubSystemProducer.java | 378 +++++++----- .../java/org/apache/samza/util/TableUtils.java | 49 +- .../samza/system/eventhub/MockEventData.java | 21 +- .../MockEventHubClientManagerFactory.java | 25 +- .../eventhub/MockEventHubConfigFactory.java | 18 +- .../system/eventhub/TestMetricsRegistry.java | 7 +- .../consumer/TestEventHubSystemConsumer.java | 75 ++- .../producer/ITestEventHubSystemProducer.java | 10 +- .../producer/TestEventHubSystemProducer.java | 232 +++++-- samza-azure/src/test/resources/log4j.xml | 43 ++ .../AbstractContainerAllocator.java | 2 +- .../ClusterBasedJobCoordinator.java | 78 ++- .../clustermanager/ContainerProcessManager.java | 14 +- .../HostAwareContainerAllocator.java | 17 +- .../clustermanager/ResourceRequestState.java | 53 +- .../apache/samza/config/JavaStorageConfig.java | 18 +- .../apache/samza/config/JavaSystemConfig.java | 12 +- .../apache/samza/config/JavaTableConfig.java | 2 +- .../samza/config/JobCoordinatorConfig.java | 4 +- .../org/apache/samza/config/TaskConfigJava.java | 35 +- .../container/ContainerHeartbeatClient.java | 4 +- .../container/ContainerHeartbeatMonitor.java | 3 + .../apache/samza/container/LocalityManager.java | 94 ++- .../apache/samza/container/RunLoopFactory.java | 25 +- .../SamzaContainerExceptionHandler.java | 57 -- .../apache/samza/container/TaskContextImpl.java | 31 +- .../disk/PollingScanDiskSpaceMonitor.java | 20 +- .../grouper/task/TaskAssignmentManager.java | 37 +- .../container/host/StatisticsMonitorImpl.java | 20 +- .../StreamPartitionCountMonitor.java | 10 +- .../AbstractCoordinatorStreamManager.java | 132 ---- .../stream/CoordinatorStreamManager.java | 170 ++++++ .../stream/CoordinatorStreamSystemConsumer.java | 32 +- .../stream/CoordinatorStreamSystemProducer.java | 37 +- .../stream/CoordinatorStreamWriter.java | 2 +- .../samza/execution/ExecutionPlanner.java | 30 +- .../org/apache/samza/execution/JobGraph.java | 17 +- .../samza/execution/JobGraphJsonGenerator.java | 4 +- .../org/apache/samza/execution/JobNode.java | 64 +- .../org/apache/samza/execution/StreamEdge.java | 1 + .../apache/samza/execution/StreamManager.java | 32 +- .../org/apache/samza/job/model/JobModel.java | 4 +- .../samza/operators/MessageStreamImpl.java | 89 +-- .../samza/operators/OperatorSpecGraph.java | 132 ++++ .../apache/samza/operators/StreamGraphImpl.java | 315 ---------- .../apache/samza/operators/StreamGraphSpec.java | 299 +++++++++ .../org/apache/samza/operators/TableImpl.java | 3 +- .../operators/impl/BroadcastOperatorImpl.java | 84 +++ .../operators/impl/ControlMessageSender.java | 43 +- .../samza/operators/impl/EndOfStreamStates.java | 6 +- .../samza/operators/impl/OperatorImpl.java | 70 ++- .../samza/operators/impl/OperatorImplGraph.java | 110 ++-- .../operators/impl/OutputOperatorImpl.java | 5 +- .../operators/impl/PartitionByOperatorImpl.java | 16 +- .../operators/impl/StreamOperatorImpl.java | 3 +- .../samza/operators/impl/WatermarkStates.java | 12 +- .../operators/impl/WindowOperatorImpl.java | 23 +- .../operators/spec/BroadcastOperatorSpec.java | 49 ++ .../operators/spec/FilterOperatorSpec.java | 74 +++ .../operators/spec/FlatMapOperatorSpec.java | 47 ++ .../samza/operators/spec/InputOperatorSpec.java | 21 +- .../samza/operators/spec/JoinOperatorSpec.java | 23 +- .../samza/operators/spec/MapOperatorSpec.java | 77 +++ .../samza/operators/spec/MergeOperatorSpec.java | 51 ++ .../samza/operators/spec/OperatorSpec.java | 29 +- .../samza/operators/spec/OperatorSpecs.java | 85 +-- .../operators/spec/OutputOperatorSpec.java | 6 + .../samza/operators/spec/OutputStreamImpl.java | 17 +- .../operators/spec/PartitionByOperatorSpec.java | 29 +- .../operators/spec/SendToTableOperatorSpec.java | 15 +- .../samza/operators/spec/SinkOperatorSpec.java | 6 + .../operators/spec/StreamOperatorSpec.java | 14 +- .../spec/StreamTableJoinOperatorSpec.java | 6 + .../operators/spec/WindowOperatorSpec.java | 22 +- .../stream/IntermediateMessageStreamImpl.java | 4 +- .../samza/operators/triggers/Cancellable.java | 2 +- .../samza/operators/triggers/TriggerImpl.java | 6 +- .../operators/util/InternalInMemoryStore.java | 6 + .../apache/samza/operators/util/MathUtils.java | 50 -- .../apache/samza/processor/StreamProcessor.java | 238 ++++---- .../runtime/AbstractApplicationRunner.java | 41 +- .../samza/runtime/ApplicationRunnerMain.java | 9 + .../samza/runtime/LocalApplicationRunner.java | 77 ++- .../samza/runtime/LocalContainerRunner.java | 30 +- .../samza/runtime/RemoteApplicationRunner.java | 78 ++- .../serializers/model/SamzaObjectMapper.java | 17 +- .../standalone/PassthroughJobCoordinator.java | 37 +- .../storage/ChangelogPartitionManager.java | 89 --- .../samza/storage/ChangelogStreamManager.java | 154 +++++ .../apache/samza/storage/StorageRecovery.java | 63 +- .../org/apache/samza/system/SystemAdmins.java | 71 +++ .../samza/system/inmemory/InMemoryManager.java | 179 ++++++ .../system/inmemory/InMemorySystemAdmin.java | 137 +++++ .../system/inmemory/InMemorySystemConsumer.java | 148 +++++ .../system/inmemory/InMemorySystemFactory.java | 50 ++ .../system/inmemory/InMemorySystemProducer.java | 103 ++++ .../org/apache/samza/table/TableManager.java | 53 +- .../samza/table/caching/CachingTable.java | 199 ++++++ .../table/caching/CachingTableDescriptor.java | 173 ++++++ .../table/caching/CachingTableProvider.java | 138 +++++ .../caching/CachingTableProviderFactory.java | 34 ++ .../samza/table/caching/SupplierGauge.java | 46 ++ .../table/caching/guava/GuavaCacheTable.java | 112 ++++ .../guava/GuavaCacheTableDescriptor.java | 76 +++ .../caching/guava/GuavaCacheTableProvider.java | 92 +++ .../guava/GuavaCacheTableProviderFactory.java | 34 ++ .../samza/table/remote/CreditFunction.java | 36 ++ .../table/remote/RemoteReadWriteTable.java | 184 ++++++ .../samza/table/remote/RemoteReadableTable.java | 181 ++++++ .../table/remote/RemoteTableDescriptor.java | 194 ++++++ .../samza/table/remote/RemoteTableProvider.java | 144 +++++ .../remote/RemoteTableProviderFactory.java | 38 ++ .../samza/table/remote/TableReadFunction.java | 66 ++ .../samza/table/remote/TableWriteFunction.java | 86 +++ .../apache/samza/table/utils/SerdeUtils.java | 66 ++ .../org/apache/samza/task/AsyncRunLoop.java | 151 ++++- .../apache/samza/task/StreamOperatorTask.java | 48 +- .../apache/samza/task/SystemTimerScheduler.java | 154 +++++ .../org/apache/samza/task/TaskCallbackImpl.java | 22 +- .../apache/samza/task/TaskCallbackManager.java | 11 +- .../task/TaskCallbackTimeoutException.java | 42 -- .../org/apache/samza/task/TaskFactoryUtil.java | 42 +- .../apache/samza/util/ClassLoaderHelper.java | 48 -- .../samza/util/EmbeddedTaggedRateLimiter.java | 136 +++++ .../java/org/apache/samza/util/MathUtil.java | 77 +++ .../samza/util/MetricsReporterLoader.java | 6 +- .../util/SamzaUncaughtExceptionHandler.java | 69 +++ .../org/apache/samza/util/ScalaToJavaUtils.java | 41 -- .../org/apache/samza/util/ShutdownUtil.java | 74 +++ .../apache/samza/util/ThrottlingExecutor.java | 4 +- .../apache/samza/util/ThrottlingScheduler.java | 2 +- .../samza/zk/ScheduleAfterDebounceTime.java | 96 ++- .../samza/zk/ZkBarrierForVersionUpgrade.java | 125 +++- .../org/apache/samza/zk/ZkControllerImpl.java | 22 +- .../samza/zk/ZkCoordinationUtilsFactory.java | 6 +- .../org/apache/samza/zk/ZkJobCoordinator.java | 229 ++++--- .../samza/zk/ZkJobCoordinatorFactory.java | 2 +- .../org/apache/samza/zk/ZkLeaderElector.java | 3 +- .../main/java/org/apache/samza/zk/ZkUtils.java | 159 +++-- .../org/apache/samza/zk/ZkUtilsMetrics.java | 6 + .../samza/checkpoint/CheckpointTool.scala | 23 +- .../apache/samza/checkpoint/OffsetManager.scala | 10 +- .../org/apache/samza/config/JobConfig.scala | 11 + .../apache/samza/config/SerializerConfig.scala | 32 + .../org/apache/samza/config/StorageConfig.scala | 7 +- .../org/apache/samza/config/StreamConfig.scala | 14 +- .../org/apache/samza/config/SystemConfig.scala | 7 + .../org/apache/samza/config/TaskConfig.scala | 58 +- .../org/apache/samza/container/RunLoop.scala | 4 +- .../apache/samza/container/SamzaContainer.scala | 215 ++++--- .../samza/container/SamzaContainerMetrics.scala | 2 + .../apache/samza/container/TaskInstance.scala | 42 +- .../samza/coordinator/JobModelManager.scala | 169 +----- .../stream/CoordinatorStreamSystemFactory.scala | 50 -- .../scala/org/apache/samza/job/JobRunner.scala | 52 +- .../org/apache/samza/job/local/ProcessJob.scala | 167 +++--- .../samza/job/local/ProcessJobFactory.scala | 44 +- .../samza/job/local/ThreadJobFactory.scala | 49 +- .../org/apache/samza/metrics/JmxServer.scala | 1 + .../org/apache/samza/metrics/JvmMetrics.scala | 24 +- .../reporter/MetricsSnapshotReporter.scala | 31 +- .../MetricsSnapshotReporterFactory.scala | 6 +- .../apache/samza/serializers/SerdeManager.scala | 2 - .../samza/storage/TaskStorageManager.scala | 45 +- .../samza/system/StreamMetadataCache.scala | 25 +- .../apache/samza/system/SystemConsumers.scala | 11 +- .../system/chooser/BootstrappingChooser.scala | 38 +- .../samza/system/chooser/DefaultChooser.scala | 12 +- .../filereader/FileReaderSystemConsumer.scala | 27 +- .../org/apache/samza/util/CommandLine.scala | 4 +- .../samza/util/CoordinatorStreamUtil.scala | 94 +++ .../apache/samza/util/DaemonThreadFactory.scala | 39 -- .../samza/util/ExponentialSleepStrategy.scala | 17 +- .../scala/org/apache/samza/util/FileUtil.scala | 104 ++++ .../scala/org/apache/samza/util/HttpUtil.scala | 89 +++ .../samza/util/LexicographicComparator.scala | 39 -- .../org/apache/samza/util/ScalaJavaUtil.scala | 62 ++ .../scala/org/apache/samza/util/TimerUtil.scala | 56 ++ .../org/apache/samza/util/TimerUtils.scala | 56 -- .../main/scala/org/apache/samza/util/Util.scala | 352 ++--------- .../MockClusterResourceManager.java | 39 +- .../MockHostAwareContainerAllocator.java | 68 +++ .../TestClusterBasedJobCoordinator.java | 7 +- .../TestContainerProcessManager.java | 138 +++-- .../TestHostAwareContainerAllocator.java | 164 ++++- .../samza/container/TestLocalityManager.java | 26 +- .../TestSamzaContainerExceptionHandler.java | 39 -- .../TestSamzaUncaughtExceptionHandler.java | 40 ++ .../grouper/task/TestTaskAssignmentManager.java | 25 +- .../MockCoordinatorStreamSystemFactory.java | 6 +- .../apache/samza/example/BroadcastExample.java | 71 --- .../samza/example/KeyValueStoreExample.java | 131 ---- .../org/apache/samza/example/MergeExample.java | 60 -- .../samza/example/OrderShipmentJoinExample.java | 115 ---- .../samza/example/PageViewCounterExample.java | 95 --- .../samza/example/RepartitionExample.java | 90 --- .../org/apache/samza/example/WindowExample.java | 81 --- .../samza/execution/TestExecutionPlanner.java | 150 ++--- .../apache/samza/execution/TestJobGraph.java | 68 +-- .../execution/TestJobGraphJsonGenerator.java | 59 +- .../org/apache/samza/execution/TestJobNode.java | 14 +- .../apache/samza/execution/TestStreamEdge.java | 4 +- .../samza/execution/TestStreamManager.java | 24 +- .../samza/operators/TestJoinOperator.java | 152 ++--- .../samza/operators/TestMessageStreamImpl.java | 55 +- .../samza/operators/TestOperatorSpecGraph.java | 185 ++++++ .../samza/operators/TestStreamGraphImpl.java | 601 ------------------- .../samza/operators/TestStreamGraphSpec.java | 601 +++++++++++++++++++ .../data/TestOutputMessageEnvelope.java | 14 + .../impl/TestControlMessageSender.java | 32 +- .../samza/operators/impl/TestOperatorImpl.java | 6 + .../operators/impl/TestOperatorImplGraph.java | 298 ++++++--- .../operators/impl/TestStreamOperatorImpl.java | 4 +- .../operators/impl/TestWindowOperator.java | 263 ++++---- .../operators/impl/store/TestInMemoryStore.java | 16 + .../operators/spec/OperatorSpecTestUtils.java | 141 +++++ .../samza/operators/spec/TestOperatorSpec.java | 465 ++++++++++++++ .../spec/TestPartitionByOperatorSpec.java | 165 +++++ .../operators/spec/TestWindowOperatorSpec.java | 306 +++++++++- .../runtime/TestAbstractApplicationRunner.java | 36 +- .../runtime/TestApplicationRunnerMain.java | 2 + .../runtime/TestLocalApplicationRunner.java | 46 +- .../runtime/TestRemoteApplicationRunner.java | 53 ++ .../model/TestSamzaObjectMapper.java | 33 +- .../system/inmemory/TestInMemorySystem.java | 211 +++++++ .../apache/samza/table/TestTableManager.java | 14 +- .../samza/table/caching/TestCachingTable.java | 299 +++++++++ .../table/remote/TestRemoteTableDescriptor.java | 244 ++++++++ .../apache/samza/task/IdentityStreamTask.java | 55 ++ .../org/apache/samza/task/TestAsyncRunLoop.java | 333 +++++----- .../samza/task/TestSystemTimerScheduler.java | 176 ++++++ .../apache/samza/task/TestTaskFactoryUtil.java | 64 +- .../testUtils/InvalidStreamApplication.java | 25 - .../util/TestEmbeddedTaggedRateLimiter.java | 230 +++++++ .../org/apache/samza/util/TestMathUtils.java | 43 +- .../org/apache/samza/util/TestShutdownUtil.java | 63 ++ .../samza/zk/TestScheduleAfterDebounceTime.java | 59 +- .../zk/TestZkBarrierForVersionUpgrade.java | 280 ++++----- .../apache/samza/zk/TestZkJobCoordinator.java | 39 +- .../apache/samza/zk/TestZkLeaderElector.java | 2 +- .../apache/samza/zk/TestZkProcessorLatch.java | 6 +- .../java/org/apache/samza/zk/TestZkUtils.java | 119 +++- .../samza/checkpoint/TestOffsetManager.scala | 28 +- .../samza/config/TestSerializerConfig.scala | 57 ++ .../samza/container/TestSamzaContainer.scala | 29 +- .../samza/container/TestTaskInstance.scala | 4 +- .../samza/coordinator/TestJobCoordinator.scala | 77 ++- .../TestStreamPartitionCountMonitor.scala | 14 +- .../coordinator/server/TestHttpServer.scala | 7 +- .../apache/samza/job/local/TestProcessJob.scala | 133 +++- .../processor/StreamProcessorTestUtils.scala | 6 +- .../samza/serializers/TestSerdeManager.scala | 3 + .../samza/storage/TestTaskStorageManager.scala | 46 +- .../samza/system/TestStreamMetadataCache.scala | 101 ++-- .../chooser/TestBootstrappingChooser.scala | 94 ++- .../system/chooser/TestDefaultChooser.scala | 17 +- .../samza/util/TestDaemonThreadFactory.scala | 37 -- .../org/apache/samza/util/TestFileUtil.scala | 84 +++ .../scala/org/apache/samza/util/TestUtil.scala | 100 +-- .../ElasticsearchSystemFactory.java | 2 +- .../samza/system/hdfs/HdfsSystemAdmin.java | 2 +- .../samza/system/hdfs/HdfsSystemConsumer.java | 2 +- .../hdfs/partitioner/HdfsFileSystemAdapter.java | 7 +- .../apache/samza/system/hdfs/HdfsConfig.scala | 4 +- .../samza/system/hdfs/HdfsSystemProducer.scala | 4 +- .../hdfs/writer/AvroDataFileHdfsWriter.scala | 16 +- .../system/hdfs/TestHdfsSystemConsumer.java | 2 +- .../partitioner/TestHdfsFileSystemAdapter.java | 2 +- .../resources/partitioner/subfolder/testfile002 | 16 + .../kafka/KafkaCheckpointLogKeySerde.java | 5 + .../samza/system/kafka/KafkaStreamSpec.java | 18 +- .../kafka/KafkaCheckpointManager.scala | 91 ++- .../kafka/KafkaCheckpointManagerFactory.scala | 2 +- .../org/apache/samza/config/KafkaConfig.scala | 24 +- .../apache/samza/system/kafka/BrokerProxy.scala | 25 +- .../apache/samza/system/kafka/GetOffset.scala | 4 +- .../samza/system/kafka/KafkaSystemAdmin.scala | 72 ++- .../system/kafka/KafkaSystemConsumer.scala | 2 + .../samza/system/kafka/KafkaSystemFactory.scala | 10 +- .../system/kafka/KafkaSystemProducer.scala | 4 +- .../samza/system/kafka/TopicMetadataCache.scala | 2 +- .../scala/org/apache/samza/util/KafkaUtil.scala | 8 +- .../kafka/TestKafkaCheckpointManagerJava.java | 45 +- .../samza/system/kafka/MockKafkaProducer.java | 25 +- .../samza/system/kafka/TestKafkaStreamSpec.java | 3 +- .../system/kafka/TestKafkaSystemAdminJava.java | 44 +- .../kafka/TestKafkaCheckpointManager.scala | 38 +- .../apache/samza/config/TestKafkaConfig.scala | 10 +- .../samza/system/kafka/TestBrokerProxy.scala | 9 +- .../system/kafka/TestKafkaSystemAdmin.scala | 54 +- .../system/kafka/TestKafkaSystemConsumer.scala | 4 +- .../system/kafka/TestTopicMetadataCache.scala | 32 +- .../org/apache/samza/utils/TestKafkaUtil.scala | 7 +- .../kv/inmemory/InMemoryKeyValueStore.scala | 14 +- .../kv/inmemory/TestInMemoryKeyValueStore.java | 84 +++ .../samza/storage/kv/RocksDbKeyValueReader.java | 5 +- .../samza/storage/kv/RocksDbOptionsHelper.java | 12 +- .../RocksDbKeyValueStorageEngineFactory.scala | 3 + .../samza/storage/kv/RocksDbKeyValueStore.scala | 76 ++- .../kv/TestRocksDbKeyValueStoreJava.java | 140 +++++ .../storage/kv/TestRocksDbKeyValueStore.scala | 6 +- .../kv/BaseLocalStoreBackedTableProvider.java | 54 +- .../kv/LocalStoreBackedReadWriteTable.java | 10 +- .../kv/LocalStoreBackedReadableTable.java | 8 +- .../samza/storage/kv/AccessLoggedStore.scala | 9 +- .../kv/BaseKeyValueStorageEngineFactory.scala | 8 +- .../apache/samza/storage/kv/CachedStore.scala | 4 + .../storage/kv/KeyValueStorageEngine.scala | 52 +- .../kv/KeyValueStorageEngineMetrics.scala | 21 +- .../samza/storage/kv/KeyValueStoreMetrics.scala | 5 +- .../apache/samza/storage/kv/LoggedStore.scala | 3 + .../storage/kv/NullSafeKeyValueStore.scala | 14 +- .../storage/kv/SerializedKeyValueStore.scala | 15 + .../TestLocalBaseStoreBackedTableProvider.java | 6 +- .../samza/storage/kv/MockKeyValueStore.scala | 4 + .../storage/kv/TestKeyValueStorageEngine.scala | 6 +- .../samza/logging/log4j/StreamAppender.java | 69 ++- .../samza/logging/log4j/MockSystemAdmin.java | 74 +++ .../samza/logging/log4j/MockSystemFactory.java | 2 +- .../samza/logging/log4j/TestStreamAppender.java | 57 ++ .../samza/monitor/LocalStoreMonitorConfig.java | 4 +- .../org/apache/samza/monitor/MonitorConfig.java | 8 + .../org/apache/samza/monitor/MonitorLoader.java | 4 +- .../samza/monitor/SamzaMonitorService.java | 8 +- .../apache/samza/rest/SamzaRestApplication.java | 4 +- .../samza/rest/proxy/job/AbstractJobProxy.java | 8 +- .../rest/proxy/job/SimpleYarnJobProxy.java | 5 +- .../samza/rest/proxy/task/SamzaTaskProxy.java | 14 +- .../rest/proxy/task/SamzaTaskProxyFactory.java | 7 +- .../samza/rest/resources/TasksResource.java | 9 +- .../samza/monitor/TestLocalStoreMonitor.java | 84 +-- .../apache/samza/sql/avro/AvroRelConverter.java | 239 +++++--- .../samza/sql/avro/AvroTypeFactoryImpl.java | 9 +- .../samza/sql/data/SamzaSqlCompositeKey.java | 82 +++ .../sql/data/SamzaSqlExecutionContext.java | 24 +- .../samza/sql/data/SamzaSqlRelMessage.java | 175 ++++-- .../org/apache/samza/sql/fn/FlattenUdf.java | 2 +- .../org/apache/samza/sql/fn/RegexMatchUdf.java | 39 ++ .../sql/impl/ConfigBasedIOResolverFactory.java | 125 ++++ .../impl/ConfigBasedSourceResolverFactory.java | 71 --- .../samza/sql/impl/ConfigBasedUdfResolver.java | 13 +- .../interfaces/RelSchemaProviderFactory.java | 3 +- .../interfaces/SamzaRelConverterFactory.java | 7 +- .../samza/sql/interfaces/SourceResolver.java | 34 -- .../sql/interfaces/SourceResolverFactory.java | 36 -- .../samza/sql/interfaces/SqlIOConfig.java | 136 +++++ .../samza/sql/interfaces/SqlIOResolver.java | 45 ++ .../sql/interfaces/SqlIOResolverFactory.java | 36 ++ .../sql/interfaces/SqlSystemStreamConfig.java | 74 --- .../apache/samza/sql/planner/QueryPlanner.java | 45 +- .../sql/planner/SamzaSqlScalarFunctionImpl.java | 7 +- .../sql/runner/SamzaSqlApplicationConfig.java | 83 +-- .../sql/runner/SamzaSqlApplicationRunner.java | 23 +- .../SamzaSqlRelMessageSerdeFactory.java | 67 +++ .../SamzaSqlRelRecordSerdeFactory.java | 67 +++ .../samza/sql/testutil/SamzaSqlQueryParser.java | 67 +-- .../samza/sql/translator/FilterTranslator.java | 47 +- .../samza/sql/translator/JoinTranslator.java | 294 +++++++++ .../translator/LogicalAggregateTranslator.java | 102 ++++ .../samza/sql/translator/ProjectTranslator.java | 71 ++- .../samza/sql/translator/QueryTranslator.java | 103 +++- .../SamzaSqlRelMessageJoinFunction.java | 121 ++++ .../samza/sql/translator/ScanTranslator.java | 52 +- .../samza/sql/translator/TranslatorContext.java | 79 ++- .../apache/samza/sql/TestQueryTranslator.java | 103 ---- .../sql/TestSamzaSqlApplicationConfig.java | 92 --- .../samza/sql/TestSamzaSqlFileParser.java | 58 -- .../samza/sql/TestSamzaSqlQueryParser.java | 70 --- .../samza/sql/TestSamzaSqlRelMessage.java | 46 -- .../samza/sql/TestSamzaSqlRelMessageSerde.java | 102 ++++ .../samza/sql/TestSamzaSqlRelRecordSerde.java | 86 +++ .../samza/sql/avro/TestAvroRelConversion.java | 132 +++- .../samza/sql/avro/schemas/AddressRecord.java | 52 ++ .../apache/samza/sql/avro/schemas/Company.avsc | 39 ++ .../apache/samza/sql/avro/schemas/Company.java | 52 ++ .../sql/avro/schemas/EnrichedPageView.avsc | 81 +++ .../sql/avro/schemas/EnrichedPageView.java | 60 ++ .../org/apache/samza/sql/avro/schemas/Kind.java | 30 + .../apache/samza/sql/avro/schemas/PageView.avsc | 39 ++ .../apache/samza/sql/avro/schemas/PageView.java | 52 ++ .../samza/sql/avro/schemas/PageViewCount.avsc | 45 ++ .../samza/sql/avro/schemas/PageViewCount.java | 56 ++ .../samza/sql/avro/schemas/PhoneNumber.java | 50 ++ .../apache/samza/sql/avro/schemas/Profile.avsc | 149 +++++ .../apache/samza/sql/avro/schemas/Profile.java | 72 +++ .../samza/sql/avro/schemas/SimpleRecord.avsc | 2 +- .../samza/sql/avro/schemas/SimpleRecord.java | 2 +- .../samza/sql/avro/schemas/StreetNumRecord.java | 48 ++ .../samza/sql/data/TestSamzaSqlRelMessage.java | 46 ++ .../samza/sql/e2e/TestSamzaSqlEndToEnd.java | 137 ----- .../apache/samza/sql/e2e/TestSamzaSqlTable.java | 69 +++ .../runner/TestSamzaSqlApplicationConfig.java | 95 +++ .../runner/TestSamzaSqlApplicationRunner.java | 56 ++ .../samza/sql/system/SimpleSystemAdmin.java | 11 +- .../samza/sql/system/TestAvroSystemFactory.java | 186 +++++- .../samza/sql/testutil/MyTestArrayUdf.java | 5 +- .../apache/samza/sql/testutil/MyTestUdf.java | 2 +- .../samza/sql/testutil/SamzaSqlTestConfig.java | 97 ++- .../sql/testutil/TestIOResolverFactory.java | 196 ++++++ .../sql/testutil/TestSamzaSqlFileParser.java | 58 ++ .../sql/testutil/TestSamzaSqlQueryParser.java | 75 +++ .../sql/translator/TestFilterTranslator.java | 136 +++++ .../sql/translator/TestJoinTranslator.java | 191 ++++++ .../sql/translator/TestProjectTranslator.java | 289 +++++++++ .../sql/translator/TestQueryTranslator.java | 596 ++++++++++++++++++ .../TestSamzaSqlRelMessageJoinFunction.java | 118 ++++ .../sql/translator/TranslatorTestBase.java | 72 +++ samza-sql/src/test/resources/log4j.xml | 6 + samza-test/src/main/config/join/README | 8 +- .../example/AppWithGlobalConfigExample.java | 86 +++ .../apache/samza/example/BroadcastExample.java | 70 +++ .../samza/example/KeyValueStoreExample.java | 138 +++++ .../org/apache/samza/example/MergeExample.java | 62 ++ .../samza/example/OrderShipmentJoinExample.java | 121 ++++ .../samza/example/PageViewCounterExample.java | 100 +++ .../samza/example/RepartitionExample.java | 96 +++ .../org/apache/samza/example/WindowExample.java | 86 +++ .../samza/test/framework/StreamAssert.java | 181 ++++++ samza-test/src/main/python/configs/tests.json | 2 +- .../performance/TestKeyValuePerformance.scala | 6 +- .../samza/processor/TestZkStreamProcessor.java | 11 +- .../processor/TestZkStreamProcessorBase.java | 8 +- .../TestZkStreamProcessorFailures.java | 8 +- .../EndOfStreamIntegrationTest.java | 8 +- .../WatermarkIntegrationTest.java | 7 +- .../samza/test/operator/BroadcastAssertApp.java | 59 ++ .../test/operator/RepartitionJoinWindowApp.java | 80 ++- .../test/operator/RepartitionWindowApp.java | 72 +++ .../samza/test/operator/SessionWindowApp.java | 21 +- ...StreamApplicationIntegrationTestHarness.java | 21 +- .../operator/TestRepartitionJoinWindowApp.java | 120 +++- .../test/operator/TestRepartitionWindowApp.java | 90 +++ .../samza/test/operator/TumblingWindowApp.java | 20 +- .../samza/test/operator/data/PageView.java | 63 +- .../test/processor/SharedContextFactories.java | 117 ++++ .../test/processor/TestStreamApplication.java | 148 +++++ .../test/processor/TestStreamProcessor.java | 9 +- .../processor/TestZkLocalApplicationRunner.java | 363 +++++------ .../test/samzasql/TestSamzaSqlEndToEnd.java | 469 +++++++++++++++ .../apache/samza/test/table/TestLocalTable.java | 257 +++++--- .../samza/test/table/TestRemoteTable.java | 248 ++++++++ .../apache/samza/test/timer/TestTimerApp.java | 87 +++ .../org/apache/samza/test/timer/TimerTest.java | 51 ++ .../samza/storage/kv/TestKeyValueStores.scala | 240 ++++---- .../AbstractIntegrationTestHarness.scala | 8 +- .../AbstractKafkaServerTestHarness.scala | 11 +- .../harness/AbstractZookeeperTestHarness.scala | 10 +- .../test/integration/StreamTaskTestUtil.scala | 39 +- .../integration/TestShutdownStatefulTask.scala | 2 +- .../test/integration/TestStatefulTask.scala | 4 +- samza-tools/config/bench-log4j.xml | 35 ++ samza-tools/config/eh-bench.properties | 26 + samza-tools/scripts/eh-consumer.sh | 2 +- samza-tools/scripts/generate-kafka-events.sh | 2 +- samza-tools/scripts/samza-sql-console.sh | 2 +- samza-tools/scripts/system-consumer-bench.sh | 34 ++ .../scripts/system-consumer-with-samza-bench.sh | 34 ++ samza-tools/scripts/system-producer-bench.sh | 34 ++ .../tools/ConsoleLoggingSystemFactory.java | 27 +- .../samza/tools/EventHubConsoleConsumer.java | 64 +- .../apache/samza/tools/GenerateKafkaEvents.java | 4 +- .../org/apache/samza/tools/SamzaSqlConsole.java | 40 +- .../tools/avro/AvroSchemaGenRelConverter.java | 4 +- .../tools/benchmark/AbstractSamzaBench.java | 153 +++++ .../benchmark/ConfigBasedSspGrouperFactory.java | 87 +++ .../tools/benchmark/SystemConsumerBench.java | 91 +++ .../benchmark/SystemConsumerWithSamzaBench.java | 117 ++++ .../tools/benchmark/SystemProducerBench.java | 124 ++++ .../tools/json/JsonRelConverterFactory.java | 4 +- .../apache/samza/tools/udf/RegexMatchUdf.java | 40 -- .../job/yarn/YarnClusterResourceManager.java | 38 +- .../samza/validation/YarnJobValidationTool.java | 15 +- .../webapp/ApplicationMasterRestClient.java | 111 ++++ .../apache/samza/job/yarn/ClientHelper.scala | 54 +- .../yarn/SamzaAppMasterSecurityManager.scala | 31 +- .../yarn/SamzaContainerSecurityManager.scala | 34 +- .../apache/samza/job/yarn/YarnContainer.scala | 4 +- .../org/apache/samza/job/yarn/YarnJob.scala | 14 +- .../webapp/ApplicationMasterRestServlet.scala | 76 ++- .../webapp/TestApplicationMasterRestClient.java | 330 ++++++++++ .../TestYarnContainerHeartbeatServlet.java | 8 +- .../samza/job/yarn/TestClientHelper.scala | 36 +- .../yarn/TestSamzaYarnAppMasterService.scala | 16 +- settings.gradle | 4 +- sonar-project.properties | 6 +- 565 files changed, 25625 insertions(+), 8630 deletions(-) ----------------------------------------------------------------------
