Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/67e611ee Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/67e611ee Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/67e611ee Branch: refs/heads/NewKafkaSystemConsumer Commit: 67e611ee47f0747b81c0de5d061f9888145b605a Parents: dd39d08 b00ebd2 Author: Boris S <[email protected]> Authored: Wed Jan 10 09:50:40 2018 -0800 Committer: Boris S <[email protected]> Committed: Wed Jan 10 09:50:40 2018 -0800 ---------------------------------------------------------------------- RELEASE.md | 4 +- bin/merge-pull-request.py | 28 +- build.gradle | 96 +++++- docs/README.md | 4 +- docs/_config.yml | 4 +- docs/_docs/replace-versioned.sh | 3 + docs/_layouts/default.html | 1 + docs/archive/index.html | 8 + .../documentation/hadoop/multi_stage_batch.png | Bin 0 -> 65364 bytes .../hadoop/unified_batch_streaming.png | Bin 0 -> 17371 bytes .../azure/eventhub_send_methods.png | Bin 0 -> 52427 bytes .../documentation/versioned/aws/kinesis.md | 104 +++++++ .../documentation/versioned/azure/eventhubs.md | 197 ++++++++++++ .../versioned/container/metrics-table.html | 12 + .../documentation/versioned/hadoop/consumer.md | 110 +++++++ .../documentation/versioned/hadoop/overview.md | 46 +++ .../documentation/versioned/hadoop/producer.md | 68 ++++ .../documentation/versioned/hdfs/consumer.md | 110 ------- .../documentation/versioned/hdfs/producer.md | 70 ----- docs/learn/documentation/versioned/index.html | 27 +- .../versioned/jobs/configuration-table.html | 128 ++++++++ .../versioned/yarn/yarn-host-affinity.md | 2 +- .../versioned/yarn/yarn-security.md | 1 - .../versioned/deploy-samza-job-from-hdfs.md | 2 +- .../tutorials/versioned/deploy-samza-to-CDH.md | 4 +- .../versioned/hello-samza-high-level-code.md | 87 +++--- .../versioned/hello-samza-high-level-yarn.md | 2 +- .../versioned/hello-samza-high-level-zk.md | 2 +- docs/learn/tutorials/versioned/index.md | 3 + .../versioned/remote-debugging-samza.md | 2 +- .../versioned/run-in-multi-node-yarn.md | 4 +- .../versioned/samza-rest-getting-started.md | 2 +- docs/learn/tutorials/versioned/samza-sql.md | 123 ++++++++ docs/learn/tutorials/versioned/samza-tools.md | 109 +++++++ docs/startup/download/index.md | 24 +- docs/startup/hello-samza/versioned/index.md | 2 +- docs/startup/preview/index.md | 135 ++++++-- .../startup/releases/versioned/release-notes.md | 82 +++++ gradle.properties | 2 +- gradle/dependency-versions.gradle | 1 + .../java/org/apache/samza/operators/KV.java | 2 +- .../apache/samza/operators/MessageStream.java | 69 ++++- .../org/apache/samza/operators/StreamGraph.java | 48 ++- .../apache/samza/operators/TableDescriptor.java | 73 +++++ .../functions/StreamTableJoinFunction.java | 59 ++++ .../org/apache/samza/serializers/KVSerde.java | 8 +- .../table/LocalStoreBackedTableProvider.java | 37 +++ .../org/apache/samza/table/ReadWriteTable.java | 75 +++++ .../org/apache/samza/table/ReadableTable.java | 61 ++++ .../main/java/org/apache/samza/table/Table.java | 31 ++ .../org/apache/samza/table/TableProvider.java | 57 ++++ .../samza/table/TableProviderFactory.java | 35 +++ .../java/org/apache/samza/table/TableSpec.java | 125 ++++++++ .../java/org/apache/samza/task/TaskContext.java | 6 +- .../kinesis/KinesisAWSCredentialsProvider.java | 69 +++++ .../samza/system/kinesis/KinesisConfig.java | 287 +++++++++++++++++ .../system/kinesis/KinesisSystemAdmin.java | 124 ++++++++ .../system/kinesis/KinesisSystemFactory.java | 87 ++++++ .../KinesisIncomingMessageEnvelope.java | 62 ++++ .../consumer/KinesisRecordProcessor.java | 208 +++++++++++++ .../KinesisRecordProcessorListener.java | 51 +++ .../kinesis/consumer/KinesisSystemConsumer.java | 256 +++++++++++++++ .../consumer/KinesisSystemConsumerOffset.java | 107 +++++++ .../consumer/NoAvailablePartitionException.java | 38 +++ .../system/kinesis/consumer/SSPAllocator.java | 73 +++++ .../metrics/KinesisSystemConsumerMetrics.java | 106 +++++++ .../system/kinesis/metrics/SamzaHistogram.java | 63 ++++ .../TestKinesisAWSCredentialsProvider.java | 60 ++++ .../samza/system/kinesis/TestKinesisConfig.java | 132 ++++++++ .../kinesis/TestKinesisSystemFactory.java | 115 +++++++ .../consumer/TestKinesisRecordProcessor.java | 301 ++++++++++++++++++ .../consumer/TestKinesisSystemConsumer.java | 270 ++++++++++++++++ .../TestKinesisSystemConsumerOffset.java | 48 +++ .../kinesis/consumer/TestSSPAllocator.java | 127 ++++++++ .../azure/AzureCheckpointManager.java | 31 +- .../azure/AzureCheckpointManagerFactory.java | 4 +- .../samza/system/eventhub/EventHubConfig.java | 71 ++++- .../eventhub/SamzaEventHubClientManager.java | 4 +- .../eventhub/admin/EventHubSystemAdmin.java | 17 +- .../consumer/EventHubSystemConsumer.java | 77 +++-- .../producer/EventHubSystemProducer.java | 103 +++--- .../azure/ITestAzureCheckpointManager.java | 2 +- .../MockEventHubClientManagerFactory.java | 18 ++ .../eventhub/admin/TestEventHubSystemAdmin.java | 15 +- .../consumer/ITestEventHubSystemConsumer.java | 5 +- .../consumer/TestEventHubSystemConsumer.java | 19 +- .../producer/ITestEventHubSystemProducer.java | 2 + .../producer/TestEventHubSystemProducer.java | 34 +- .../AbstractContainerAllocator.java | 17 +- .../clustermanager/ClusterResourceManager.java | 26 +- .../clustermanager/ContainerProcessManager.java | 74 ++++- .../clustermanager/SamzaApplicationState.java | 8 +- .../samza/clustermanager/SamzaResource.java | 8 + .../apache/samza/config/JavaStorageConfig.java | 5 +- .../apache/samza/config/JavaTableConfig.java | 87 ++++++ .../container/ContainerHeartbeatClient.java | 20 +- .../container/ContainerHeartbeatMonitor.java | 11 +- .../apache/samza/container/TaskContextImpl.java | 24 +- .../AllSspToSingleTaskGrouperFactory.java | 50 ++- .../samza/execution/ExecutionPlanner.java | 5 + .../org/apache/samza/execution/JobGraph.java | 16 + .../samza/execution/JobGraphJsonGenerator.java | 60 +++- .../org/apache/samza/execution/JobNode.java | 54 +++- .../samza/operators/BaseTableDescriptor.java | 94 ++++++ .../samza/operators/MessageStreamImpl.java | 35 ++- .../apache/samza/operators/StreamGraphImpl.java | 46 ++- .../org/apache/samza/operators/TableImpl.java | 40 +++ .../samza/operators/impl/InputOperatorImpl.java | 2 +- .../samza/operators/impl/OperatorImpl.java | 16 +- .../samza/operators/impl/OperatorImplGraph.java | 44 ++- .../operators/impl/OutputOperatorImpl.java | 2 +- .../operators/impl/PartitionByOperatorImpl.java | 7 +- .../operators/impl/SendToTableOperatorImpl.java | 71 +++++ .../impl/StreamTableJoinOperatorImpl.java | 82 +++++ .../samza/operators/spec/InputOperatorSpec.java | 10 +- .../samza/operators/spec/OperatorSpec.java | 12 +- .../samza/operators/spec/OperatorSpecs.java | 44 ++- .../samza/operators/spec/OutputStreamImpl.java | 10 +- .../operators/spec/SendToTableOperatorSpec.java | 65 ++++ .../spec/StreamTableJoinOperatorSpec.java | 67 ++++ .../stream/IntermediateMessageStreamImpl.java | 13 + .../samza/runtime/LocalContainerRunner.java | 9 +- .../samza/runtime/RemoteApplicationRunner.java | 7 +- .../standalone/PassthroughJobCoordinator.java | 6 +- .../org/apache/samza/table/TableManager.java | 153 +++++++++ .../org/apache/samza/zk/ZkJobCoordinator.java | 30 +- .../main/java/org/apache/samza/zk/ZkUtils.java | 2 +- .../org/apache/samza/config/JobConfig.scala | 1 + .../apache/samza/container/SamzaContainer.scala | 75 ++--- .../apache/samza/container/TaskInstance.scala | 40 ++- .../samza/coordinator/JobModelManager.scala | 23 +- .../org/apache/samza/metrics/JvmMetrics.scala | 28 +- .../MockClusterResourceManager.java | 19 +- .../MockClusterResourceManagerCallback.java | 10 + .../clustermanager/TestContainerAllocator.java | 55 ---- .../TestContainerProcessManager.java | 101 +++--- .../TestHostAwareContainerAllocator.java | 56 ---- .../samza/config/TestJavaStorageConfig.java | 13 + .../samza/config/TestJavaTableConfig.java | 58 ++++ .../stream/TestAllSspToSingleTaskGrouper.java | 125 ++++++++ .../execution/TestJobGraphJsonGenerator.java | 75 +++++ .../samza/operators/TestMessageStreamImpl.java | 70 ++++- .../samza/operators/TestStreamGraphImpl.java | 25 +- .../samza/operators/impl/TestOperatorImpl.java | 1 + .../impl/TestStreamTableJoinOperatorImpl.java | 101 ++++++ .../apache/samza/table/TestTableManager.java | 176 +++++++++++ .../org/apache/samza/task/TestAsyncRunLoop.java | 14 +- .../kafka/KafkaCheckpointManager.scala | 2 +- .../system/kafka/KafkaSystemProducer.scala | 135 ++++---- .../kafka/TestKafkaCheckpointManager.scala | 1 - .../system/kafka/TestKafkaSystemProducer.scala | 39 ++- .../kv/inmemory/InMemoryTableDescriptor.java | 59 ++++ .../kv/inmemory/InMemoryTableProvider.java | 65 ++++ .../inmemory/InMemoryTableProviderFactory.java | 33 ++ .../inmemory/TestInMemoryTableDescriptor.java | 48 +++ .../kv/inmemory/TestInMemoryTableProvider.java | 65 ++++ .../storage/kv/RocksDbTableDescriptor.java | 232 ++++++++++++++ .../samza/storage/kv/RocksDbTableProvider.java | 64 ++++ .../storage/kv/RocksDbTableProviderFactory.java | 31 ++ .../samza/storage/kv/RocksDbKeyValueStore.scala | 12 +- .../storage/kv/TestRocksDbTableDescriptor.java | 87 ++++++ .../storage/kv/TestRocksDbTableProvider.java | 66 ++++ .../kv/BaseLocalStoreBackedTableDescriptor.java | 56 ++++ .../kv/BaseLocalStoreBackedTableProvider.java | 92 ++++++ .../kv/LocalStoreBackedReadWriteTable.java | 68 ++++ .../kv/LocalStoreBackedReadableTable.java | 61 ++++ .../samza/storage/kv/AccessLoggedStore.scala | 4 +- .../storage/kv/KeyValueStorageEngine.scala | 14 +- .../kv/KeyValueStorageEngineMetrics.scala | 4 +- .../apache/samza/storage/kv/LoggedStore.scala | 4 +- .../storage/kv/NullSafeKeyValueStore.scala | 4 +- .../storage/kv/SerializedKeyValueStore.scala | 4 +- .../TestLocalBaseStoreBackedTableProvider.java | 77 +++++ .../storage/kv/TestKeyValueStorageEngine.scala | 16 + .../samza/logging/log4j/StreamAppender.java | 110 ++++++- .../logging/log4j/StreamAppenderMetrics.java | 43 +++ .../samza/logging/log4j/MockSystemProducer.java | 12 +- .../samza/logging/log4j/TestStreamAppender.java | 144 +++++++-- .../src/main/visualizer/js/planToDagre.js | 6 +- .../apache/samza/sql/avro/AvroRelConverter.java | 6 +- samza-test/src/main/config/join/README | 8 +- samza-test/src/main/python/configs/tests.json | 2 +- .../test/processor/TestStreamProcessor.java | 9 +- .../processor/TestZkLocalApplicationRunner.java | 152 +++++++-- .../apache/samza/test/table/TestLocalTable.java | 304 ++++++++++++++++++ .../apache/samza/test/table/TestTableData.java | 200 ++++++++++++ .../samza/test/util/ArraySystemConsumer.java | 4 +- .../samza/test/util/SimpleSystemAdmin.java | 26 +- samza-tools/config/eh-consumer-log4j.xml | 35 +++ .../config/generate-kafka-events-log4j.xml | 35 +++ samza-tools/config/samza-sql-console-log4j.xml | 35 +++ samza-tools/scripts/eh-consumer.sh | 34 ++ samza-tools/scripts/generate-kafka-events.sh | 34 ++ samza-tools/scripts/samza-sql-console.sh | 34 ++ .../apache/samza/tools/CommandLineHelper.java | 42 +++ .../tools/ConsoleLoggingSystemFactory.java | 126 ++++++++ .../samza/tools/EventHubConsoleConsumer.java | 120 +++++++ .../apache/samza/tools/GenerateKafkaEvents.java | 205 ++++++++++++ .../samza/tools/RandomValueGenerator.java | 87 ++++++ .../org/apache/samza/tools/SamzaSqlConsole.java | 188 +++++++++++ .../tools/avro/AvroSchemaGenRelConverter.java | 94 ++++++ .../avro/AvroSchemaGenRelConverterFactory.java | 43 +++ .../samza/tools/avro/AvroSerDeFactory.java | 96 ++++++ .../tools/json/JsonRelConverterFactory.java | 93 ++++++ .../samza/tools/schemas/PageViewEvent.avsc | 51 +++ .../samza/tools/schemas/PageViewEvent.java | 60 ++++ .../samza/tools/schemas/ProfileChangeEvent.avsc | 51 +++ .../samza/tools/schemas/ProfileChangeEvent.java | 60 ++++ .../apache/samza/tools/udf/RegexMatchUdf.java | 40 +++ samza-tools/src/main/resources/log4j.xml | 43 +++ .../org/apache/samza/job/yarn/YarnAppState.java | 4 +- .../job/yarn/YarnClusterResourceManager.java | 310 +++++++++++++++++-- .../samza/job/yarn/YarnContainerRunner.java | 272 ---------------- settings.gradle | 6 +- 214 files changed, 10944 insertions(+), 1302 deletions(-) ----------------------------------------------------------------------
