Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/44329cf4 Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/44329cf4 Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/44329cf4 Branch: refs/heads/master Commit: 44329cf4842252edf0f0a9c4c2e57f6a61696f7b Parents: d9431b7 623661e Author: Jagadish <jvenkatra...@linkedin.com> Authored: Wed Oct 10 18:42:16 2018 -0700 Committer: Jagadish <jvenkatra...@linkedin.com> Committed: Wed Oct 10 18:42:16 2018 -0700 ---------------------------------------------------------------------- build.gradle | 50 ++- docs/_docs/replace-versioned.sh | 3 + docs/_layouts/default.html | 6 +- docs/_meetups/april-2018.md | 52 --- docs/_meetups/aug-2016.md | 44 ++ docs/_meetups/aug-2017.md | 31 ++ docs/_meetups/dec-2017.md | 45 ++ docs/_meetups/december-2018.md | 24 +- docs/_meetups/feb-2015.md | 42 ++ docs/_meetups/feb-2016.md | 32 ++ docs/_meetups/feb-2017.md | 45 ++ docs/_meetups/july-2015.md | 43 ++ docs/_meetups/july-2018.md | 6 - docs/_meetups/jun-2015.md | 31 ++ docs/_meetups/jun-2016.md | 56 +++ docs/_meetups/june-2018.md | 31 ++ docs/_meetups/mar-2018.md | 46 ++ docs/_meetups/may-2014.md | 31 ++ docs/_meetups/may-2015.md | 54 +++ docs/_meetups/may-2016.md | 31 ++ docs/_meetups/may-2017.md | 53 +++ docs/_meetups/nov-2013.md | 41 ++ docs/_meetups/nov-2014.md | 50 +++ docs/_meetups/nov-2016.md | 42 ++ docs/_meetups/nov-2017.md | 31 ++ docs/_meetups/oct-2014.md | 41 ++ docs/_meetups/oct-2015.md | 66 +++ docs/_meetups/sep-2013.md | 42 ++ docs/_meetups/sep-2014.md | 42 ++ docs/_meetups/sep-2015.md | 30 ++ docs/_meetups/sep-2017.md | 56 +++ docs/_menu/index.html | 2 +- docs/css/google-fonts.css | 18 + .../versioned/jobs/configuration-table.html | 4 +- .../documentation/versioned/jobs/logging.md | 156 ++++++- .../versioned/hello-samza-high-level-code.md | 2 +- docs/meetups/index.html | 20 +- docs/startup/quick-start/versioned/index.md | 254 +++++++++++ gradle.properties | 2 +- gradle/dependency-versions.gradle | 3 +- .../application/ApplicationDescriptor.java | 30 +- .../samza/container/SamzaContainerContext.java | 55 --- .../context/ApplicationContainerContext.java | 7 +- .../samza/context/ApplicationTaskContext.java | 4 + .../org/apache/samza/context/JobContext.java | 1 + .../apache/samza/operators/ContextManager.java | 49 --- .../operators/functions/InitableFunction.java | 9 +- .../samza/scheduler/CallbackScheduler.java | 1 + .../samza/scheduler/ScheduledCallback.java | 5 +- .../samza/storage/StorageEngineFactory.java | 8 +- .../org/apache/samza/table/ReadableTable.java | 9 +- .../org/apache/samza/table/TableProvider.java | 9 +- .../org/apache/samza/task/InitableTask.java | 6 +- .../java/org/apache/samza/task/TaskContext.java | 98 ----- .../java/org/apache/samza/util/RateLimiter.java | 9 +- samza-azure/src/test/resources/log4j.xml | 8 - samza-azure/src/test/resources/log4j2.xml | 32 ++ .../application/ApplicationDescriptorImpl.java | 55 ++- .../apache/samza/container/TaskContextImpl.java | 169 ------- .../org/apache/samza/context/ContextImpl.java | 60 ++- .../apache/samza/context/JobContextImpl.java | 22 +- .../apache/samza/context/TaskContextImpl.java | 34 +- .../samza/execution/ExecutionPlanner.java | 144 +++++- .../execution/IntermediateStreamManager.java | 253 +++-------- .../org/apache/samza/execution/JobGraph.java | 34 +- .../execution/OperatorSpecGraphAnalyzer.java | 134 +++++- .../operators/impl/BroadcastOperatorImpl.java | 9 +- .../samza/operators/impl/InputOperatorImpl.java | 5 +- .../samza/operators/impl/OperatorImpl.java | 54 +-- .../samza/operators/impl/OperatorImplGraph.java | 84 ++-- .../operators/impl/OutputOperatorImpl.java | 5 +- .../operators/impl/PartialJoinOperatorImpl.java | 11 +- .../operators/impl/PartitionByOperatorImpl.java | 17 +- .../operators/impl/SendToTableOperatorImpl.java | 15 +- .../samza/operators/impl/SinkOperatorImpl.java | 9 +- .../operators/impl/StreamOperatorImpl.java | 7 +- .../impl/StreamTableJoinOperatorImpl.java | 18 +- .../operators/impl/WindowOperatorImpl.java | 15 +- .../operators/spec/FilterOperatorSpec.java | 7 +- .../samza/operators/spec/MapOperatorSpec.java | 7 +- .../apache/samza/processor/StreamProcessor.java | 78 ++-- .../samza/runtime/LocalApplicationRunner.java | 3 +- .../samza/runtime/LocalContainerRunner.java | 14 +- .../apache/samza/storage/StorageRecovery.java | 9 +- .../org/apache/samza/table/TableManager.java | 18 +- .../samza/table/caching/CachingTable.java | 28 +- .../table/caching/CachingTableProvider.java | 8 +- .../table/caching/guava/GuavaCacheTable.java | 18 +- .../caching/guava/GuavaCacheTableProvider.java | 2 +- .../table/remote/RemoteReadWriteTable.java | 24 +- .../samza/table/remote/RemoteReadableTable.java | 30 +- .../samza/table/remote/RemoteTableProvider.java | 26 +- .../samza/table/utils/BaseTableProvider.java | 11 +- .../table/utils/DefaultTableReadMetrics.java | 11 +- .../table/utils/DefaultTableWriteMetrics.java | 11 +- .../samza/table/utils/TableMetricsUtil.java | 21 +- .../org/apache/samza/task/AsyncRunLoop.java | 5 +- .../samza/task/AsyncStreamTaskAdapter.java | 6 +- .../apache/samza/task/StreamOperatorTask.java | 33 +- .../org/apache/samza/task/TaskFactoryUtil.java | 4 +- .../samza/util/EmbeddedTaggedRateLimiter.java | 30 +- .../apache/samza/container/SamzaContainer.scala | 71 +-- .../apache/samza/container/TaskInstance.scala | 53 ++- .../samza/job/local/ThreadJobFactory.scala | 8 +- .../TestStreamApplicationDescriptorImpl.java | 37 +- .../TestTaskApplicationDescriptorImpl.java | 36 +- .../org/apache/samza/context/MockContext.java | 73 ++++ .../apache/samza/context/TestContextImpl.java | 12 +- .../samza/context/TestTaskContextImpl.java | 15 +- .../execution/ExecutionPlannerTestBase.java | 2 +- .../samza/execution/TestExecutionPlanner.java | 399 ++++++++++++++--- .../TestIntermediateStreamManager.java | 68 --- .../TestJobNodeConfigurationGenerator.java | 22 +- .../samza/operators/TestJoinOperator.java | 29 +- .../samza/operators/impl/TestOperatorImpl.java | 52 ++- .../operators/impl/TestOperatorImplGraph.java | 137 +++--- .../operators/impl/TestSinkOperatorImpl.java | 7 +- .../operators/impl/TestStreamOperatorImpl.java | 6 - .../impl/TestStreamTableJoinOperatorImpl.java | 17 +- .../operators/impl/TestWindowOperator.java | 114 ++--- .../samza/operators/spec/TestOperatorSpec.java | 2 +- .../spec/TestPartitionByOperatorSpec.java | 2 +- .../operators/spec/TestWindowOperatorSpec.java | 9 +- .../samza/processor/TestStreamProcessor.java | 9 +- .../samza/storage/MockStorageEngineFactory.java | 16 +- .../apache/samza/table/TestTableManager.java | 18 +- .../samza/table/caching/TestCachingTable.java | 48 +- .../samza/table/remote/TestRemoteTable.java | 39 +- .../table/remote/TestRemoteTableDescriptor.java | 41 +- .../retry/TestRetriableTableFunctions.java | 12 +- .../apache/samza/task/IdentityStreamTask.java | 6 +- .../org/apache/samza/task/TestAsyncRunLoop.java | 44 +- .../samza/task/TestAsyncStreamAdapter.java | 6 +- .../samza/task/TestEpochTimeScheduler.java | 3 +- .../samza/task/TestStreamOperatorTask.java | 27 ++ .../util/TestEmbeddedTaggedRateLimiter.java | 48 +- .../samza/container/TestSamzaContainer.scala | 51 ++- .../samza/container/TestTaskInstance.scala | 90 +++- .../processor/StreamProcessorTestUtils.scala | 31 +- .../InMemoryKeyValueStorageEngineFactory.scala | 13 +- .../samza/storage/kv/RocksDbKeyValueReader.java | 11 +- .../samza/storage/kv/RocksDbOptionsHelper.java | 15 +- .../RocksDbKeyValueStorageEngineFactory.scala | 23 +- .../storage/kv/TestRocksDbTableDescriptor.java | 3 +- .../kv/BaseLocalStoreBackedTableProvider.java | 18 +- .../kv/LocalStoreBackedReadWriteTable.java | 10 +- .../kv/LocalStoreBackedReadableTable.java | 10 +- .../kv/BaseKeyValueStorageEngineFactory.scala | 41 +- .../TestBaseLocalStoreBackedTableProvider.java | 18 +- .../apache/samza/config/Log4jSystemConfig.java | 88 ++++ .../samza/logging/log4j2/StreamAppender.java | 436 +++++++++++++++++++ .../logging/log4j2/StreamAppenderMetrics.java | 43 ++ .../serializers/LoggingEventJsonSerde.java | 194 +++++++++ .../LoggingEventJsonSerdeFactory.java | 36 ++ .../serializers/LoggingEventStringSerde.java | 76 ++++ .../LoggingEventStringSerdeFactory.java | 32 ++ .../samza/logging/log4j2/MockSystemAdmin.java | 74 ++++ .../samza/logging/log4j2/MockSystemFactory.java | 45 ++ .../logging/log4j2/MockSystemProducer.java | 61 +++ .../log4j2/MockSystemProducerAppender.java | 77 ++++ .../logging/log4j2/TestStreamAppender.java | 298 +++++++++++++ .../TestLoggingEventStringSerde.java | 52 +++ samza-log4j2/src/test/resources/log4j2.xml | 37 ++ samza-rest/src/main/resources/log4j2.xml | 40 ++ samza-shell/src/main/bash/checkpoint-tool.sh | 6 +- samza-shell/src/main/bash/kill-all.sh | 8 +- .../src/main/bash/kill-yarn-job-by-name.sh | 7 +- samza-shell/src/main/bash/kill-yarn-job.sh | 6 +- samza-shell/src/main/bash/list-yarn-job.sh | 6 +- samza-shell/src/main/bash/read-rocksdb-tool.sh | 6 +- samza-shell/src/main/bash/run-app.sh | 6 +- samza-shell/src/main/bash/run-class.sh | 10 +- samza-shell/src/main/bash/run-config-manager.sh | 6 +- .../main/bash/run-coordinator-stream-writer.sh | 6 +- samza-shell/src/main/bash/run-job.sh | 6 +- samza-shell/src/main/bash/stat-yarn-job.sh | 6 +- samza-shell/src/main/bash/state-storage-tool.sh | 6 +- samza-shell/src/main/bash/validate-yarn-job.sh | 6 +- .../src/main/resources/log4j2-console.xml | 35 ++ .../sql/runner/SamzaSqlApplicationContext.java | 44 ++ .../samza/sql/translator/FilterTranslator.java | 9 +- .../samza/sql/translator/ModifyTranslator.java | 11 +- .../samza/sql/translator/ProjectTranslator.java | 8 +- .../samza/sql/translator/QueryTranslator.java | 49 +-- .../samza/sql/translator/ScanTranslator.java | 11 +- .../apache/samza/sql/e2e/TestSamzaSqlTable.java | 1 - .../runner/TestSamzaSqlApplicationRunner.java | 2 - .../samza/sql/system/TestAvroSystemFactory.java | 1 - .../sql/testutil/TestIOResolverFactory.java | 1 - .../sql/testutil/TestSamzaSqlFileParser.java | 2 - .../sql/translator/TestFilterTranslator.java | 16 +- .../sql/translator/TestProjectTranslator.java | 25 +- .../sql/translator/TestQueryTranslator.java | 47 +- samza-sql/src/test/resources/log4j.xml | 9 - samza-sql/src/test/resources/log4j2.xml | 35 ++ .../samza/example/KeyValueStoreExample.java | 7 +- .../test/framework/MessageStreamAssert.java | 15 +- .../test/integration/NegateNumberTask.java | 9 +- .../test/integration/SimpleStatefulTask.java | 13 +- .../test/integration/StatePerfTestTask.java | 9 +- .../samza/test/integration/join/Checker.java | 19 +- .../samza/test/integration/join/Emitter.java | 23 +- .../samza/test/integration/join/Joiner.java | 26 +- .../samza/test/integration/join/Watcher.java | 17 +- samza-test/src/main/resources/log4j2.xml | 41 ++ .../performance/TestKeyValuePerformance.scala | 21 +- .../test/performance/TestPerformanceTask.scala | 19 +- .../processor/TestZkStreamProcessorBase.java | 6 +- .../test/framework/FaultInjectionTest.java | 1 - .../samza/test/framework/TestSchedulingApp.java | 2 +- .../test/processor/IdentityStreamTask.java | 5 +- .../test/processor/TestStreamProcessor.java | 5 +- .../apache/samza/test/table/TestLocalTable.java | 33 +- .../table/TestLocalTableWithSideInputs.java | 28 +- .../samza/test/table/TestRemoteTable.java | 27 +- .../table/TestTableDescriptorsProvider.java | 3 +- .../test/integration/StreamTaskTestUtil.scala | 19 +- .../integration/TestShutdownStatefulTask.scala | 8 +- .../test/integration/TestStatefulTask.scala | 8 +- samza-tools/src/main/resources/log4j.xml | 9 - samza-tools/src/main/resources/log4j2.xml | 32 ++ settings.gradle | 1 + 222 files changed, 5649 insertions(+), 2121 deletions(-) ----------------------------------------------------------------------