Merge branch 'master' of https://github.com/apache/samza
Project: http://git-wip-us.apache.org/repos/asf/samza/repo Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/dd39d089 Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/dd39d089 Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/dd39d089 Branch: refs/heads/NewKafkaSystemConsumer Commit: dd39d089437905abb93aa1074474b28a18292a54 Parents: 1ad58d4 7a2e192 Author: Boris S <[email protected]> Authored: Wed Nov 22 11:46:37 2017 -0800 Committer: Boris S <[email protected]> Committed: Wed Nov 22 11:46:37 2017 -0800 ---------------------------------------------------------------------- .gitignore | 3 +- NOTICE | 2 - build.gradle | 35 +- .../versioned/jobs/configuration-table.html | 14 +- gradle/dependency-versions.gradle | 4 +- .../samza/checkpoint/CheckpointManager.java | 6 +- .../samza/container/SamzaContainerContext.java | 7 +- .../operators/functions/WatermarkFunction.java | 7 +- .../org/apache/samza/sql/udfs/ScalarUdf.java | 48 ++ .../org/apache/samza/system/ControlMessage.java | 31 + .../apache/samza/system/WatermarkMessage.java | 26 + .../azure/AzureCheckpointManager.java | 236 +++++++ .../azure/AzureCheckpointManagerFactory.java | 33 + .../checkpoint/azure/TaskCheckpointEntity.java | 43 ++ .../org/apache/samza/config/AzureConfig.java | 2 +- .../coordinator/AzureCoordinationUtils.java | 2 +- .../samza/coordinator/AzureJobCoordinator.java | 2 +- .../producer/EventHubSystemProducer.java | 2 +- .../azure/ITestAzureCheckpointManager.java | 181 +++++ .../consumer/TestEventHubSystemConsumer.java | 56 +- .../producer/SwapFirstLastByteInterceptor.java | 36 + .../producer/TestEventHubSystemProducer.java | 57 ++ .../apache/samza/PartitionChangeException.java | 31 + .../ClusterBasedJobCoordinator.java | 108 ++- .../clustermanager/ContainerProcessManager.java | 26 +- .../clustermanager/SamzaApplicationState.java | 7 +- .../org/apache/samza/config/TaskConfigJava.java | 2 +- .../StreamPartitionCountMonitor.java | 116 ++-- .../org/apache/samza/execution/StreamEdge.java | 1 + .../apache/samza/operators/StreamGraphImpl.java | 7 + .../samza/operators/impl/OperatorImpl.java | 68 +- .../operators/impl/PartitionByOperatorImpl.java | 7 +- .../operators/impl/WindowOperatorImpl.java | 13 + .../samza/runtime/LocalApplicationRunner.java | 2 +- .../apache/samza/storage/StorageRecovery.java | 2 +- .../samza/checkpoint/CheckpointTool.scala | 15 +- .../org/apache/samza/config/JobConfig.scala | 2 - .../org/apache/samza/config/StorageConfig.scala | 8 + .../org/apache/samza/config/StreamConfig.scala | 30 +- .../org/apache/samza/config/TaskConfig.scala | 5 + .../apache/samza/container/SamzaContainer.scala | 2 +- .../samza/coordinator/JobModelManager.scala | 49 +- .../samza/storage/TaskStorageManager.scala | 54 +- .../main/scala/org/apache/samza/util/Util.scala | 1 - .../MockClusterResourceManagerFactory.java | 32 + .../clustermanager/MockContainerListener.java | 1 + .../TestClusterBasedJobCoordinator.java | 108 +++ .../clustermanager/TestContainerAllocator.java | 12 +- .../TestContainerProcessManager.java | 22 +- .../TestHostAwareContainerAllocator.java | 16 +- .../apache/samza/config/TestTaskConfigJava.java | 20 + .../coordinator/JobModelManagerTestUtil.java | 4 +- .../samza/coordinator/TestJobModelManager.java | 24 +- .../apache/samza/execution/TestStreamEdge.java | 1 + .../samza/operators/TestStreamGraphImpl.java | 38 ++ .../samza/operators/TestWindowOperator.java | 565 ---------------- .../operators/impl/TestOperatorImplGraph.java | 10 +- .../operators/impl/TestWindowOperator.java | 677 +++++++++++++++++++ .../samza/storage/MockSystemConsumer.java | 59 -- .../apache/samza/storage/MockSystemFactory.java | 45 -- .../samza/storage/TestStorageRecovery.java | 37 +- .../apache/samza/system/MockSystemFactory.java | 181 +++++ .../zk/TestZkBarrierForVersionUpgrade.java | 23 +- .../samza/checkpoint/TestCheckpointTool.scala | 23 +- .../samza/container/TestSamzaContainer.scala | 13 +- .../samza/container/TestTaskInstance.scala | 10 +- .../samza/coordinator/TestJobCoordinator.scala | 65 +- .../TestStreamPartitionCountMonitor.scala | 82 ++- .../processor/StreamProcessorTestUtils.scala | 3 +- .../samza/serializers/TestCheckpointSerde.scala | 8 + .../samza/storage/TestTaskStorageManager.scala | 210 ++++-- .../TestRangeSystemStreamPartitionMatcher.scala | 1 - .../TestRegexSystemStreamPartitionMatcher.scala | 1 - .../scala/org/apache/samza/util/TestUtil.scala | 16 + .../checkpoint/kafka/KafkaCheckpointLogKey.java | 110 +++ .../kafka/KafkaCheckpointLogKeySerde.java | 68 ++ .../samza/system/kafka/KafkaStreamSpec.java | 4 + .../kafka/KafkaCheckpointLogKey.scala | 171 ----- .../kafka/KafkaCheckpointManager.scala | 385 ++++++----- .../kafka/KafkaCheckpointManagerFactory.scala | 81 +-- .../system/kafka/KafkaSystemConsumer.scala | 8 +- .../kafka/KafkaSystemConsumerMetrics.scala | 2 - .../kafka/TestKafkaCheckpointLogKeySerde.java | 53 ++ .../kafka/TestKafkaCheckpointManagerJava.java | 247 +++++++ .../kafka/TeskKafkaCheckpointLogKey.scala | 61 -- .../kafka/TestKafkaCheckpointManager.scala | 388 ++++------- .../system/kafka/TestKafkaSystemAdmin.scala | 2 +- .../samza/storage/kv/RocksDbKeyValueReader.java | 3 +- .../apache/samza/monitor/LocalStoreMonitor.java | 1 + .../samza/monitor/LocalStoreMonitorMetrics.java | 4 + .../apache/samza/sql/avro/AvroRelConverter.java | 183 +++++ .../samza/sql/avro/AvroRelConverterFactory.java | 44 ++ .../samza/sql/avro/AvroRelSchemaProvider.java | 28 + .../samza/sql/avro/AvroTypeFactoryImpl.java | 132 ++++ ...ConfigBasedAvroRelSchemaProviderFactory.java | 63 ++ .../org/apache/samza/sql/data/Expression.java | 38 ++ .../samza/sql/data/RexToJavaCompiler.java | 224 ++++++ .../sql/data/SamzaSqlExecutionContext.java | 61 ++ .../samza/sql/data/SamzaSqlRelMessage.java | 123 ++++ .../org/apache/samza/sql/fn/FlattenUdf.java | 36 + .../impl/ConfigBasedSourceResolverFactory.java | 71 ++ .../samza/sql/impl/ConfigBasedUdfResolver.java | 97 +++ .../samza/sql/interfaces/RelSchemaProvider.java | 36 + .../interfaces/RelSchemaProviderFactory.java | 33 + .../samza/sql/interfaces/SamzaRelConverter.java | 46 ++ .../interfaces/SamzaRelConverterFactory.java | 39 ++ .../samza/sql/interfaces/SourceResolver.java | 34 + .../sql/interfaces/SourceResolverFactory.java | 36 + .../sql/interfaces/SqlSystemStreamConfig.java | 74 ++ .../samza/sql/interfaces/UdfMetadata.java | 61 ++ .../samza/sql/interfaces/UdfResolver.java | 35 + .../org/apache/samza/sql/planner/Checker.java | 93 +++ .../apache/samza/sql/planner/QueryPlanner.java | 153 +++++ .../sql/planner/SamzaSqlOperatorTable.java | 101 +++ .../sql/planner/SamzaSqlScalarFunctionImpl.java | 84 +++ .../sql/planner/SamzaSqlUdfOperatorTable.java | 62 ++ .../samza/sql/runner/SamzaSqlApplication.java | 56 ++ .../sql/runner/SamzaSqlApplicationConfig.java | 245 +++++++ .../sql/runner/SamzaSqlApplicationRunner.java | 133 ++++ .../apache/samza/sql/testutil/ConfigUtil.java | 62 ++ .../org/apache/samza/sql/testutil/JsonUtil.java | 91 +++ .../samza/sql/testutil/ReflectionUtils.java | 62 ++ .../samza/sql/testutil/SamzaSqlQueryParser.java | 188 +++++ .../samza/sql/testutil/SqlFileParser.java | 103 +++ .../samza/sql/translator/FilterTranslator.java | 62 ++ .../samza/sql/translator/ProjectTranslator.java | 108 +++ .../samza/sql/translator/QueryTranslator.java | 96 +++ .../samza/sql/translator/ScanTranslator.java | 70 ++ .../samza/sql/translator/TranslatorContext.java | 162 +++++ .../apache/samza/sql/TestQueryTranslator.java | 103 +++ .../sql/TestSamzaSqlApplicationConfig.java | 92 +++ .../samza/sql/TestSamzaSqlFileParser.java | 58 ++ .../samza/sql/TestSamzaSqlQueryParser.java | 70 ++ .../samza/sql/TestSamzaSqlRelMessage.java | 46 ++ .../samza/sql/avro/TestAvroRelConversion.java | 239 +++++++ .../samza/sql/avro/schemas/ComplexRecord.avsc | 143 ++++ .../samza/sql/avro/schemas/ComplexRecord.java | 92 +++ .../apache/samza/sql/avro/schemas/MyFixed.java | 29 + .../samza/sql/avro/schemas/SimpleRecord.avsc | 39 ++ .../samza/sql/avro/schemas/SimpleRecord.java | 52 ++ .../samza/sql/avro/schemas/SubRecord.java | 53 ++ .../samza/sql/avro/schemas/TestEnumType.java | 31 + .../samza/sql/e2e/TestSamzaSqlEndToEnd.java | 137 ++++ .../sql/system/ConsoleLoggingSystemFactory.java | 83 +++ .../samza/sql/system/SimpleSystemAdmin.java | 61 ++ .../samza/sql/system/TestAvroSystemFactory.java | 156 +++++ .../samza/sql/testutil/MyTestArrayUdf.java | 37 + .../apache/samza/sql/testutil/MyTestUdf.java | 45 ++ .../samza/sql/testutil/SamzaSqlTestConfig.java | 103 +++ samza-sql/src/test/resources/log4j.xml | 43 ++ .../performance/TestKeyValuePerformance.scala | 2 +- .../samza/processor/TestZkStreamProcessor.java | 11 +- .../processor/TestZkStreamProcessorBase.java | 3 +- .../TestZkStreamProcessorFailures.java | 8 +- .../processor/TestZkStreamProcessorSession.java | 5 +- .../operator/TestRepartitionJoinWindowApp.java | 13 +- .../processor/TestZkLocalApplicationRunner.java | 15 +- settings.gradle | 5 +- 158 files changed, 8640 insertions(+), 1843 deletions(-) ----------------------------------------------------------------------
