This closes #1750
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/24ecf6bb Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/24ecf6bb Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/24ecf6bb Branch: refs/heads/master Commit: 24ecf6bbfccefb33e846f7dac941b2f2e30842fd Parents: e720a7c 3784b54 Author: Sela <ans...@paypal.com> Authored: Wed Feb 15 11:28:27 2017 +0200 Committer: Sela <ans...@paypal.com> Committed: Wed Feb 15 11:28:27 2017 +0200 ---------------------------------------------------------------------- runners/spark/pom.xml | 1 - .../beam/runners/spark/SparkPipelineResult.java | 5 +- .../apache/beam/runners/spark/SparkRunner.java | 38 +++- .../beam/runners/spark/TestSparkRunner.java | 5 + .../spark/aggregators/AccumulatorSingleton.java | 137 ------------ .../aggregators/AggregatorsAccumulator.java | 111 ++++++++++ .../spark/aggregators/NamedAggregators.java | 2 +- .../spark/aggregators/SparkAggregators.java | 4 +- .../aggregators/metrics/AggregatorMetric.java | 44 ---- .../metrics/AggregatorMetricSource.java | 50 ----- .../metrics/WithNamedAggregatorsSupport.java | 174 --------------- .../spark/aggregators/metrics/sink/CsvSink.java | 39 ---- .../aggregators/metrics/sink/GraphiteSink.java | 39 ---- .../aggregators/metrics/sink/package-info.java | 23 -- .../runners/spark/metrics/AggregatorMetric.java | 43 ++++ .../spark/metrics/AggregatorMetricSource.java | 50 +++++ .../runners/spark/metrics/CompositeSource.java | 49 +++++ .../spark/metrics/MetricsAccumulator.java | 124 +++++++++++ .../spark/metrics/MetricsAccumulatorParam.java | 42 ++++ .../runners/spark/metrics/SparkBeamMetric.java | 62 ++++++ .../spark/metrics/SparkBeamMetricSource.java | 50 +++++ .../spark/metrics/SparkMetricResults.java | 181 ++++++++++++++++ .../spark/metrics/SparkMetricsContainer.java | 155 ++++++++++++++ .../spark/metrics/WithMetricsSupport.java | 209 +++++++++++++++++++ .../runners/spark/metrics/package-info.java | 20 ++ .../runners/spark/metrics/sink/CsvSink.java | 38 ++++ .../spark/metrics/sink/GraphiteSink.java | 38 ++++ .../spark/metrics/sink/package-info.java | 22 ++ .../runners/spark/translation/DoFnFunction.java | 26 ++- .../translation/DoFnRunnerWithMetrics.java | 91 ++++++++ .../spark/translation/EvaluationContext.java | 4 + .../spark/translation/MultiDoFnFunction.java | 25 ++- .../spark/translation/TransformTranslator.java | 29 ++- .../spark/translation/streaming/Checkpoint.java | 137 ++++++++++++ .../translation/streaming/CheckpointDir.java | 69 ------ .../SparkRunnerStreamingContextFactory.java | 1 + .../streaming/StreamingTransformTranslator.java | 27 ++- .../spark/aggregators/ClearAggregatorsRule.java | 5 +- .../metrics/sink/InMemoryMetrics.java | 10 +- .../ResumeFromCheckpointStreamingTest.java | 43 +++- .../spark/src/test/resources/metrics.properties | 10 +- .../src/main/resources/beam/findbugs-filter.xml | 4 +- .../beam/sdk/metrics/DistributionData.java | 3 +- .../org/apache/beam/sdk/metrics/MetricKey.java | 3 +- .../apache/beam/sdk/metrics/MetricUpdates.java | 3 +- .../apache/beam/sdk/metrics/MetricMatchers.java | 96 +++++++++ .../apache/beam/sdk/metrics/MetricsTest.java | 49 ++++- 47 files changed, 1727 insertions(+), 663 deletions(-) ----------------------------------------------------------------------
