Repository: incubator-beam Updated Branches: refs/heads/master f81b9a041 -> 643cf63d5
Fixed the Graphite metrics sink configuration so it actually works when submitting using spark-submit on yarn. Made the metrics configuration info a bit clearer. Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/8451b312 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/8451b312 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/8451b312 Branch: refs/heads/master Commit: 8451b31200ff47d99f40217afc743ff85ee10351 Parents: f81b9a0 Author: Stas Levin <stasle...@gmail.com> Authored: Mon Sep 12 11:27:22 2016 +0300 Committer: Sela <ans...@paypal.com> Committed: Mon Sep 12 13:18:22 2016 +0300 ---------------------------------------------------------------------- .../spark/src/test/resources/metrics.properties | 61 ++++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8451b312/runners/spark/src/test/resources/metrics.properties ---------------------------------------------------------------------- diff --git a/runners/spark/src/test/resources/metrics.properties b/runners/spark/src/test/resources/metrics.properties index 4aa01d2..143532d 100644 --- a/runners/spark/src/test/resources/metrics.properties +++ b/runners/spark/src/test/resources/metrics.properties @@ -14,16 +14,55 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# The "org.apache.beam.runners.spark.aggregators.metrics.sink.XSink" +# (a.k.a Beam.XSink) is only configured for the driver, the executors are set with a Spark native +# implementation "org.apache.spark.metrics.sink.XSink" (a.k.a Spark.XSink). +# This is due to sink class loading behavior, which is different on the driver and executors nodes. +# Since Beam aggregator metrics are reported via Spark accumulators and thus make their way to the +# driver, we only need the "Beam.XSink" on the driver side. Executor nodes can keep +# reporting Spark native metrics using the traditional Spark.XSink. +# +# The the current sink configuration pattern is therefore: +# +# driver.**.class = Beam.XSink +# executor.**.class = Spark.XSink + + +# ************* A metrics sink for tests ************* *.sink.memory.class=org.apache.beam.runners.spark.aggregators.metrics.sink.InMemoryMetrics +# ************* End of InMemoryMetrics sink configuration section ************* + + +# ************* A sample configuration for outputting metrics to Graphite ************* + +#driver.sink.graphite.class=org.apache.beam.runners.spark.aggregators.metrics.sink.GraphiteSink +#driver.sink.graphite.host=YOUR_HOST +#driver.sink.graphite.port=2003 +#driver.sink.graphite.prefix=spark +#driver.sink.graphite.period=1 +#driver.sink.graphite.unit=SECONDS + +#executor.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink +#executor.sink.graphite.host=YOUR_HOST +#executor.sink.graphite.port=2003 +#executor.sink.graphite.prefix=spark +#executor.sink.graphite.period=1 +#executor.sink.graphite.unit=SECONDS + +# ************* End of Graphite sik configuration section ************* + + +# ************* A sample configuration for outputting metrics to a CSV file. ************* + +#driver.sink.csv.class=org.apache.beam.runners.spark.aggregators.metrics.sink.CsvSink +#driver.sink.csv.directory=/tmp/spark-metrics +#driver.sink.csv.period=1 +#driver.sink.graphite.unit=SECONDS + +#executor.sink.csv.class=org.apache.spark.metrics.sink.CsvSink +#executor.sink.csv.directory=/tmp/spark-metrics +#executor.sink.csv.period=1 +#executor.sink.graphite.unit=SECONDS -#*.sink.csv.class=org.apache.beam.runners.spark.aggregators.metrics.sink.CsvSink -#*.sink.csv.directory=/tmp/spark-metrics -#*.sink.csv.period=1 -#*.sink.graphite.unit=SECONDS - -#*.sink.graphite.class=org.apache.beam.runners.spark.aggregators.metrics.sink.GraphiteSink -#*.sink.graphite.host=YOUR_HOST -#*.sink.graphite.port=2003 -#*.sink.graphite.prefix=spark -#*.sink.graphite.period=1 -#*.sink.graphite.unit=SECONDS +# ************* End of CSV sink configuration section *************
