This is an automated email from the ASF dual-hosted git repository. lcwik pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
commit dfd16f2b2c782685a02092629970d7b5f87c373b Author: Ćukasz Gajowy <lukasz.gaj...@polidea.com> AuthorDate: Tue Dec 12 18:03:32 2017 +0100 [BEAM-981] Add parameter allowing adding jars to spark context --- .../apache/beam/runners/spark/SparkPipelineOptions.java | 15 +++++++++++++++ .../runners/spark/translation/SparkContextFactory.java | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java index 26b549b..2053b44 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java @@ -18,6 +18,9 @@ package org.apache.beam.runners.spark; +import java.util.ArrayList; +import java.util.List; + import org.apache.beam.sdk.options.ApplicationNameOptions; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.DefaultValueFactory; @@ -101,4 +104,16 @@ public interface SparkPipelineOptions boolean getUsesProvidedSparkContext(); void setUsesProvidedSparkContext(boolean value); + @Description("Jars for spark context") + @Default.InstanceFactory(SparkContextOptions.EmptyPathList.class) + List<String> getJarsForSparkContext(); + void setJarsForSparkContext(List<String> jars); + + /** Returns an empty path list, top avoid handling null. */ + class EmptyPathList implements DefaultValueFactory<List<String>> { + @Override + public List<String> create(PipelineOptions options) { + return new ArrayList<>(); + } + } } diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java index 0132de3..68c4093 100644 --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java @@ -92,6 +92,11 @@ public final class SparkContextFactory { // set master if not set. conf.setMaster(contextOptions.getSparkMaster()); } + + if (contextOptions.getJarsForSparkContext().size() > 0) { + conf.setJars(contextOptions.getJarsForSparkContext().toArray(new String[0])); + } + conf.setAppName(contextOptions.getAppName()); // register immutable collections serializers because the SDK uses them. conf.set("spark.kryo.registrator", BeamSparkRunnerRegistrator.class.getName()); -- To stop receiving notification emails like this one, please contact "commits@beam.apache.org" <commits@beam.apache.org>.