Repository: tinkerpop Updated Branches: refs/heads/master 77bbb427c -> 3fc700fdc
if the System.getProperty() has a shim-service set, then it is also set as a DRIVER_JAVA_OPTIONS and an EXECUTOR_JAVA_OPTIONS in SparkGraphComputer (this is necessary for Spark 2.0). Also, some is.testing class registrations are now also needed outside of testing -- Spark 2.0 anonomaly. Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4489247f Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4489247f Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4489247f Branch: refs/heads/master Commit: 4489247fc191bfb014d6ed4e581e0acbbf407cf4 Parents: cafeefc Author: Marko A. Rodriguez <[email protected]> Authored: Mon Oct 24 15:21:23 2016 -0600 Committer: Marko A. Rodriguez <[email protected]> Committed: Tue Nov 29 04:54:21 2016 -0700 ---------------------------------------------------------------------- .../spark/process/computer/SparkGraphComputer.java | 8 ++++++++ .../spark/structure/io/gryo/GryoRegistrator.java | 12 +++++------- 2 files changed, 13 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4489247f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java index 40598c0..36618a1 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java @@ -70,6 +70,7 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorage; import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.io.Storage; +import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader; import java.io.File; import java.io.IOException; @@ -105,6 +106,13 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer { super(hadoopGraph); this.sparkConfiguration = new HadoopConfiguration(); ConfigurationUtils.copy(this.hadoopGraph.configuration(), this.sparkConfiguration); + if (null != System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE, null)) { + final String shimService = System.getProperty(KryoShimServiceLoader.KRYO_SHIM_SERVICE); + this.sparkConfiguration.setProperty(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, + (this.sparkConfiguration.getString(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim()); + this.sparkConfiguration.setProperty(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, + (this.sparkConfiguration.getString(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "") + " -D" + KryoShimServiceLoader.KRYO_SHIM_SERVICE + "=" + shimService).trim()); + } } @Override http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4489247f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java index aff8f49..aa64936 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java @@ -178,13 +178,11 @@ public class GryoRegistrator implements KryoRegistrator { // duplication, but it would be a bit cumbersome to do so without disturbing // the ordering of the existing entries in that constructor, since not all // of the entries are for TinkerPop (and the ordering is significant). - if (Boolean.valueOf(System.getProperty("is.testing", "false"))) { - try { - m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer()); - m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer()); - } catch (final ClassNotFoundException e) { - throw new IllegalStateException(e.getMessage(), e); - } + try { + m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer()); + m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer()); + } catch (final ClassNotFoundException e) { + throw new IllegalStateException(e.getMessage(), e); } m.put(WrappedArray.ofRef.class, null); m.put(MessagePayload.class, null);
