split up the GryoRegistrator and GryoSerializer based tests into two different full runs of ProcessComputerSuite. The former is an Integration test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/293fe29c Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/293fe29c Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/293fe29c Branch: refs/heads/tp31 Commit: 293fe29c6735952a7a6b0ed4814eea72adfdde66 Parents: 08040f8 Author: Marko A. Rodriguez <okramma...@gmail.com> Authored: Mon Jun 6 20:59:21 2016 -0600 Committer: Marko A. Rodriguez <okramma...@gmail.com> Committed: Mon Jun 6 20:59:21 2016 -0600 ---------------------------------------------------------------------- .../gremlin/spark/AbstractSparkTest.java | 4 ++ ...tratorGraphComputerProcessIntegrateTest.java | 33 ++++++++++++ ...SparkHadoopGraphGryoRegistratorProvider.java | 56 ++++++++++++++++++++ .../computer/SparkHadoopGraphProvider.java | 19 ++----- 4 files changed, 97 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java index 6d2231f..4d584ff 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java @@ -26,6 +26,7 @@ import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaSparkContext; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService; import org.apache.tinkerpop.gremlin.spark.structure.Spark; import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer; import org.apache.tinkerpop.gremlin.structure.Graph; @@ -34,6 +35,8 @@ import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.SHIM_CLASS_SYSTEM_PROPERTY; + /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ @@ -43,6 +46,7 @@ public abstract class AbstractSparkTest { @After @Before public void setupTest() { + System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY); SparkConf sparkConfiguration = new SparkConf(); sparkConfiguration.setAppName(this.getClass().getCanonicalName() + "-setupTest"); sparkConfiguration.set("spark.master", "local[4]"); http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java new file mode 100644 index 0000000..29f627d --- /dev/null +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.spark.process.computer; + +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; +import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite; +import org.junit.runner.RunWith; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +@RunWith(ProcessComputerSuite.class) +@GraphProviderClass(provider = SparkHadoopGraphGryoRegistratorProvider.class, graph = HadoopGraph.class) +public class SparkGryoRegistratorGraphComputerProcessIntegrateTest { +} http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java new file mode 100644 index 0000000..df36a13 --- /dev/null +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.spark.process.computer; + +import org.apache.spark.serializer.KryoSerializer; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.spark.structure.Spark; +import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator; +import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService; +import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader; + +import java.util.Map; + +import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.SHIM_CLASS_SYSTEM_PROPERTY; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +public final class SparkHadoopGraphGryoRegistratorProvider extends SparkHadoopGraphProvider { + + private static boolean firstTest = true; + + public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { + final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); + config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false); // ensure the context doesn't stay open for the GryoSerializer tests + config.put("spark.serializer", KryoSerializer.class.getCanonicalName()); + config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName()); + // + if (firstTest) { + firstTest = false; + Spark.close(); + System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName()); + KryoShimServiceLoader.load(true); + } + // + return config; + } +} http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java index b1f610d..1fc29cf 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java @@ -55,13 +55,12 @@ import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimSe * @author Marko A. Rodriguez (http://markorodriguez.com) */ @GraphProvider.Descriptor(computer = SparkGraphComputer.class) -public final class SparkHadoopGraphProvider extends HadoopGraphProvider { +public class SparkHadoopGraphProvider extends HadoopGraphProvider { @Override public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); - config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, - !test.equals(ProgramTest.Traversals.class) && !test.equals(GroovyProgramTest.class)); // this makes the test suite go really fast + config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast // toy graph inputRDD does not have corresponding outputRDD so where jobs chain, it fails (failing makes sense) if (null != loadGraphWith && @@ -89,20 +88,10 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider { SugarTestHelper.clearRegistry(this); } + System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY); config.put(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName()); - - config.put("spark.master", "local[4]"); - if (RANDOM.nextBoolean()) { - System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, HadoopPoolShimService.class.getCanonicalName()); - KryoShimServiceLoader.load(true); - config.put("spark.serializer", GryoSerializer.class.getCanonicalName()); - } else { - System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName()); - KryoShimServiceLoader.load(true); - config.put("spark.serializer", KryoSerializer.class.getCanonicalName()); - config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName()); - } + config.put("spark.serializer", GryoSerializer.class.getCanonicalName()); config.put("spark.kryo.registrationRequired", true); return config; }