TINKERPOP-1996 Moved IoStep implementations to sideEffect package These steps really aren't quite sideEffects and not quite map steps either but they seem to fit better as sideEffect. meh
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/6d05805a Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/6d05805a Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/6d05805a Branch: refs/heads/TINKERPOP-1967 Commit: 6d05805ada657bcb3f50a60aa0c313c29d4611bb Parents: 62175c2 Author: Stephen Mallette <sp...@genoprime.com> Authored: Sat Jul 14 06:23:54 2018 -0400 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Thu Jul 19 13:40:10 2018 -0400 ---------------------------------------------------------------------- .../dsl/graph/GraphTraversalSource.java | 2 +- .../process/traversal/step/map/IoStep.java | 241 ------------------- .../traversal/step/sideEffect/IoStep.java | 241 +++++++++++++++++++ .../traversal/step/map/HadoopIoStep.java | 170 ------------- .../traversal/step/sideEffect/HadoopIoStep.java | 163 +++++++++++++ .../traversal/strategy/HadoopIoStrategy.java | 5 +- 6 files changed, 407 insertions(+), 415 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java index 7357418..df1b108 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalSource.java @@ -31,7 +31,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddEdgeStartStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddVertexStartStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.map.IoStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.InjectStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.RequirementsStrategy; import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement; http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java deleted file mode 100644 index 668b3dc..0000000 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/IoStep.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tinkerpop.gremlin.process.traversal.step.map; - -import org.apache.tinkerpop.gremlin.process.traversal.IO; -import org.apache.tinkerpop.gremlin.process.traversal.Traversal; -import org.apache.tinkerpop.gremlin.process.traversal.Traverser; -import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; -import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser; -import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.io.GraphReader; -import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; -import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; -import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; -import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; -import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; -import org.apache.tinkerpop.gremlin.structure.util.StringFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.reflect.Method; - -/** - * Handles read and write operations into the {@link Graph}. - * - * @author Stephen Mallette (http://stephen.genoprime.com) - */ -public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting { - - private enum Format { - GRYO, - GRAPHSON, - GRAPHML - } - - private Parameters parameters = new Parameters(); - private boolean first = true; - private String file; - private Mode mode = Mode.UNSET; - - public IoStep(final Traversal.Admin traversal, final String file) { - super(traversal); - - if (null == file || file.isEmpty()) - throw new IllegalArgumentException("file cannot be null or empty"); - - this.file = file; - } - - @Override - public void setMode(final Mode mode) { - this.mode = mode; - } - - @Override - public Mode getMode() { - return mode; - } - - @Override - public String getFile() { - return file; - } - - @Override - public Parameters getParameters() { - return this.parameters; - } - - @Override - public void configure(final Object... keyValues) { - this.parameters.set(null, keyValues); - } - - @Override - protected Traverser.Admin<S> processNextStart() { - if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()"); - if (!this.first) throw FastNoSuchElementException.instance(); - - this.first = false; - final File file = new File(this.file); - - if (mode == Mode.READING) { - if (!file.exists()) throw new IllegalStateException(this.file + " does not exist"); - return read(file); - } else if (mode == Mode.WRITING) { - return write(file); - } else { - throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name()); - } - } - - private Traverser.Admin<S> write(final File file) { - try (final OutputStream stream = new FileOutputStream(file)) { - final Graph graph = (Graph) this.traversal.getGraph().get(); - constructWriter().writeGraph(stream, graph); - - return EmptyTraverser.instance(); - } catch (IOException ioe) { - throw new IllegalStateException(String.format("Could not write file %s from graph", this.file), ioe); - } - } - - private Traverser.Admin<S> read(final File file) { - try (final InputStream stream = new FileInputStream(file)) { - final Graph graph = (Graph) this.traversal.getGraph().get(); - constructReader().readGraph(stream, graph); - - return EmptyTraverser.instance(); - } catch (IOException ioe) { - throw new IllegalStateException(String.format("Could not read file %s into graph", this.file), ioe); - } - } - - /** - * Builds a {@link GraphReader} instance to use. Attempts to detect the file format to be read using the file - * extension or simply uses configurations provided by the user on the parameters given to the step. - */ - private GraphReader constructReader() { - final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0); - if (objectOrClass instanceof GraphReader) - return (GraphReader) objectOrClass; - else if (objectOrClass instanceof String) { - if (objectOrClass.equals(IO.graphson)) - return GraphSONReader.build().create(); - else if (objectOrClass.equals(IO.gryo)) - return GryoReader.build().create(); - else if (objectOrClass.equals(IO.graphml)) - return GraphMLReader.build().create(); - else { - try { - final Class<?> graphReaderClazz = Class.forName((String) objectOrClass); - final Method build = graphReaderClazz.getMethod("build"); - final GraphReader.ReaderBuilder builder = (GraphReader.ReaderBuilder) build.invoke(null); - return builder.create(); - } catch (Exception ex) { - throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex); - } - } - } else { - throw new IllegalStateException("GraphReader could not be determined"); - } - } - - private GraphReader detectReader() { - if (file.endsWith(".kryo")) - return GryoReader.build().create(); - else if (file.endsWith(".json")) - return GraphSONReader.build().create(); - else if (file.endsWith(".xml")) - return GraphMLReader.build().create(); - else - throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); - } - - /** - * Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file - * extension or simply uses configurations provided by the user on the parameters given to the step. - */ - private GraphWriter constructWriter() { - final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0); - if (objectOrClass instanceof GraphWriter) - return (GraphWriter) objectOrClass; - else if (objectOrClass instanceof String) { - if (objectOrClass.equals(IO.graphson)) - return GraphSONWriter.build().create(); - else if (objectOrClass.equals(IO.gryo)) - return GryoWriter.build().create(); - else if (objectOrClass.equals(IO.graphml)) - return GraphMLWriter.build().create(); - else { - try { - final Class<?> graphWriterClazz = Class.forName((String) objectOrClass); - final Method build = graphWriterClazz.getMethod("build"); - final GraphWriter.WriterBuilder builder = (GraphWriter.WriterBuilder) build.invoke(null); - return builder.create(); - } catch (Exception ex) { - throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex); - } - } - } else { - throw new IllegalStateException("GraphReader could not be determined"); - } - } - - private GraphWriter detectWriter() { - if (file.endsWith(".kryo")) - return GryoWriter.build().create(); - else if (file.endsWith(".json")) - return GraphSONWriter.build().create(); - else if (file.endsWith(".xml")) - return GraphMLWriter.build().create(); - else - throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension"); - } - - @Override - public int hashCode() { - final int hash = super.hashCode() ^ this.parameters.hashCode(); - return (null != this.file) ? (hash ^ file.hashCode()) : hash; - } - - @Override - public String toString() { - return StringFactory.stepString(this, file, this.parameters); - } - - @Override - public IoStep clone() { - final IoStep clone = (IoStep) super.clone(); - clone.parameters = this.parameters.clone(); - clone.file = this.file; - clone.mode = this.mode; - return clone; - } -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java new file mode 100644 index 0000000..74b295b --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect; + +import org.apache.tinkerpop.gremlin.process.traversal.IO; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser; +import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Method; + +/** + * Handles read and write operations into the {@link Graph}. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting { + + private enum Format { + GRYO, + GRAPHSON, + GRAPHML + } + + private Parameters parameters = new Parameters(); + private boolean first = true; + private String file; + private Mode mode = Mode.UNSET; + + public IoStep(final Traversal.Admin traversal, final String file) { + super(traversal); + + if (null == file || file.isEmpty()) + throw new IllegalArgumentException("file cannot be null or empty"); + + this.file = file; + } + + @Override + public void setMode(final Mode mode) { + this.mode = mode; + } + + @Override + public Mode getMode() { + return mode; + } + + @Override + public String getFile() { + return file; + } + + @Override + public Parameters getParameters() { + return this.parameters; + } + + @Override + public void configure(final Object... keyValues) { + this.parameters.set(null, keyValues); + } + + @Override + protected Traverser.Admin<S> processNextStart() { + if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()"); + if (!this.first) throw FastNoSuchElementException.instance(); + + this.first = false; + final File file = new File(this.file); + + if (mode == Mode.READING) { + if (!file.exists()) throw new IllegalStateException(this.file + " does not exist"); + return read(file); + } else if (mode == Mode.WRITING) { + return write(file); + } else { + throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name()); + } + } + + private Traverser.Admin<S> write(final File file) { + try (final OutputStream stream = new FileOutputStream(file)) { + final Graph graph = (Graph) this.traversal.getGraph().get(); + constructWriter().writeGraph(stream, graph); + + return EmptyTraverser.instance(); + } catch (IOException ioe) { + throw new IllegalStateException(String.format("Could not write file %s from graph", this.file), ioe); + } + } + + private Traverser.Admin<S> read(final File file) { + try (final InputStream stream = new FileInputStream(file)) { + final Graph graph = (Graph) this.traversal.getGraph().get(); + constructReader().readGraph(stream, graph); + + return EmptyTraverser.instance(); + } catch (IOException ioe) { + throw new IllegalStateException(String.format("Could not read file %s into graph", this.file), ioe); + } + } + + /** + * Builds a {@link GraphReader} instance to use. Attempts to detect the file format to be read using the file + * extension or simply uses configurations provided by the user on the parameters given to the step. + */ + private GraphReader constructReader() { + final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0); + if (objectOrClass instanceof GraphReader) + return (GraphReader) objectOrClass; + else if (objectOrClass instanceof String) { + if (objectOrClass.equals(IO.graphson)) + return GraphSONReader.build().create(); + else if (objectOrClass.equals(IO.gryo)) + return GryoReader.build().create(); + else if (objectOrClass.equals(IO.graphml)) + return GraphMLReader.build().create(); + else { + try { + final Class<?> graphReaderClazz = Class.forName((String) objectOrClass); + final Method build = graphReaderClazz.getMethod("build"); + final GraphReader.ReaderBuilder builder = (GraphReader.ReaderBuilder) build.invoke(null); + return builder.create(); + } catch (Exception ex) { + throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex); + } + } + } else { + throw new IllegalStateException("GraphReader could not be determined"); + } + } + + private GraphReader detectReader() { + if (file.endsWith(".kryo")) + return GryoReader.build().create(); + else if (file.endsWith(".json")) + return GraphSONReader.build().create(); + else if (file.endsWith(".xml")) + return GraphMLReader.build().create(); + else + throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); + } + + /** + * Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file + * extension or simply uses configurations provided by the user on the parameters given to the step. + */ + private GraphWriter constructWriter() { + final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0); + if (objectOrClass instanceof GraphWriter) + return (GraphWriter) objectOrClass; + else if (objectOrClass instanceof String) { + if (objectOrClass.equals(IO.graphson)) + return GraphSONWriter.build().create(); + else if (objectOrClass.equals(IO.gryo)) + return GryoWriter.build().create(); + else if (objectOrClass.equals(IO.graphml)) + return GraphMLWriter.build().create(); + else { + try { + final Class<?> graphWriterClazz = Class.forName((String) objectOrClass); + final Method build = graphWriterClazz.getMethod("build"); + final GraphWriter.WriterBuilder builder = (GraphWriter.WriterBuilder) build.invoke(null); + return builder.create(); + } catch (Exception ex) { + throw new IllegalStateException(String.format("Could not construct the specified GraphReader of %s", objectOrClass), ex); + } + } + } else { + throw new IllegalStateException("GraphReader could not be determined"); + } + } + + private GraphWriter detectWriter() { + if (file.endsWith(".kryo")) + return GryoWriter.build().create(); + else if (file.endsWith(".json")) + return GraphSONWriter.build().create(); + else if (file.endsWith(".xml")) + return GraphMLWriter.build().create(); + else + throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension"); + } + + @Override + public int hashCode() { + final int hash = super.hashCode() ^ this.parameters.hashCode(); + return (null != this.file) ? (hash ^ file.hashCode()) : hash; + } + + @Override + public String toString() { + return StringFactory.stepString(this, file, this.parameters); + } + + @Override + public IoStep clone() { + final IoStep clone = (IoStep) super.clone(); + clone.parameters = this.parameters.clone(); + clone.file = this.file; + clone.mode = this.mode; + return clone; + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java deleted file mode 100644 index 97fdea4..0000000 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map; - -import org.apache.hadoop.mapred.InputFormat; -import org.apache.tinkerpop.gremlin.hadoop.Constants; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; -import org.apache.tinkerpop.gremlin.process.computer.GraphFilter; -import org.apache.tinkerpop.gremlin.process.computer.Memory; -import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; -import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; -import org.apache.tinkerpop.gremlin.process.traversal.IO; -import org.apache.tinkerpop.gremlin.process.traversal.Traversal; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.io.GraphReader; -import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; -import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; -import org.apache.tinkerpop.gremlin.structure.util.StringFactory; - -import java.lang.reflect.Method; - -/** - * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the - * {@link CloneVertexProgram} for its implementation. - * - * @author Stephen Mallette (http://stephen.genoprime.com) - */ -public class HadoopIoStep extends VertexProgramStep implements ReadWriting { - - private Parameters parameters = new Parameters(); - private Mode mode = Mode.UNSET; - private String file; - - public HadoopIoStep(final Traversal.Admin traversal, final String file) { - super(traversal); - this.file = file; - } - - @Override - public void setMode(final Mode mode) { - this.mode = mode; - } - - @Override - public Mode getMode() { - return mode; - } - - @Override - public String getFile() { - return file; - } - - @Override - public void configure(final Object... keyValues) { - // TODO: probably should write to the Configuration selectively - no need for actual Parameters????????? - this.parameters.set(null, keyValues); - } - - @Override - public Parameters getParameters() { - return parameters; - } - - @Override - public String toString() { - return StringFactory.stepString(this, new GraphFilter(this.computer)); - } - - @Override - public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) { - if (mode == Mode.UNSET) - throw new IllegalStateException("IO mode was not set to read() or write()"); - else if (mode == Mode.READING) - configureForRead(graph); - else if (mode == Mode.WRITING) - configureForWrite(graph); - else - throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name()); - - return CloneVertexProgram.build().create(graph); - } - - @Override - public HadoopIoStep clone() { - return (HadoopIoStep) super.clone(); - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - private void configureForRead(final Graph graph) { - final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0); - String inputFormatClassName; - if (inputFormatClassNameOrKeyword.equals(IO.graphson)) - inputFormatClassName = GraphSONInputFormat.class.getName(); - else if (inputFormatClassNameOrKeyword.equals(IO.gryo)) - inputFormatClassName = GryoInputFormat.class.getName(); - else if (inputFormatClassNameOrKeyword.equals(IO.graphml)) - throw new IllegalStateException("GraphML is not a supported file format for OLAP"); - else - inputFormatClassName = inputFormatClassNameOrKeyword; - - graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName); - graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file); - } - - private void configureForWrite(final Graph graph) { - final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0); - String outputFormatClassName; - if (outputFormatClassNameOrKeyword.equals(IO.graphson)) - outputFormatClassName = GraphSONOutputFormat.class.getName(); - else if (outputFormatClassNameOrKeyword.equals(IO.gryo)) - outputFormatClassName = GryoOutputFormat.class.getName(); - else if (outputFormatClassNameOrKeyword.equals(IO.graphml)) - throw new IllegalStateException("GraphML is not a supported file format for OLAP"); - else - outputFormatClassName = outputFormatClassNameOrKeyword; - - graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName); - graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file); - } - - private String detectReader() { - if (file.endsWith(".kryo")) - return GryoInputFormat.class.getName(); - else if (file.endsWith(".json")) - return GraphSONInputFormat.class.getName(); - else if (file.endsWith(".xml")) - throw new IllegalStateException("GraphML is not a supported file format for OLAP"); - else - throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); - } - - private String detectWriter() { - if (file.endsWith(".kryo")) - return GryoOutputFormat.class.getName(); - else if (file.endsWith(".json")) - return GraphSONOutputFormat.class.getName(); - else if (file.endsWith(".xml")) - throw new IllegalStateException("GraphML is not a supported file format for OLAP"); - else - throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); - } -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java new file mode 100644 index 0000000..ca369b6 --- /dev/null +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.sideEffect; + +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; +import org.apache.tinkerpop.gremlin.process.computer.GraphFilter; +import org.apache.tinkerpop.gremlin.process.computer.Memory; +import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; +import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; +import org.apache.tinkerpop.gremlin.process.traversal.IO; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +/** + * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the + * {@link CloneVertexProgram} for its implementation. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class HadoopIoStep extends VertexProgramStep implements ReadWriting { + + private Parameters parameters = new Parameters(); + private Mode mode = Mode.UNSET; + private String file; + + public HadoopIoStep(final Traversal.Admin traversal, final String file) { + super(traversal); + this.file = file; + } + + @Override + public void setMode(final Mode mode) { + this.mode = mode; + } + + @Override + public Mode getMode() { + return mode; + } + + @Override + public String getFile() { + return file; + } + + @Override + public void configure(final Object... keyValues) { + // TODO: probably should write to the Configuration selectively - no need for actual Parameters????????? + this.parameters.set(null, keyValues); + } + + @Override + public Parameters getParameters() { + return parameters; + } + + @Override + public String toString() { + return StringFactory.stepString(this, new GraphFilter(this.computer)); + } + + @Override + public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) { + if (mode == Mode.UNSET) + throw new IllegalStateException("IO mode was not set to read() or write()"); + else if (mode == Mode.READING) + configureForRead(graph); + else if (mode == Mode.WRITING) + configureForWrite(graph); + else + throw new IllegalStateException("Invalid ReadWriting.Mode configured in IoStep: " + mode.name()); + + return CloneVertexProgram.build().create(graph); + } + + @Override + public HadoopIoStep clone() { + return (HadoopIoStep) super.clone(); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + private void configureForRead(final Graph graph) { + final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0); + String inputFormatClassName; + if (inputFormatClassNameOrKeyword.equals(IO.graphson)) + inputFormatClassName = GraphSONInputFormat.class.getName(); + else if (inputFormatClassNameOrKeyword.equals(IO.gryo)) + inputFormatClassName = GryoInputFormat.class.getName(); + else if (inputFormatClassNameOrKeyword.equals(IO.graphml)) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + inputFormatClassName = inputFormatClassNameOrKeyword; + + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName); + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file); + } + + private void configureForWrite(final Graph graph) { + final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0); + String outputFormatClassName; + if (outputFormatClassNameOrKeyword.equals(IO.graphson)) + outputFormatClassName = GraphSONOutputFormat.class.getName(); + else if (outputFormatClassNameOrKeyword.equals(IO.gryo)) + outputFormatClassName = GryoOutputFormat.class.getName(); + else if (outputFormatClassNameOrKeyword.equals(IO.graphml)) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + outputFormatClassName = outputFormatClassNameOrKeyword; + + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName); + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file); + } + + private String detectReader() { + if (file.endsWith(".kryo")) + return GryoInputFormat.class.getName(); + else if (file.endsWith(".json")) + return GraphSONInputFormat.class.getName(); + else if (file.endsWith(".xml")) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); + } + + private String detectWriter() { + if (file.endsWith(".kryo")) + return GryoOutputFormat.class.getName(); + else if (file.endsWith(".json")) + return GraphSONOutputFormat.class.getName(); + else if (file.endsWith(".xml")) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/6d05805a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java index cbc9b07..8348410 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java @@ -19,15 +19,14 @@ package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy; -import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map.HadoopIoStep; +import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.sideEffect.HadoopIoStep; import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; import org.apache.tinkerpop.gremlin.process.traversal.Step; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; -import org.apache.tinkerpop.gremlin.process.traversal.step.filter.NoneStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.map.IoStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;