http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingComputation.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingComputation.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingComputation.java new file mode 100644 index 0000000..38a357c --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingComputation.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import java.io.IOException; + +import org.apache.giraph.comm.WorkerClientRequestProcessor; +import org.apache.giraph.graph.GraphState; +import org.apache.giraph.graph.GraphTaskManager; +import org.apache.giraph.graph.Vertex; +import org.apache.giraph.worker.WorkerContext; +import org.apache.giraph.worker.WorkerGlobalCommUsage; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +/** + * The intercepting Computation class to be instrumented as one that extends + * user's actual Computation class, and run by Graft for debugging. + * + * @param <I> Vertex id type. + * @param <V> Vertex value type. + * @param <E> Edge value type. + * @param <M1> Incoming message type. + * @param <M2> Outgoing message type. + */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class BottomInterceptingComputation<I extends WritableComparable, + V extends Writable, E extends Writable, M1 extends Writable, + M2 extends Writable> extends UserComputation<I, V, E, M1, M2> { + + /** + * A flag to quickly decide whether to skip intercepting compute(). + */ + private boolean shouldStopInterceptingCompute; + + @Intercept + @Override + public void initialize(GraphState graphState, + WorkerClientRequestProcessor<I, V, E> workerClientRequestProcessor, + GraphTaskManager<I, V, E> graphTaskManager, + WorkerGlobalCommUsage workerGlobalCommUsage, WorkerContext workerContext) { + try { + // We first call super.initialize so that the getConf() call below + // returns a non-null value. + super.initialize(graphState, workerClientRequestProcessor, + graphTaskManager, workerGlobalCommUsage, workerContext); + } finally { + if (!AbstractInterceptingComputation.IS_INITIALIZED) { // short circuit + initializeAbstractInterceptingComputation(); + } + } + } + + @Intercept + @Override + public void preSuperstep() { + shouldStopInterceptingCompute = interceptPreSuperstepBegin(); + super.preSuperstep(); + } + + @Intercept + @Override + public final void compute(Vertex<I, V, E> vertex, Iterable<M1> messages) + throws IOException { + if (shouldStopInterceptingCompute) { + super.compute(vertex, messages); + } else { + interceptComputeBegin(vertex, messages); + if (AbstractInterceptingComputation.SHOULD_CATCH_EXCEPTIONS) { + // CHECKSTYLE: stop IllegalCatch + try { + super.compute(vertex, messages); + } catch (Throwable e) { + interceptComputeException(vertex, messages, e); + throw e; + } + // CHECKSTYLE: resume IllegalCatch + } else { + super.compute(vertex, messages); + } + shouldStopInterceptingCompute = interceptComputeEnd(vertex, messages); + } + } + + @Intercept + @Override + public void postSuperstep() { + super.postSuperstep(); + interceptPostSuperstepEnd(); + } + + @Override + public Class getActualTestedClass() { + return getClass(); + } + +}
http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingMasterCompute.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingMasterCompute.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingMasterCompute.java new file mode 100644 index 0000000..4b54eb5 --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/BottomInterceptingMasterCompute.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * The intercepting MasterCompute class to be instrumented as one that extends + * user's actual MasterCompute class, and run by Graft for debugging. + */ +public class BottomInterceptingMasterCompute extends UserMasterCompute { + + @Intercept + @Override + public void compute() { + interceptComputeBegin(); + // CHECKSTYLE: stop IllegalCatch + try { + super.compute(); + interceptComputeEnd(); + } catch (Exception e) { + interceptComputeException(e); + throw e; + } + // CHECKSTYLE: resume IllegalCatch + } + + @Override + public void readFields(DataInput in) throws IOException { + } + + @Override + public void write(DataOutput out) throws IOException { + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/CommonVertexMasterInterceptionUtil.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/CommonVertexMasterInterceptionUtil.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/CommonVertexMasterInterceptionUtil.java new file mode 100644 index 0000000..dbaff4b --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/CommonVertexMasterInterceptionUtil.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; +import org.apache.giraph.debugger.utils.AggregatedValueWrapper; +import org.apache.giraph.debugger.utils.BaseWrapper; +import org.apache.giraph.debugger.utils.CommonVertexMasterContextWrapper; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Writable; +import org.apache.log4j.Logger; + +/** + * Common class used by both {@link AbstractInterceptingComputation} and + * {@link AbstractInterceptingMasterCompute}. Serves following functions: + * <ul> + * <li>Maintains a {@link CommonVertexMasterContextWrapper} which contains + * common information captured by both the Master and the Vertex class, such as + * aggregators that the user accesses, superstepNo, totalNumberOfVertices and + * edges. + * <li>Contains helper methods to save a master or vertex trace file to HDFS and + * maintains a {@link FileSystem} object that can be used to write other traces + * to HDFS. + * <li>Contains a helper method to return the trace directory for a particular + * job. + * </ul> + * + * TODO: We might consider adding a method to {@link AbstractComputation} and + * {@link MasterCompute} to return all registered aggregators, such as + * getAllRegisteredAggregators. Right now we do not intercept aggregators that + * were never called. + */ +@SuppressWarnings("rawtypes") +public class CommonVertexMasterInterceptionUtil { + /** + * Logger for this class. + */ + private static final Logger LOG = Logger + .getLogger(CommonVertexMasterInterceptionUtil.class); + /** + * The HDFS file system instance to load and save data for debugging. + */ + private static FileSystem FILE_SYSTEM = null; + /** + * The Giraph job id of the job being debugged. + */ + private final String jobId; + /** + * A list of Giraph aggregator values. + */ + private List<AggregatedValueWrapper> previousAggregatedValueWrappers; + /** + * The master context being captured. + */ + private CommonVertexMasterContextWrapper commonVertexMasterContextWrapper; + + /** + * Constructs a new instance for the given job. + * + * Warning: Caller's should create a new object at least once each superstep. + * + * @param jobId The job id of the job being debugged. + */ + public CommonVertexMasterInterceptionUtil(String jobId) { + this.jobId = jobId; + previousAggregatedValueWrappers = new ArrayList<>(); + if (FILE_SYSTEM == null) { + try { + FILE_SYSTEM = FileSystem.get(new Configuration()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Initializes this instance. + * + * @param immutableClassesConfig The Giraph configuration. + * @param superstepNo The superstep number. + * @param totalNumVertices Total number of vertices at this superstep. + * @param totalNumEdges Total number of edges at this superstep. + */ + public void initCommonVertexMasterContextWrapper( + ImmutableClassesGiraphConfiguration immutableClassesConfig, + long superstepNo, long totalNumVertices, long totalNumEdges) { + this.commonVertexMasterContextWrapper = new + CommonVertexMasterContextWrapper( + immutableClassesConfig, superstepNo, totalNumVertices, totalNumEdges); + commonVertexMasterContextWrapper + .setPreviousAggregatedValues(previousAggregatedValueWrappers); + } + + /** + * Captures value of a Giraph aggregator. + * + * @param <A> The aggregator value type. + * @param name The Giraph aggregator name. + * @param value The aggregator value to capture. + */ + public <A extends Writable> void addAggregatedValueIfNotExists(String name, + A value) { + if (getPreviousAggregatedValueWrapper(name) == null && value != null) { + previousAggregatedValueWrappers.add(new AggregatedValueWrapper(name, + value)); + } + } + + /** + * Returns captured values of a Giraph aggregator. + * + * @param name The Giraph aggregator name. + * @return The captured aggregator values. + */ + private AggregatedValueWrapper getPreviousAggregatedValueWrapper(String name) + { + for (AggregatedValueWrapper previousAggregatedValueWrapper : + previousAggregatedValueWrappers) { + if (name.equals(previousAggregatedValueWrapper.getKey())) { + return previousAggregatedValueWrapper; + } + } + return null; + } + + /** + * Saves captured scenario. + * + * @param masterOrVertexScenarioWrapper The scenario to save. + * @param fullFileName HDFS path for the saved file. + */ + public void saveScenarioWrapper(BaseWrapper masterOrVertexScenarioWrapper, + String fullFileName) { + try { + masterOrVertexScenarioWrapper.saveToHDFS(FILE_SYSTEM, fullFileName); + } catch (IOException e) { + LOG.error("Could not save the " + + masterOrVertexScenarioWrapper.getClass().getName() + + " protobuf trace. IOException was thrown. exceptionMessage: " + + e.getMessage()); + e.printStackTrace(); + } + } + + public List<AggregatedValueWrapper> getPreviousAggregatedValueWrappers() { + return previousAggregatedValueWrappers; + } + + public void setPreviousAggregatedValueWrappers( + ArrayList<AggregatedValueWrapper> previousAggregatedValueWrappers) { + this.previousAggregatedValueWrappers = previousAggregatedValueWrappers; + } + + public CommonVertexMasterContextWrapper getCommonVertexMasterContextWrapper() + { + return commonVertexMasterContextWrapper; + } + + public FileSystem getFileSystem() { + return FILE_SYSTEM; + } + + public String getJobId() { + return jobId; + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/InstrumentGiraphClasses.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/InstrumentGiraphClasses.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/InstrumentGiraphClasses.java new file mode 100644 index 0000000..0817eca --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/InstrumentGiraphClasses.java @@ -0,0 +1,413 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.util.Collection; + +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtMethod; +import javassist.NotFoundException; +import javassist.bytecode.ConstPool; +import javassist.bytecode.Descriptor; + +import org.apache.giraph.graph.Computation; +import org.apache.giraph.master.MasterCompute; +import org.apache.log4j.Logger; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +/** + * The main class that instruments user's ordinary Giraph Computation class with + * necessary changes for debugging. + */ +public class InstrumentGiraphClasses { + + /** + * Logger for this class. + */ + private static Logger LOG = Logger.getLogger(InstrumentGiraphClasses.class); + + /** + * Suffix to append to the original class names. + */ + private static final String ORIGINAL_CLASS_NAME_SUFFIX = System.getProperty( + "giraph.debugger.classNameSuffix", "Original"); + /** + * Path prefix for determining the temporary directory name used for + * instrumentation. + */ + private static final String TMP_DIR_NAME_PREFIX = + InstrumentGiraphClasses.class.getSimpleName(); + + /** + * Disallowing instantiation. + */ + private InstrumentGiraphClasses() { } + + /** + * Main entry point for instrumenting Giraph Computation and MasterCompute + * classes with Graft classes. + * + * @param args Command-line arguments. + * @throws IOException + * @throws ClassNotFoundException + */ + public static void main(String[] args) throws IOException, + ClassNotFoundException { + // CHECKSTYLE: stop Regexp + if (args.length < 1) { + System.err.println("Usage: java ... " + TMP_DIR_NAME_PREFIX + + " GIRAPH_COMPUTATION_CLASS_NAME [OUTPUT_DIR]"); + System.exit(1); + } + + Collection<String> userComputationClassNames = Sets.newHashSet(args[0]); + String outputDir = args.length > 1 ? args[1] : null; + String masterComputeClassName = args.length > 2 ? args[2] : null; + // Additional Computation classes + boolean shouldAnalyzeMaster = masterComputeClassName != null; + for (int i = 3; i < args.length; i++) { + userComputationClassNames.add(args[i]); + // Don't analyze the MasterCompute class when a chosen list of + // Computation classes were given + shouldAnalyzeMaster = false; + } + + try { + Collection<CtClass> classesModified = Sets.newHashSet(); + ClassPool classPool = ClassPool.getDefault(); + if (masterComputeClassName != null) { + if (shouldAnalyzeMaster) { + // Collect all Computation class names referenced in + // masterComputeClassName + LOG.info("Analyzing MasterCompute class: " + masterComputeClassName); + userComputationClassNames.addAll( + collectComputationClassNames( + masterComputeClassName, classPool)); + } + LOG + .info("Instrumenting MasterCompute class: " + masterComputeClassName); + classesModified.addAll( + instrumentSandwich(masterComputeClassName, + AbstractInterceptingMasterCompute.class.getName(), + UserMasterCompute.class.getName(), + BottomInterceptingMasterCompute.class.getName(), classPool)); + } + for (String userComputationClassName : userComputationClassNames) { + LOG + .info("Instrumenting Computation class: " + userComputationClassName); + classesModified.addAll( + instrumentSandwich(userComputationClassName, + AbstractInterceptingComputation.class.getName(), + UserComputation.class.getCanonicalName(), + BottomInterceptingComputation.class.getName(), classPool)); + } + + // Finally, write the modified classes so that a new jar can be + // created or an existing one can be updated. + String jarRoot = outputDir != null ? outputDir : Files + .createTempDirectory(TMP_DIR_NAME_PREFIX).toString(); + LOG.info("Writing " + classesModified.size() + + " instrumented classes to " + jarRoot); + for (CtClass c : classesModified) { + LOG.debug(" writing class " + c.getName()); + c.writeFile(jarRoot); + } + + LOG.info("Finished instrumentation"); + + if (outputDir == null) { + // Show where we produced the instrumented .class files (unless + // specified) + System.out.println(jarRoot); + } + System.exit(0); + } catch (NotFoundException e) { + e.printStackTrace(); + System.err + .println("Some Giraph Computation or MasterCompute classes " + + "were not found"); + System.exit(1); + } catch (CannotCompileException e) { + e.printStackTrace(); + System.err + .println("Cannot instrument the given Giraph Computation or " + + "MasterCompute classes"); + System.exit(2); + } catch (IOException e) { + e.printStackTrace(); + System.err + .println("Cannot write the instrumented Giraph Computation and/or " + + "MasterCompute classes"); + System.exit(4); + } + // CHECKSTYLE: resume Regexp + } + + /** + * Finds all class names referenced in the master compute class. + * + * @param masterComputeClassName The name of the master compute class. + * @param classPool The Javassist class pool being used. + * @return A collection of class names. + * @throws NotFoundException + */ + protected static Collection<String> collectComputationClassNames( + String masterComputeClassName, ClassPool classPool) + throws NotFoundException { + Collection<String> classNames = Lists.newArrayList(); + CtClass computationClass = classPool.get(Computation.class.getName()); + CtClass rootMasterComputeClass = classPool.get(MasterCompute.class + .getName()); + CtClass mc = classPool.get(masterComputeClassName); + while (mc != null && !mc.equals(rootMasterComputeClass)) { + // find all class names appearing in the master compute class + @SuppressWarnings("unchecked") + Collection<String> classNamesInMasterCompute = Lists.newArrayList(mc + .getRefClasses()); + // as well as in string literals + ConstPool constPool = mc.getClassFile().getConstPool(); + for (int i = 1; i < constPool.getSize(); i++) { + switch (constPool.getTag(i)) { + case ConstPool.CONST_String: + classNamesInMasterCompute.add(constPool.getStringInfo(i)); + break; + default: + break; + } + } + // collect Computation class names + for (String className : classNamesInMasterCompute) { + // CHECKSTYLE: stop IllegalCatch + try { + if (classPool.get(className).subtypeOf(computationClass)) { + classNames.add(className); + } + } catch (NotFoundException e) { + // ignored + assert true; + } catch (Exception e) { + e.printStackTrace(); + } + // CHECKSTYLE: resume IllegalCatch + } + + // repeat above for all superclasses of given masterComputeClass + mc = mc.getSuperclass(); + } + return classNames; + } + + /** + * Instruments the given class with a bottom and top class (sandwich), + * rewiring the inheritance chain and references. + * + * @param targetClassName + * The class to be placed in the middle of the sandwich + * instrumentation. + * @param topClassName + * The class to be placed as a base class (on top) of the target + * class (middle), which intercepts any calls made from the target + * class. + * @param mockTargetClassName + * The name of the mock class used to compile the top class. + * @param bottomClassName + * The class to be placed as a extending class (beneath) of the + * target class, which will receive any calls to the target class + * first. + * @param classPool + * The Javassist class pool being used. + * @return A collection of instrumented Javassist CtClasses. + * @throws NotFoundException + * @throws CannotCompileException + * @throws ClassNotFoundException + */ + protected static Collection<CtClass> instrumentSandwich( + String targetClassName, String topClassName, String mockTargetClassName, + String bottomClassName, ClassPool classPool) throws NotFoundException, + CannotCompileException, ClassNotFoundException { + Collection<CtClass> classesModified = Sets.newHashSet(); + // Load the involved classes with Javassist + LOG.debug("Looking for classes to instrument: " + targetClassName + "..."); + String alternativeClassName = targetClassName + ORIGINAL_CLASS_NAME_SUFFIX; + CtClass targetClass = classPool.getAndRename(targetClassName, + alternativeClassName); + // We need two classes: one at the bottom (subclass) and + // another at the top (superclass). + CtClass topClass = classPool.get(topClassName); + CtClass bottomClass = classPool.getAndRename(bottomClassName, + targetClassName); + + LOG.debug(" target class to instrument (targetClass):\n" + + getGenericsName(targetClass)); + LOG.debug(" class to instrument at top (topClass):\n" + + getGenericsName(topClass)); + LOG.debug(" class to instrument at bottom (bottomClass):\n" + + getGenericsName(bottomClass)); + + // 1. To intercept other Giraph API calls by user's computation + // class: + // We need to inject a superclass at the highest level of the + // inheritance hierarchy, i.e., make the top class become a + // superclass of the class that directly extends + // AbstractComputation. + // 1-a. Find the user's base class that extends the top class' + // superclass. + LOG.debug("Looking for user's top class that extends " + + getGenericsName(topClass.getSuperclass())); + CtClass targetTopClass = targetClass; + while (!targetTopClass.getName().equals(Object.class.getName()) && + !targetTopClass.getSuperclass().equals(topClass.getSuperclass())) { + targetTopClass = targetTopClass.getSuperclass(); + } + if (targetTopClass.getName().equals(Object.class.getName())) { + throw new NotFoundException(targetClass.getName() + " must extend " + + topClass.getSuperclass().getName()); + } + LOG.debug(" class to inject topClass on top of (targetTopClass):\n" + + getGenericsName(targetTopClass)); + // 1-b. Mark user's class as abstract and erase any final modifier. + LOG.debug("Marking targetClass as abstract and non-final..."); + int modClass = targetClass.getModifiers(); + modClass |= Modifier.ABSTRACT; + modClass &= ~Modifier.FINAL; + targetClass.setModifiers(modClass); + classesModified.add(targetClass); + if (!targetTopClass.equals(topClass)) { + // 1-c. Inject the top class by setting it as the superclass of + // user's class that extends its superclass (AbstractComputation). + LOG.debug("Injecting topClass on top of targetTopClass..."); + targetTopClass.setSuperclass(topClass); + targetTopClass.replaceClassName(topClass.getSuperclass().getName(), + topClass.getName()); + classesModified.add(targetTopClass); + // XXX Unless we take care of generic signature as well, + // GiraphConfigurationValidator will complain. + String jvmNameForTopClassSuperclass = Descriptor.of( + topClass.getSuperclass()).replaceAll(";$", ""); + String jvmNameForTopClass = Descriptor.of(topClass) + .replaceAll(";$", ""); + String genSig = targetTopClass.getGenericSignature(); + if (genSig != null) { + String genSig2 = genSig.replace(jvmNameForTopClassSuperclass, + jvmNameForTopClass); + targetTopClass.setGenericSignature(genSig2); + } + } + // 1-d. Then, make the bottomClass extend user's computation, taking + // care of generics signature as well. + LOG.debug("Attaching bottomClass beneath targetClass..."); + bottomClass.replaceClassName(mockTargetClassName, targetClass.getName()); + bottomClass.setSuperclass(targetClass); + bottomClass.setGenericSignature(Descriptor.of(targetClass)); + classesModified.add(bottomClass); + + // 2. To intercept compute() and other calls that originate from + // Giraph: + // We need to extend user's computation class pass that as the + // computation class to Giraph. The new subclass will override + // compute(), so we may have to remove the "final" marker on user's + // compute() method. + // 2-a. Find all methods that we override in bottomClass. + LOG.debug("For each method to intercept," + + " changing Generics signature of bottomClass, and" + + " erasing final modifier..."); + for (CtMethod overridingMethod : bottomClass.getMethods()) { + if (!overridingMethod.hasAnnotation(Intercept.class)) { + continue; + } + Intercept annotation = (Intercept) overridingMethod + .getAnnotation(Intercept.class); + String targetMethodName = annotation.renameTo(); + if (targetMethodName == null || targetMethodName.isEmpty()) { + targetMethodName = overridingMethod.getName(); + } + // 2-b. Copy generics signature to the overriding method if + // necessary. + CtMethod targetMethod = targetClass.getMethod(targetMethodName, + overridingMethod.getSignature()); + LOG.debug(" from: " + overridingMethod.getName() + + overridingMethod.getGenericSignature()); + LOG.debug(" to: " + targetMethod.getName() + + targetMethod.getGenericSignature()); + if (overridingMethod.getGenericSignature() != null) { + overridingMethod + .setGenericSignature(targetMethod.getGenericSignature()); + classesModified.add(overridingMethod.getDeclaringClass()); + } + // 2-c. Remove final marks from them. + int modMethod = targetMethod.getModifiers(); + if ((modMethod & Modifier.FINAL) == 0) { + continue; + } + modMethod &= ~Modifier.FINAL; + targetMethod.setModifiers(modMethod); + LOG.debug(" erasing final modifier from " + targetMethod.getName() + + "() of " + targetMethod.getDeclaringClass()); + // 2-d. Rename the overriding method if necessary. + if (!overridingMethod.getName().equals(targetMethodName)) { + overridingMethod.setName(targetMethodName); + classesModified.add(overridingMethod.getDeclaringClass()); + } + // 2-e. Remember them for later. + classesModified.add(targetMethod.getDeclaringClass()); + } + LOG.debug("Finished instrumenting " + targetClassName); + LOG.debug(" topClass=\n" + getGenericsName(topClass) + "\n" + + topClass); + LOG.debug(" targetTopClass=\n" + getGenericsName(targetTopClass) + + "\n" + targetTopClass); + LOG.debug(" targetClass=\n" + getGenericsName(targetClass) + "\n" + + targetClass); + LOG.debug(" bottomClass=\n" + getGenericsName(bottomClass) + "\n" + + bottomClass); + return classesModified; + } + + /** + * Format Javassist class name for log messages. + * @param clazz The Javassist class. + * @return Formatted name of the class including Generic type signature. + */ + protected static String getGenericsName(CtClass clazz) { + return clazz.getName() + " (" + clazz.getGenericSignature() + ")"; + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/Intercept.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/Intercept.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/Intercept.java new file mode 100644 index 0000000..07ccaeb --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/Intercept.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation normally for the {@link BottomInterceptingComputation} class to + * communicate which methods are to be intercepted and checked by the + * instrumenter. + * + * author netj + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@interface Intercept { + /** + * The name to which this method should be renamed by the instrumenter. This + * is for telling the instrumenter to rename certain methods that are not + * normally overridable because of its final modifier. + */ + String renameTo() default ""; + +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserComputation.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserComputation.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserComputation.java new file mode 100644 index 0000000..9f45c2a --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserComputation.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import java.io.IOException; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.giraph.graph.Vertex; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; + +/** + * A dummy Computation class that will sit between the + * {@link AbstractInterceptingComputation} class at the top and the + * {@link BottomInterceptingComputation} at the bottom. + * + * author netj + * + * @param <I> Vertex id type. + * @param <V> Vertex value type. + * @param <E> Edge value type. + * @param <M1> Incoming message type. + * @param <M2> Outgoing message type. + */ +@SuppressWarnings("rawtypes") +public abstract class UserComputation<I extends WritableComparable, + V extends Writable, E extends Writable, + M1 extends Writable, M2 extends Writable> + extends AbstractInterceptingComputation<I, V, E, M1, M2> { + + @Override + public void compute(Vertex<I, V, E> vertex, Iterable<M1> messages) + throws IOException { + throw new NotImplementedException(); + } + +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserMasterCompute.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserMasterCompute.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserMasterCompute.java new file mode 100644 index 0000000..1fb3509 --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/UserMasterCompute.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.instrumenter; + +import org.apache.commons.lang.NotImplementedException; + +/** + * A dummy MasterCompute class that will sit between the + * {@link AbstractInterceptingMasterCompute} class at the top and the + * {@link BottomInterceptingMasterCompute} at the bottom. + * + * author netj + */ +public abstract class UserMasterCompute extends + AbstractInterceptingMasterCompute { + + @Override + public void compute() { + throw new NotImplementedException(); + } + + @Override + public void initialize() throws InstantiationException, + IllegalAccessException { + throw new NotImplementedException(); + } + +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/package-info.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/package-info.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/package-info.java new file mode 100644 index 0000000..a92935d --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/instrumenter/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Classes for instrumenting Giraph programs to be run with debugging code. + */ +package org.apache.giraph.debugger.instrumenter; http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/ComputationComputeTestGenerator.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/ComputationComputeTestGenerator.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/ComputationComputeTestGenerator.java new file mode 100644 index 0000000..32c977d --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/ComputationComputeTestGenerator.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collection; +import java.util.HashMap; + +import org.apache.giraph.debugger.utils.GiraphVertexScenarioWrapper; +import org.apache.giraph.debugger.utils.GiraphVertexScenarioWrapper.VertexContextWrapper; +import org.apache.giraph.debugger.utils.GiraphVertexScenarioWrapper.VertexContextWrapper.OutgoingMessageWrapper; +import org.apache.giraph.debugger.utils.GiraphVertexScenarioWrapper.VertexScenarioClassesWrapper; +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; + +/** + * This is a code generator which can generate the JUnit test cases for a + * Giraph. + */ +public class ComputationComputeTestGenerator extends TestGenerator { + + /** + * Public constructor. + */ + public ComputationComputeTestGenerator() { + super(); + } + + /** + * Generates a unit test file as a string from the given + * {@link GiraphVertexScenarioWrapper} object. + * @param input {@link GiraphVertexScenarioWrapper} object to generate a test + * file from. + * @param testPackage package name for the unit test. + * @return a unit test file as a string from the given + * {@link GiraphVertexScenarioWrapper} object + */ + @SuppressWarnings("rawtypes") + public String generateTest(GiraphVertexScenarioWrapper input, + String testPackage) throws IOException { + return generateTest(input, testPackage, null); + } + + /** + * Generates a unit test file as a string from the given + * {@link GiraphVertexScenarioWrapper} object. + * @param input {@link GiraphVertexScenarioWrapper} object to generate a test + * file from. + * @param testPackage package name for the unit test. + * @param className name of the test class. + * @return a unit test file as a string from the given + * {@link GiraphVertexScenarioWrapper} object + */ + @SuppressWarnings("rawtypes") + public String generateTest(GiraphVertexScenarioWrapper input, + String testPackage, String className) throws IOException { + VelocityContext context = buildContext(input, testPackage, className); + + try (StringWriter sw = new StringWriter()) { + Template template = Velocity.getTemplate("ComputeTestTemplate.vm"); + template.merge(context, sw); + return sw.toString(); + } + } + + /** + * @param scenario scenario object from which a unit test file is being + * generated. + * @return the classUnderTest field inside the unit test. + */ + @SuppressWarnings("rawtypes") + public String generateClassUnderTestField( + GiraphVertexScenarioWrapper scenario) { + return "private " + + scenario.getVertexScenarioClassesWrapper().getClassUnderTest() + .getSimpleName() + " classUnderTest;"; + } + + /** + * @param scenario scenario object from which a unit test file is being + * generated. + * @return the line declaring the ImmutableClassesGiraphConfiguration field + * inside the unit test. + */ + @SuppressWarnings("rawtypes") + public String generateConfField(GiraphVertexScenarioWrapper scenario) { + return String.format( + "private ImmutableClassesGiraphConfiguration<%s, %s, %s> conf;", scenario + .getVertexScenarioClassesWrapper().getVertexIdClass().getSimpleName(), + scenario.getVertexScenarioClassesWrapper().getVertexValueClass() + .getSimpleName(), scenario.getVertexScenarioClassesWrapper() + .getEdgeValueClass().getSimpleName()); + } + + /** + * @param scenario scenario object from which a unit test file is being + * generated. + * @return the line declaring the MockedEnvironment field + * inside the unit test. + */ + @SuppressWarnings("rawtypes") + public String generateMockEnvField(GiraphVertexScenarioWrapper scenario) { + return String.format("private MockedEnvironment<%s, %s, %s, %s> mockEnv;", + scenario.getVertexScenarioClassesWrapper().getVertexIdClass() + .getSimpleName(), scenario.getVertexScenarioClassesWrapper() + .getVertexValueClass().getSimpleName(), scenario + .getVertexScenarioClassesWrapper().getEdgeValueClass().getSimpleName(), + scenario.getVertexScenarioClassesWrapper().getOutgoingMessageClass() + .getSimpleName()); + } + + /** + * @param scenario scenario object from which a unit test file is being + * generated. + * @return the line declaring the WorkerClientRequestProcessor field + * inside the unit test. + */ + @SuppressWarnings("rawtypes") + public String generateProcessorField(GiraphVertexScenarioWrapper scenario) { + return String.format( + "private WorkerClientRequestProcessor<%s, %s, %s> processor;", scenario + .getVertexScenarioClassesWrapper().getVertexIdClass().getSimpleName(), + scenario.getVertexScenarioClassesWrapper().getVertexValueClass() + .getSimpleName(), scenario.getVertexScenarioClassesWrapper() + .getEdgeValueClass().getSimpleName()); + } + + /** + * @param scenario scenario object from which a unit test file is being + * generated. + * @return the line declaring the setup method field inside the unit test. + */ + @SuppressWarnings("rawtypes") + public String generateSetUp(GiraphVertexScenarioWrapper scenario) + throws IOException { + VelocityContext context = buildContext(scenario); + + try (StringWriter sw = new StringWriter()) { + Template template = Velocity.getTemplate("ComputeSetUpFuncTemplate.vm"); + template.merge(context, sw); + return sw.toString(); + } + } + + /** + * @param scenario scenario object from which a unit test file is being + * generated. + * @return the line declaring the testCompute method field inside the unit + * test. + */ + @SuppressWarnings({ "rawtypes" }) + public String generateTestCompute(GiraphVertexScenarioWrapper scenario) + throws IOException { + resetComplexWritableList(); + + VelocityContext context = buildContext(scenario); + + try (StringWriter sw = new StringWriter()) { + Template template = Velocity.getTemplate("ComputeTestFuncTemplate.vm"); + template.merge(context, sw); + return sw.toString(); + } + } + + /** + * Generates the lines that construct {@link Writable} objects that are + * used in the unittest. + * @param className writable object's class name. + * @return lines for constructing the {@link Writable} object. + */ + public String generateReadWritableFromString(String className) + throws IOException { + VelocityContext context = new VelocityContext(); + context.put("class", className); + + try (StringWriter sw = new StringWriter()) { + Template template = Velocity + .getTemplate("ReadWritableFromStringTemplate.vm"); + template.merge(context, sw); + return sw.toString(); + } + } + + /** + * @see #buildContext(GiraphVertexScenarioWrapper, String, String) + * @param giraphScenarioWrapper {@link GiraphVertexScenarioWrapper} object. + * @return {@link VelocityContext} to be used in generating the unittest file. + */ + @SuppressWarnings("rawtypes") + private VelocityContext buildContext( + GiraphVertexScenarioWrapper giraphScenarioWrapper) { + return buildContext(giraphScenarioWrapper, null, null); + } + + /** + * @param giraphScenarioWrapper {@link GiraphVertexScenarioWrapper} object. + * @param testPackage name of the package for the unit test. + * @param className name of the unit test class. + * @return {@link VelocityContext} that will be used to generate the unit + * test. + */ + @SuppressWarnings("rawtypes") + private VelocityContext buildContext( + GiraphVertexScenarioWrapper giraphScenarioWrapper, String testPackage, + String className) { + ComputeContextBuilder builder = new ComputeContextBuilder(); + VertexScenarioClassesWrapper vertexScenarioClassesWrapper + = giraphScenarioWrapper + .getVertexScenarioClassesWrapper(); + builder.addVertexScenarioClassesWrapper(vertexScenarioClassesWrapper); + builder.addTestClassInfo(testPackage, + vertexScenarioClassesWrapper.getClassUnderTest(), className); + builder.addCommonMasterVertexContext(giraphScenarioWrapper + .getContextWrapper().getCommonVertexMasterContextWrapper()); + builder.addVertexTypes(vertexScenarioClassesWrapper); + builder.addVertexData(giraphScenarioWrapper.getContextWrapper()); + return builder.getContext(); + } + + /** + * Wrapper to store information about the "context" of the compute method + * being tested. Stores the vertex id type, vertex value types, message + * types, etc. In addition stores the actual id, vertex values, etc.. + */ + protected class ComputeContextBuilder extends ContextBuilder { + + /** + * @param vertexScenarioClassesWrapper + * {@link VertexScenarioClassesWrapper} object. + */ + @SuppressWarnings("rawtypes") + public void addVertexTypes( + VertexScenarioClassesWrapper vertexScenarioClassesWrapper) { + context.put("vertexIdType", vertexScenarioClassesWrapper + .getVertexIdClass().getSimpleName()); + context.put("vertexValueType", vertexScenarioClassesWrapper + .getVertexValueClass().getSimpleName()); + context.put("edgeValueType", vertexScenarioClassesWrapper + .getEdgeValueClass().getSimpleName()); + context.put("inMsgType", vertexScenarioClassesWrapper + .getIncomingMessageClass().getSimpleName()); + context.put("outMsgType", vertexScenarioClassesWrapper + .getOutgoingMessageClass().getSimpleName()); + } + + /** + * @param vertexContextWrapper {@link VertexContextWrapper} object to read + * vertex data from. + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void addVertexData(VertexContextWrapper vertexContextWrapper) { + context.put("vertexId", vertexContextWrapper.getVertexIdWrapper()); + context.put("vertexValue", + vertexContextWrapper.getVertexValueBeforeWrapper()); + context.put("vertexValueAfter", + vertexContextWrapper.getVertexValueAfterWrapper()); + context.put("inMsgs", vertexContextWrapper.getIncomingMessageWrappers()); + context.put("neighbors", vertexContextWrapper.getNeighborWrappers()); + + HashMap<OutgoingMessageWrapper, OutMsg> outMsgMap = new HashMap<>(); + for (OutgoingMessageWrapper msg : + (Collection<OutgoingMessageWrapper>) + vertexContextWrapper.getOutgoingMessageWrappers()) { + if (outMsgMap.containsKey(msg)) { + outMsgMap.get(msg).incrementTimes(); + } else { + outMsgMap.put(msg, new OutMsg(msg)); + } + } + context.put("outMsgs", outMsgMap.values()); + } + } + + /** + * In-memory representation of the hadoop config values. + */ + public static class Config { + /** + * Key of the configuration flag. + */ + private final String key; + /** + * Value of the configuration flag. + */ + private final Object value; + + /** + * Constructor. + * @param key key of the configuration flag. + * @param value value of the configuration flag. + */ + public Config(String key, Object value) { + this.key = key; + this.value = value; + } + + public String getKey() { + return key; + } + + /** + * @return value of the configuration flag. + */ + public Object getValue() { + if (value instanceof String) { + return "\"" + value + '"'; + } else { + return value; + } + } + + /** + * @return returns type of the configuration's flag, e.g., Int, Float, ... . + */ + public String getClassStr() { + // TODO(brian):additional cases can be added up to the input + if (value instanceof Integer) { + return "Int"; + } else if (value instanceof Long) { + return "Long"; + } else if (value instanceof Float) { + return "Float"; + } else if (value instanceof Boolean) { + return "Boolean"; + } else { + return ""; + } + } + } + + /** + * A wrapper around the {@link OutgoingMessageWrapper} that stores the + * outgoing messages from a vertex and the number of times the message has + * been sent. + */ + @SuppressWarnings("rawtypes") + public static class OutMsg { + /** + * {@link OutgoingMessageWrapper} object. + */ + private final OutgoingMessageWrapper msg; + /** + * How many times the message has been sent. + */ + private int times; + + /** + * Constructor that initializes the outgoing message wrapper to msg and + * the number of times the message has been sent. + * @param msg outgoing message. + */ + public OutMsg(OutgoingMessageWrapper msg) { + this.msg = msg; + this.times = 1; + } + + public OutgoingMessageWrapper getMsg() { + return msg; + } + + public int getTimes() { + return times; + } + + /** + * Increments the number of times this message has been sent by 1. + */ + public void incrementTimes() { + this.times++; + } + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/FormatHelper.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/FormatHelper.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/FormatHelper.java new file mode 100644 index 0000000..cd26ca3 --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/FormatHelper.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import java.math.BigDecimal; +import java.text.DecimalFormat; +import java.util.Set; + +import org.apache.giraph.utils.WritableUtils; +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.ByteWritable; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; + +/** + * Utility class to help format some of the strings used in the generated unit + * test. E.g., if an a float is initialized to 1.3 and we read that from + * a protocolbuffer and write it again, it will be written as 1.299999.... + * So we need to truncate it when we generate the unittest and this class + * contains such methods. + */ +public class FormatHelper { + + /** + * {@link DecimalFormat} to use when formatting decimals. + */ + private final DecimalFormat decimalFormat = new DecimalFormat("#.#####"); + + /** + * A set of complex writable object, i.e., user-defined ones that are not like + * IntWritable, FloatWritable, etc..used by this compute function. + */ + @SuppressWarnings("rawtypes") + private Set<Class> complexWritables; + + /** + * Default constructor. + */ + public FormatHelper() { + } + + /** + * @param complexWritables complex writable objects to register. + */ + @SuppressWarnings("rawtypes") + public FormatHelper(Set<Class> complexWritables) { + registerComplexWritableClassList(complexWritables); + } + + /** + * @param complexWritables complex writable objects to register. + */ + @SuppressWarnings("rawtypes") + public void registerComplexWritableClassList(Set<Class> complexWritables) { + this.complexWritables = complexWritables; + } + + /** + * Generates the line that constructs the given writable. + * @param writable writable object to construct in the unit test. + * @return string generating the line that constructs the given writable. + */ + public String formatWritable(Writable writable) { + if (writable instanceof NullWritable) { + return "NullWritable.get()"; + } else if (writable instanceof BooleanWritable) { + return String.format("new BooleanWritable(%s)", + format(((BooleanWritable) writable).get())); + } else if (writable instanceof ByteWritable) { + return String.format("new ByteWritable(%s)", + format(((ByteWritable) writable).get())); + } else if (writable instanceof IntWritable) { + return String.format("new IntWritable(%s)", + format(((IntWritable) writable).get())); + } else if (writable instanceof LongWritable) { + return String.format("new LongWritable(%s)", + format(((LongWritable) writable).get())); + } else if (writable instanceof FloatWritable) { + return String.format("new FloatWritable(%s)", + format(((FloatWritable) writable).get())); + } else if (writable instanceof DoubleWritable) { + return String.format("new DoubleWritable(%s)", + format(((DoubleWritable) writable).get())); + } else if (writable instanceof Text) { + return String.format("new Text(%s)", ((Text) writable).toString()); + } else { + if (complexWritables != null) { + complexWritables.add(writable.getClass()); + } + String str = toByteArrayString(WritableUtils.writeToByteArray(writable)); + return String.format("(%s)read%sFromByteArray(new byte[] {%s})", writable + .getClass().getSimpleName(), writable.getClass().getSimpleName(), str); + } + } + + /** + * Generates the line that constructs the given object, which can be + * an int, boolean, char, byte, short, etc. + * @param input object to construct in the unit test. + * @return string generating the line that constructs the given object. + */ + public String format(Object input) { + if (input instanceof Boolean || input instanceof Byte || + input instanceof Character || input instanceof Short || + input instanceof Integer) { + return input.toString(); + } else if (input instanceof Long) { + return input.toString() + "l"; + } else if (input instanceof Float) { + return decimalFormat.format(input) + "f"; + } else if (input instanceof Double) { + double val = ((Double) input).doubleValue(); + if (val == Double.MAX_VALUE) { + return "Double.MAX_VALUE"; + } else if (val == Double.MIN_VALUE) { + return "Double.MIN_VALUE"; + } else { + BigDecimal bd = new BigDecimal(val); + return bd.toEngineeringString() + "d"; + } + } else { + return input.toString(); + } + } + + /** + * Generates a line constructing a byte array. + * @param byteArray byte array to construct in the unit test. + * @return string generating the line that constructs the given byte array. + */ + private String toByteArrayString(byte[] byteArray) { + StringBuilder strBuilder = new StringBuilder(); + for (int i = 0; i < byteArray.length; i++) { + if (i != 0) { + strBuilder.append(','); + } + strBuilder.append(Byte.toString(byteArray[i])); + } + return strBuilder.toString(); + } + +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/MasterComputeTestGenerator.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/MasterComputeTestGenerator.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/MasterComputeTestGenerator.java new file mode 100644 index 0000000..efdd8ac --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/MasterComputeTestGenerator.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import java.io.IOException; +import java.io.StringWriter; + +import org.apache.giraph.debugger.utils.GiraphMasterScenarioWrapper; +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; + +/** + * A code generator to generate test cases to test {@link MasterCompute} + * + * author: Brian Truong Ba Quan + */ +public class MasterComputeTestGenerator extends TestGenerator { + + /** + * Default constructor. + */ + public MasterComputeTestGenerator() { + super(); + } + + /** + * Generates a unit test for the scenario stored in the + * {@link GiraphMasterScenarioWrapper}. + * @param scenario {@link GiraphMasterScenarioWrapper} object to generate a + * unit test from. + * @param testPackage package of the unit test file. + * @return unit test file's contents stored inside a string. + */ + public String generateTest(GiraphMasterScenarioWrapper scenario, + String testPackage) throws IOException, + ClassNotFoundException { + return generateTest(scenario, testPackage, null); + } + + /** + * Generates a unit test for the scenario stored in the + * {@link GiraphMasterScenarioWrapper}. + * @param scenario {@link GiraphMasterScenarioWrapper} object to generate a + * unit test from. + * @param testPackage package of the unit test file. + * @param className name of the unit test class. + * @return unit test file's contents stored inside a string. + */ + public String generateTest(GiraphMasterScenarioWrapper scenario, + String testPackage, String className) throws IOException, + ClassNotFoundException { + VelocityContext context = buildContext(scenario, testPackage, className); + + try (StringWriter sw = new StringWriter()) { + Template template = Velocity.getTemplate("MasterComputeTestTemplate.vm"); + template.merge(context, sw); + return sw.toString(); + } + } + + /** + * Builds the {@link VelocityContext}, which stores data about the context of + * the unit test, e.g. class types, that will be generated for the scenario + * stored in the given {@link GiraphMasterScenarioWrapper}. + * + * @param scenario + * {@link GiraphMasterScenarioWrapper} object to generate a unit test + * from. + * @param testPackage + * package of the unit test file. + * @param className + * name of the unit test class. + * @return {@link VelocityContext} object storing data about the generated + * unit test. + */ + private VelocityContext buildContext( + GiraphMasterScenarioWrapper scenario, String testPackage, + String className) throws ClassNotFoundException { + ContextBuilder builder = new ContextBuilder(); + + Class<?> classUnderTest = Class.forName(scenario + .getMasterClassUnderTest()); + builder.addTestClassInfo(testPackage, classUnderTest, className); + builder.addCommonMasterVertexContext(scenario + .getCommonVertexMasterContextWrapper()); + + return builder.getContext(); + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/PrefixedClasspathResourceLoader.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/PrefixedClasspathResourceLoader.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/PrefixedClasspathResourceLoader.java new file mode 100644 index 0000000..7c70c2b --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/PrefixedClasspathResourceLoader.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import java.io.InputStream; + +import org.apache.commons.collections.ExtendedProperties; +import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader; + +/** + * @see http://stackoverflow.com/a/9693749/390044 + */ +public class PrefixedClasspathResourceLoader extends ClasspathResourceLoader { + + /** Prefix to be added to any names */ + private String prefix = ""; + + @Override + public void init(ExtendedProperties configuration) { + prefix = configuration.getString("prefix", ""); + } + + @Override + public InputStream getResourceStream(String name) { + return super.getResourceStream(prefix + name); + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGenerator.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGenerator.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGenerator.java new file mode 100644 index 0000000..1375f12 --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGenerator.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.giraph.debugger.mock.ComputationComputeTestGenerator.Config; +import org.apache.giraph.debugger.utils.AggregatedValueWrapper; +import org.apache.giraph.debugger.utils.CommonVertexMasterContextWrapper; +import org.apache.giraph.debugger.utils.GiraphVertexScenarioWrapper.VertexScenarioClassesWrapper; +import org.apache.velocity.VelocityContext; + +/** + * Base clas for {@link ComputationComputeTestGenerator} and + * {@link MasterComputeTestGenerator}. + */ +public abstract class TestGenerator extends VelocityBasedGenerator { + + /** + * A set of complex writable classes, i.e., (typically) user-defined + * writables more complex that the regular {@link IntWritable}, + * {@link LongWritable}, etc. + */ + @SuppressWarnings("rawtypes") + private final Set<Class> complexWritables = new HashSet<>(); + + @SuppressWarnings("rawtypes") + public Set<Class> getComplexWritableList() { + return complexWritables; + } + + /** + * Clears {@link #complexWritables}. + */ + protected void resetComplexWritableList() { + this.complexWritables.clear(); + } + + /** + * A wrapper class to populate a {@link VelocityContext} object. + */ + protected class ContextBuilder { + + /** + * {@link VelocityContext} object being wrapped. + */ + protected VelocityContext context; + + /** + * Default constructor. + */ + public ContextBuilder() { + context = new VelocityContext(); + addHelper(); + addWritableReadFromString(); + } + + public VelocityContext getContext() { + return context; + } + + /** + * Adds a {@link FormatHelper} to the context. + */ + private void addHelper() { + context.put("helper", new FormatHelper(complexWritables)); + } + + /** + * Adds the complex writables to the context. + */ + private void addWritableReadFromString() { + context.put("complexWritables", complexWritables); + } + + /** + * Adds the given package name to the context. + * @param testPackage name of the package for the unit test file. + */ + public void addPackage(String testPackage) { + context.put("package", testPackage); + } + + /** + * Adds the type of the class that is being tested to the context. + * @param classUnderTest the class that is being tested. + */ + @SuppressWarnings("rawtypes") + public void addClassUnderTest(Class classUnderTest) { + context.put("classUnderTestFullName", classUnderTest.getName()); + context.put("classUnderTestName", classUnderTest.getSimpleName()); + context.put("classUnderTestPackage", classUnderTest.getPackage() + .getName()); + } + + /** + * Adds the string name of the class that is being tested to the context. + * @param className name of the class being tested. + */ + public void addClassName(String className) { + if (className == null) { + context.put("className", context.get("classUnderTestName") + "Test"); + } else { + context.put("className", className); + } + } + + /** + * Adds the package, type, and name of the class that is being tested to + * the context. + * @param testPackage name of the package for the unit test file. + * @param classUnderTest the class that is being tested. + * @param className name of the class being tested. + */ + @SuppressWarnings("rawtypes") + public void addTestClassInfo(String testPackage, Class classUnderTest, + String className) { + addPackage(testPackage); + addClassUnderTest(classUnderTest); + addClassName(className); + } + + /** + * Adds data stored in the given vertex scenario wrapper to the context. + * @param vertexScenarioClassesWrapper + * {@link VertexScenarioClassesWrapper} object. + */ + @SuppressWarnings("rawtypes") + public void addVertexScenarioClassesWrapper( + VertexScenarioClassesWrapper vertexScenarioClassesWrapper) { + HashSet<Class> usedTypes = new LinkedHashSet<>(6); + usedTypes.add(vertexScenarioClassesWrapper.getClassUnderTest()); + usedTypes.add(vertexScenarioClassesWrapper.getVertexIdClass()); + usedTypes.add(vertexScenarioClassesWrapper.getVertexValueClass()); + usedTypes.add(vertexScenarioClassesWrapper.getEdgeValueClass()); + usedTypes.add(vertexScenarioClassesWrapper.getIncomingMessageClass()); + usedTypes.add(vertexScenarioClassesWrapper.getOutgoingMessageClass()); + context.put("usedTypes", usedTypes); + } + + /** + * Adds data stored in the given master scenario wrapper to the context. + * @param commonVertexMasterContextWrapper + * {@link CommonVertexMasterContextWrapper} object. + */ + @SuppressWarnings("rawtypes") + public void addCommonMasterVertexContext( + CommonVertexMasterContextWrapper commonVertexMasterContextWrapper) { + context.put("superstepNo", + commonVertexMasterContextWrapper.getSuperstepNoWrapper()); + context.put("nVertices", + commonVertexMasterContextWrapper.getTotalNumVerticesWrapper()); + context.put("nEdges", + commonVertexMasterContextWrapper.getTotalNumEdgesWrapper()); + + context.put("aggregators", + commonVertexMasterContextWrapper.getPreviousAggregatedValues()); + Set<Class> usedTypes = new LinkedHashSet<>(); + Collection<AggregatedValueWrapper> aggregatedValues = + commonVertexMasterContextWrapper.getPreviousAggregatedValues(); + for (AggregatedValueWrapper aggregatedValueWrapper : aggregatedValues) { + usedTypes.add(aggregatedValueWrapper.getValue().getClass()); + } + context.put("usedTypesByAggregators", usedTypes); + + List<Config> configs = new ArrayList<>(); + if (commonVertexMasterContextWrapper.getConfig() != null) { + for (Map.Entry<String, String> entry : (Iterable<Map.Entry< + String, String>>) commonVertexMasterContextWrapper.getConfig()) { + configs.add(new Config(entry.getKey(), entry.getValue())); + } + } + context.put("configs", configs); + } + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGraphGenerator.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGraphGenerator.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGraphGenerator.java new file mode 100644 index 0000000..2eabe54 --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/TestGraphGenerator.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Writable; +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; + +/** + * The code generator to generate the end-to-end test case. + * + * author Brian Truong Ba Quan + */ +public class TestGraphGenerator extends VelocityBasedGenerator { + + /** + * Currently supported writables on vertices and edges or ids of vertices. + */ + private enum WritableType { + /** + * {@link NullWritable}. + */ + NULL, + /** + * {@link LongWritable}. + */ + LONG, + /** + * {@link DoubleWritable}. + */ + DOUBLE + }; + + /** + * Generates an end-to-end unit test. + * @param inputStrs a set of strings storing edge and vertex values and + * ids of vertices. + * @return an end-to-end unit test stored as a string (to be saved in a file.) + */ + public String generate(String[] inputStrs) throws IOException { + VelocityContext context = buildContext(inputStrs); + + try (StringWriter sw = new StringWriter()) { + Template template = Velocity.getTemplate("TestGraphTemplate.vm"); + template.merge(context, sw); + return sw.toString(); + } + } + + /** + * Builds the velocity context from the given input strings storing edge, + * vertex values and ids of vertices. + * @param inputStrs an array of strings storing edge and vertex values. + * @return {@link VelocityContext} object. + */ + private VelocityContext buildContext(String[] inputStrs) { + VelocityContext context = new VelocityContext(); + context.put("helper", new FormatHelper()); + // Parse the string and check whether the inputs are integers or + // floating-point numbers + String[][] tokens = new String[inputStrs.length][]; + WritableType idWritableType = WritableType.NULL; + WritableType valueWritableType = WritableType.NULL; + WritableType edgeValueWritableType = WritableType.NULL; + for (int i = 0; i < inputStrs.length; i++) { + tokens[i] = inputStrs[i].trim().split("\\s+"); + String[] nums = tokens[i][0].split(":"); + WritableType type = parseWritableType(nums[0]); + idWritableType = type.ordinal() > idWritableType.ordinal() ? type : + idWritableType; + if (nums.length > 1) { + type = parseWritableType(nums[1]); + valueWritableType = type.ordinal() > valueWritableType.ordinal() ? + type : valueWritableType; + } + + for (int j = 1; j < tokens[i].length; j++) { + nums = tokens[i][j].split(":"); + type = parseWritableType(nums[0]); + idWritableType = type.ordinal() > idWritableType + .ordinal() ? type : idWritableType; + if (nums.length > 1) { + type = parseWritableType(nums[1]); + edgeValueWritableType = type.ordinal() > edgeValueWritableType + .ordinal() ? type : edgeValueWritableType; + } + } + } + + Map<Object, TemplateVertex> vertexMap = new LinkedHashMap<>( + inputStrs.length); + String str; + for (int i = 0; i < inputStrs.length; i++) { + String[] nums = tokens[i][0].split(":"); + Object id = convertToSuitableType(nums[0], idWritableType); + str = nums.length > 1 ? nums[1] : "0"; + Object value = convertToSuitableType(str, valueWritableType); + TemplateVertex vertex = vertexMap.get(id); + if (vertex == null) { + vertex = new TemplateVertex(id); + vertexMap.put(id, vertex); + } + vertex.setValue(value); + + for (int j = 1; j < tokens[i].length; j++) { + nums = tokens[i][j].split(":"); + Object nbrId = convertToSuitableType(nums[0], idWritableType); + str = nums.length > 1 ? nums[1] : "0"; + Object edgeValue = convertToSuitableType(str, edgeValueWritableType); + if (!vertexMap.containsKey(nbrId)) { + vertexMap.put(nbrId, new TemplateVertex(nbrId)); + } + vertex.addNeighbor(nbrId, edgeValue); + } + } + + updateContextByWritableType(context, "vertexIdClass", idWritableType); + updateContextByWritableType(context, "vertexValueClass", valueWritableType); + updateContextByWritableType(context, "edgeValueClass", + edgeValueWritableType); + context.put("vertices", vertexMap); + + return context; + } + + /** + * Returns the {@link Writable} type of the given string value. Tries to + * parse into different types, Long, double, and if one succeeds returns that + * type. Otherwise returns null type. + * @param str string containing a value. + * @return {@link Writable} type of the given value of the string. + */ + private WritableType parseWritableType(String str) { + if (str == null) { + return WritableType.NULL; + } else { + try { + Long.valueOf(str); + return WritableType.LONG; + } catch (NumberFormatException ex) { + return WritableType.DOUBLE; + } + } + } + + /** + * Puts the a given type of a value of an edge or a vertex or id + * of a vertex into the context. + * @param context {@link VelocityContext} to populate. + * @param contextKey currently one of vertexIdClass, vertexValueClass, or + * edgeValueClass. + * @param type currently one of {@link NullWritable}, {@link LongWritable}, + * {@link DoubleWritable}. + */ + private void updateContextByWritableType(VelocityContext context, + String contextKey, WritableType type) { + switch (type) { + case NULL: + context.put(contextKey, NullWritable.class.getSimpleName()); + break; + case LONG: + context.put(contextKey, LongWritable.class.getSimpleName()); + break; + case DOUBLE: + context.put(contextKey, DoubleWritable.class.getSimpleName()); + break; + default: + throw new IllegalStateException("Unknown type!"); + } + } + + /** + * Constructs a {@link Writable} object with the appropriate type that + * contains the specified content. For example, type can be LongWritable, + * and content can be 100L, and this method would return a new + * {@link LongWritable} that has value 100. + * @param contents contetns of the writable. + * @param type type of the writable. + * @return a {@link Writable} object of appropriate type, whose value contains + * the given contents. + */ + private Writable convertToSuitableType(String contents, WritableType type) { + switch (type) { + case NULL: + return NullWritable.get(); + case LONG: + return new LongWritable(Long.valueOf(contents)); + case DOUBLE: + return new DoubleWritable(Double.valueOf(contents)); + default: + throw new IllegalStateException("Unknown type!"); + } + } + + /** + * A wrapper around a simple in-memory representation of a vertex to use + * during test generation. + */ + public static class TemplateVertex { + /** + * Id of the vertex. + */ + private final Object id; + /** + * Value of the vertex. + */ + private Object value; + /** + * Neighbors of the vertex. + */ + private final ArrayList<TemplateNeighbor> neighbors; + + /** + * Constructor. + * @param id if the vertex. + */ + public TemplateVertex(Object id) { + super(); + this.id = id; + this.neighbors = new ArrayList<>(); + } + + public Object getId() { + return id; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } + + public ArrayList<TemplateNeighbor> getNeighbors() { + return neighbors; + } + + /** + * Adds a neighbor to the vertex's adjacency list. + * @param nbrId id of the neighbor. + * @param edgeValue value on the edge to the neighbor. + */ + public void addNeighbor(Object nbrId, Object edgeValue) { + neighbors.add(new TemplateNeighbor(nbrId, edgeValue)); + } + } + + /** + * A wrapper around a simple in-memory representation of a neighbor of a + * vertex to use during test generation. + */ + + public static class TemplateNeighbor { + /** + * Id of the neighbor. + */ + private final Object id; + /** + * Value on the edge from vertex to the neighbor. + */ + private final Object edgeValue; + + /** + * Constructor. + * @param id id of the neighbor. + * @param edgeValue value on the edge from vertex to the neighbor. + */ + public TemplateNeighbor(Object id, Object edgeValue) { + super(); + this.id = id; + this.edgeValue = edgeValue; + } + + public Object getId() { + return id; + } + + public Object getEdgeValue() { + return edgeValue; + } + } +} http://git-wip-us.apache.org/repos/asf/giraph/blob/8675c84a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/VelocityBasedGenerator.java ---------------------------------------------------------------------- diff --git a/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/VelocityBasedGenerator.java b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/VelocityBasedGenerator.java new file mode 100644 index 0000000..b4d6978 --- /dev/null +++ b/giraph-debugger/src/main/java/org/apache/giraph/debugger/mock/VelocityBasedGenerator.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.giraph.debugger.mock; + +import org.apache.velocity.app.Velocity; +import org.apache.velocity.runtime.RuntimeConstants; + +/** + * Base class for {@link TestGenerator} and {@link TestGraphGenerator}. + */ +class VelocityBasedGenerator { + + /** + * Default constructor. + */ + protected VelocityBasedGenerator() { + Velocity.setProperty(RuntimeConstants.RESOURCE_LOADER, "class"); + Velocity.setProperty( + "class." + RuntimeConstants.RESOURCE_LOADER + ".class", + PrefixedClasspathResourceLoader.class.getName()); + Velocity.setProperty("class." + RuntimeConstants.RESOURCE_LOADER + + ".prefix", "/" + + getClass().getPackage().getName().replaceAll("\\.", "/") + "/"); + Velocity.init(); + } + +}
