Add Serializer which serialize/deserialize the ClassHierarchy from/to many type of formats
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/7a095daf Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/7a095daf Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/7a095daf Branch: refs/heads/REEF-395 Commit: 7a095daf13dd8d690c16ad31ce3abba3ec3b0120 Parents: 1d37095 Author: Yunseong Lee <[email protected]> Authored: Sat Jun 20 02:16:56 2015 +0900 Committer: Yunseong Lee <[email protected]> Committed: Sat Jun 20 02:16:56 2015 +0900 ---------------------------------------------------------------------- .../formats/AvroClassHierarchySerializer.java | 254 +++++++++++++++++++ .../tang/formats/ClassHierarchySerializer.java | 43 ++++ 2 files changed, 297 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7a095daf/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroClassHierarchySerializer.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroClassHierarchySerializer.java b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroClassHierarchySerializer.java new file mode 100644 index 0000000..e4f52cf --- /dev/null +++ b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/AvroClassHierarchySerializer.java @@ -0,0 +1,254 @@ +package org.apache.reef.tang.formats; + +import org.apache.avro.file.DataFileReader; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.io.*; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.reef.tang.ClassHierarchy; +import org.apache.reef.tang.formats.avro.*; +import org.apache.reef.tang.implementation.java.AvroClassHierarchy; +import org.apache.reef.tang.types.*; + +import javax.inject.Inject; +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * (De-)Serializing ClassHierarchy to and from AvroClassHierarchy. + * <p/> + * This class is stateless and is therefore safe to reuse. + */ +public class AvroClassHierarchySerializer implements ClassHierarchySerializer { + + /** + * The Charset used for the JSON encoding. + * <p/> + * Copied from <code>org.apache.avro.io.JsonDecoder.CHARSET</code> + */ + private static final String JSON_CHARSET = "ISO-8859-1"; + + @Inject + public AvroClassHierarchySerializer() { + } + + private static AvroNode newClassNode(final String name, + final String fullName, + final boolean isInjectionCandidate, + final boolean isExternalConstructor, + final boolean isUnit, + final List<AvroConstructorDef> injectableConstructors, + final List<AvroConstructorDef> otherConstructors, + final List<String> implFullNames, + final String defaultImplementation, + final List<AvroNode> children) { + return AvroNode.newBuilder() + .setName(name) + .setFullName(fullName) + .setClassNode(AvroClassNode.newBuilder() + .setIsInjectionCandidate(isInjectionCandidate) + .setIsExternalConstructor(isExternalConstructor) + .setIsUnit(isUnit) + .setInjectableConstructors(injectableConstructors) + .setOtherConstructors(otherConstructors) + .setImplFullNames(implFullNames) + .setDefaultImplementation(defaultImplementation) + .build()) + .setChildren(children).build(); + } + + private static AvroNode newNamedParameterNode(final String name, + final String fullName, + final String simpleArgClassName, + final String fullArgClassName, + final boolean isSet, + final boolean isList, + final String documentation, // can be null + final String shortName, // can be null + final List<String> instanceDefault, // can be null + final List<AvroNode> children) { + final AvroNamedParameterNode.Builder namedParameterNodeBuilder = AvroNamedParameterNode.newBuilder() + .setSimpleArgClassName(simpleArgClassName) + .setFullArgClassName(fullArgClassName) + .setIsSet(isSet) + .setIsList(isList); + namedParameterNodeBuilder.setDocumentation(documentation); + namedParameterNodeBuilder.setShortName(shortName); + namedParameterNodeBuilder.setInstanceDefault(instanceDefault); + + return AvroNode.newBuilder() + .setName(name) + .setFullName(fullName) + .setNamedParameterNode(namedParameterNodeBuilder.build()) + .setChildren(children).build(); + } + + private static AvroNode newPackageNode(final String name, + final String fullName, + final List<AvroNode> children) { + return AvroNode.newBuilder() + .setPackageNode(AvroPackageNode.newBuilder().build()) + .setName(name).setFullName(fullName).setChildren(children).build(); + } + + private static AvroConstructorDef newConstructorDef( + String fullClassName, List<AvroConstructorArg> args) { + return AvroConstructorDef.newBuilder() + .setFullArgClassName(fullClassName).setConstructorArg(args).build(); + } + + private static AvroConstructorArg newConstructorArg(final String fullArgClassName, + final String namedParameterName, + final boolean isFuture) { + AvroConstructorArg.Builder builder = AvroConstructorArg.newBuilder() + .setFullArgClassName(fullArgClassName) + .setIsInjectionFuture(isFuture); + if (namedParameterName != null) { + builder.setNamedParameterName(namedParameterName).build(); + } + return builder.build(); + } + + private static AvroConstructorDef serializeConstructorDef(final ConstructorDef<?> def) { + final List<AvroConstructorArg> args = new ArrayList<>(); + for (ConstructorArg arg : def.getArgs()) { + args.add(newConstructorArg(arg.getType(), arg.getNamedParameterName(), arg.isInjectionFuture())); + } + return newConstructorDef(def.getClassName(), args); + } + + private static AvroNode toAvroNode(final Node n) { + final List<AvroNode> children = new ArrayList<>(); + for (final Node child : n.getChildren()) { + children.add(toAvroNode(child)); + } + if (n instanceof ClassNode) { + final ClassNode<?> cn = (ClassNode<?>) n; + final ConstructorDef<?>[] injectable = cn.getInjectableConstructors(); + final ConstructorDef<?>[] all = cn.getAllConstructors(); + final List<ConstructorDef<?>> others = new ArrayList<>(Arrays.asList(all)); + others.removeAll(Arrays.asList(injectable)); + + final List<AvroConstructorDef> injectableConstructors = new ArrayList<>(); + for (final ConstructorDef<?> inj : injectable) { + injectableConstructors.add(serializeConstructorDef(inj)); + } + final List<AvroConstructorDef> otherConstructors = new ArrayList<>(); + for (final ConstructorDef<?> other : others) { + otherConstructors.add(serializeConstructorDef(other)); + } + final List<String> implFullNames = new ArrayList<>(); + for (final ClassNode<?> impl : cn.getKnownImplementations()) { + implFullNames.add(impl.getFullName()); + } + return newClassNode(n.getName(), n.getFullName(), cn.isInjectionCandidate(), cn.isExternalConstructor(), + cn.isUnit(), injectableConstructors, otherConstructors, implFullNames, cn.getDefaultImplementation(), children); + } else if (n instanceof NamedParameterNode) { + final NamedParameterNode<?> np = (NamedParameterNode<?>) n; + return newNamedParameterNode(np.getName(), np.getFullName(), np.getSimpleArgName(), np.getFullArgName(), + np.isSet(), np.isList(), np.getDocumentation(), np.getShortName(), + Arrays.asList(np.getDefaultInstanceAsStrings()), children); + } else if (n instanceof PackageNode) { + return newPackageNode(n.getName(), n.getFullName(), children); + } else { + throw new IllegalStateException("Encountered unknown type of Node: " + n); + } + } + + public AvroClassHierarchy toAvro(final ClassHierarchy classHierarchy) { + return new AvroClassHierarchy(toAvroNode(classHierarchy.getNamespace())); + } + + @Override + public void toFile(final ClassHierarchy classHierarchy, final File file) throws IOException { + final AvroNode avroNode = toAvroNode(classHierarchy.getNamespace()); + final DatumWriter<AvroNode> avroNodeWriter = new SpecificDatumWriter<>(AvroNode.class); + try (DataFileWriter<AvroNode> dataFileWriter = new DataFileWriter<>(avroNodeWriter)) { + dataFileWriter.create(avroNode.getSchema(), file); + dataFileWriter.append(avroNode); + } + } + + @Override + public void toTextFile(ClassHierarchy classHierarchy, File file) throws IOException { + try (final Writer w = new FileWriter(file)) { + w.write(this.toString(classHierarchy)); + } + } + + @Override + public byte[] toByteArray(final ClassHierarchy classHierarchy) throws IOException { + final DatumWriter<AvroNode> requestWriter = new SpecificDatumWriter<>(AvroNode.class); + try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) { + final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + // TODO Should be a better way + requestWriter.write(toAvroNode(classHierarchy.getNamespace()), encoder); + encoder.flush(); + return out.toByteArray(); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + // TODO Improve the interface? + @Override + public String toString(ClassHierarchy classHierarchy) throws IOException { + final DatumWriter<AvroNode> configurationWriter = new SpecificDatumWriter<>(AvroNode.class); + try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) { + final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(AvroConfiguration.SCHEMA$, out); + configurationWriter.write(toAvroNode(classHierarchy.getNamespace()), encoder); + encoder.flush(); + out.flush(); + return out.toString(JSON_CHARSET); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ClassHierarchy fromFile(final File file) throws IOException { + final AvroNode avroNode; + try (final DataFileReader<AvroNode> dataFileReader = + new DataFileReader<>(file, new SpecificDatumReader<>(AvroNode.class))) { + avroNode = dataFileReader.next(); + } + return new AvroClassHierarchy(avroNode); + } + + @Override + public ClassHierarchy fromTextFile(final File file) throws IOException { + final StringBuilder stringBuilder = new StringBuilder(); + try (final BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line = reader.readLine(); + while (line != null) { + stringBuilder.append(line); + line = reader.readLine(); + } + } + return fromString(stringBuilder.toString()); + } + + @Override + public ClassHierarchy fromByteArray(final byte[] theBytes) throws IOException { + try { + final BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(theBytes, null); + final SpecificDatumReader<AvroNode> reader = new SpecificDatumReader<>(AvroNode.class); + return new AvroClassHierarchy(reader.read(null, decoder)); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ClassHierarchy fromString(final String theString) throws IOException { + try { + final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(AvroNode.getClassSchema(), theString); + final SpecificDatumReader<AvroNode> reader = new SpecificDatumReader<>(AvroNode.class); + return new AvroClassHierarchy(reader.read(null, decoder)); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/7a095daf/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/ClassHierarchySerializer.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/ClassHierarchySerializer.java b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/ClassHierarchySerializer.java new file mode 100644 index 0000000..bf3918b --- /dev/null +++ b/lang/java/reef-tang/tang/src/main/java/org/apache/reef/tang/formats/ClassHierarchySerializer.java @@ -0,0 +1,43 @@ +package org.apache.reef.tang.formats; + +import org.apache.reef.tang.ClassHierarchy; +import org.apache.reef.tang.annotations.DefaultImplementation; +import org.apache.reef.tang.formats.avro.AvroNode; + +import java.io.*; + +/** + * + */ +@DefaultImplementation(org.apache.reef.tang.formats.AvroClassHierarchySerializer.class) +public interface ClassHierarchySerializer { + /** + * serialize a class hierarchy into a file. + * + * @param file + * @param classHierarchy + * @throws IOException + */ + void toFile(final ClassHierarchy classHierarchy, final File file) throws IOException; + + void toTextFile(final ClassHierarchy classHierarchy, final File file) throws IOException; + + byte[] toByteArray(final ClassHierarchy classHierarchy) throws IOException; + + String toString(final ClassHierarchy classHierarchy) throws IOException; + + /** + * Deserialize a class hierarchy from a file. The file can be generated from either Java or C# + * + * @param file + * @return + * @throws IOException + */ + ClassHierarchy fromFile(final File file) throws IOException; + + ClassHierarchy fromTextFile(final File file) throws IOException; + + ClassHierarchy fromByteArray(final byte[] theBytes) throws IOException; + + ClassHierarchy fromString(final String theString) throws IOException; +}
