http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/pom.xml new file mode 100644 index 0000000..4f0ed4a --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/pom.xml @@ -0,0 +1,245 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-elasticsearch</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <packaging>maven-plugin</packaging> + + <parent> + <groupId>org.apache.streams</groupId> + <artifactId>streams-plugins</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <properties> + <elasticsearch.version>2.3.0</elasticsearch.version> + <lucene.version>5.4.0</lucene.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-config</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schemas</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>0.9.9</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-aether-provider</artifactId> + <version>3.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-artifact</artifactId> + <version>3.3.3</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-compat</artifactId> + <version>3.3.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <version>3.3.3</version> + <exclusions> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>3.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.7</version> + <exclusions> + <exclusion> + <groupId>org.apache.maven</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven.shared</groupId> + <artifactId>maven-verifier</artifactId> + <version>1.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + <version>3.0.15</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.4</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-tools-api</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-tools-generators</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-testing</groupId> + <artifactId>maven-plugin-testing-harness</artifactId> + <version>3.3.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.6</version> + </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>2.2</version> + </dependency> + <dependency> + <groupId>org.joda</groupId> + <artifactId>joda-convert</artifactId> + <version>1.8.1</version> + </dependency> + </dependencies> + + <build> + <sourceDirectory>src/main/java</sourceDirectory> + <testSourceDirectory>src/test/java</testSourceDirectory> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + </resources> + <testResources> + <testResource> + <directory>src/test/resources</directory> + </testResource> + </testResources> + <plugins> + <plugin> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.4</version> + <configuration> + <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound> + </configuration> + <executions> + <execution> + <id>mojo-descriptor</id> + <phase>process-classes</phase> + <goals> + <goal>descriptor</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.reflections</groupId> + <artifactId>reflections-maven</artifactId> + <executions> + <execution> + <goals> + <goal>reflections</goal> + </goals> + <phase>process-classes</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <id>resource-dependencies</id> + <phase>process-test-resources</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeGroupIds>org.apache.streams</includeGroupIds> + <includeArtifactIds>streams-schemas</includeArtifactIds> + <includes>**/*.json</includes> + <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + </build> + +</project>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java new file mode 100644 index 0000000..ef5b9ef --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java @@ -0,0 +1,81 @@ +package org.apache.streams.plugins.elasticsearch; + +import org.apache.streams.schema.GenerationConfig; +import org.jsonschema2pojo.DefaultGenerationConfig; +import org.jsonschema2pojo.util.URLUtil; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Configures StreamsElasticsearchResourceGenerator + * + * + */ +public class StreamsElasticsearchGenerationConfig extends DefaultGenerationConfig implements GenerationConfig { + + public String getSourceDirectory() { + return sourceDirectory; + } + + public List<String> getSourcePaths() { + return sourcePaths; + } + + private String sourceDirectory; + private List<String> sourcePaths = new ArrayList<String>(); + private String targetDirectory; + private int maxDepth = 1; + + public Set<String> getExclusions() { + return exclusions; + } + + public void setExclusions(Set<String> exclusions) { + this.exclusions = exclusions; + } + + private Set<String> exclusions = new HashSet<String>(); + + public int getMaxDepth() { + return maxDepth; + } + + public void setSourceDirectory(String sourceDirectory) { + this.sourceDirectory = sourceDirectory; + } + + public void setSourcePaths(List<String> sourcePaths) { + this.sourcePaths = sourcePaths; + } + + public void setTargetDirectory(String targetDirectory) { + this.targetDirectory = targetDirectory; + } + + public File getTargetDirectory() { + return new File(targetDirectory); + } + + public Iterator<URL> getSource() { + if (null != sourceDirectory) { + return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator(); + } + List<URL> sourceURLs = new ArrayList<URL>(); + if( sourcePaths != null && sourcePaths.size() > 0) + for (String source : sourcePaths) { + sourceURLs.add(URLUtil.parseURL(source)); + } + return sourceURLs.iterator(); + } + + public void setMaxDepth(int maxDepth) { + this.maxDepth = maxDepth; + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java new file mode 100644 index 0000000..0f506ca --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java @@ -0,0 +1,373 @@ +package org.apache.streams.plugins.elasticsearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.streams.schema.FieldType; +import org.apache.streams.schema.FieldUtil; +import org.apache.streams.schema.GenerationConfig; +import org.apache.streams.schema.Schema; +import org.apache.streams.schema.SchemaStore; +import org.jsonschema2pojo.util.URLUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.streams.schema.FileUtil.dropExtension; +import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.schema.FileUtil.resolveRecursive; +import static org.apache.streams.schema.FileUtil.writeFile; + +/** + * Created by sblackmon on 5/3/16. + */ +public class StreamsElasticsearchResourceGenerator implements Runnable { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGenerator.class); + + private final static String LS = System.getProperty("line.separator"); + + private StreamsElasticsearchGenerationConfig config; + + private SchemaStore schemaStore = new SchemaStore(); + + private int currentDepth = 0; + + public void main(String[] args) { + StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); + + String sourceDirectory = "target/test-classes/activities"; + String targetDirectory = "target/generated-resources/streams-plugin-elasticsearch"; + + if( args.length > 0 ) + sourceDirectory = args[0]; + if( args.length > 1 ) + targetDirectory = args[1]; + + config.setSourceDirectory(sourceDirectory); + config.setTargetDirectory(targetDirectory); + + StreamsElasticsearchResourceGenerator StreamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); + Thread thread = new Thread(StreamsElasticsearchResourceGenerator); + thread.start(); + try { + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + } + return; + } + + public StreamsElasticsearchResourceGenerator(StreamsElasticsearchGenerationConfig config) { + this.config = config; + } + + public void run() { + + checkNotNull(config); + + generate(config); + + } + + public void generate(StreamsElasticsearchGenerationConfig config) { + + LinkedList<File> sourceFiles = new LinkedList<File>(); + + for (Iterator<URL> sources = config.getSource(); sources.hasNext();) { + URL source = sources.next(); + sourceFiles.add(URLUtil.getFileFromURL(source)); + } + + LOGGER.info("Seeded with {} source paths:", sourceFiles.size()); + + resolveRecursive((GenerationConfig)config, sourceFiles); + + LOGGER.info("Resolved {} schema files:", sourceFiles.size()); + + for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) { + File item = iterator.next(); + schemaStore.create(item.toURI()); + } + + LOGGER.info("Identified {} objects:", schemaStore.getSize()); + + String outputFile = config.getTargetDirectory() + "/" + "types.cql"; + StringBuilder typesContent = new StringBuilder(); + + for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { + Schema schema = schemaIterator.next(); + currentDepth = 0; + if( schema.getURI().getScheme().equals("file")) { + String inputFile = schema.getURI().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + + LOGGER.info("Processing {}:", resourcePath); + + String resourceId = schemaSymbol(schema); + + String resourceContent = generateResource(schema, resourceId); + + typesContent.append(resourceContent); + + LOGGER.info("Added {}:", resourceId); + } + } + + writeFile(outputFile, typesContent.toString()); + + } + + public String generateResource(Schema schema, String resourceId) { + StringBuilder resourceBuilder = new StringBuilder(); + resourceBuilder.append("CREATE TYPE "); + resourceBuilder.append(resourceId); + resourceBuilder.append(" IF NOT EXISTS ("); + resourceBuilder.append(LS); + resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' '); + resourceBuilder.append(");"); + resourceBuilder.append(LS); + return resourceBuilder.toString(); + } + + public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) { + ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId); + if( propertiesNode.get("id") != null ) { + builder.append("id text PRIMARY KEY,"); + builder.append(LS); + propertiesNode.remove("id"); + } + if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) { + builder = appendPropertiesNode(builder, schema, propertiesNode, seperator); + } + return builder; + } + + private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) { + // safe to append nothing + checkNotNull(builder); + builder.append(cqlEscape(fieldId)); + builder.append(seperator); + builder.append(cqlType(fieldType)); + return builder; + } + + public StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) { + // not safe to append nothing + checkNotNull(builder); + if( itemsNode == null ) return builder; + if( itemsNode.has("type")) { + try { + FieldType itemType = FieldUtil.determineFieldType(itemsNode); + switch( itemType ) { + case OBJECT: + Schema objectSchema = null; + URI parentURI = null; + if( itemsNode.has("$ref") || itemsNode.has("extends") ) { + JsonNode refNode = itemsNode.get("$ref"); + JsonNode extendsNode = itemsNode.get("extends"); + if (refNode != null && refNode.isValueNode()) + parentURI = URI.create(refNode.asText()); + else if (extendsNode != null && extendsNode.isObject()) + parentURI = URI.create(extendsNode.get("$ref").asText()); + URI absoluteURI; + if (parentURI.isAbsolute()) + absoluteURI = parentURI; + else { + absoluteURI = schema.getURI().resolve(parentURI); + if (!absoluteURI.isAbsolute() || (absoluteURI.isAbsolute() && !schemaStore.getByUri(absoluteURI).isPresent() )) + absoluteURI = schema.getParentURI().resolve(parentURI); + } + if (absoluteURI != null && absoluteURI.isAbsolute()) { + Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteURI); + if (schemaLookup.isPresent()) { + objectSchema = schemaLookup.get(); + } + } + } + // have to resolve schema here + + builder = appendArrayObject(builder, objectSchema, fieldId, seperator); + break; + case ARRAY: + ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items"); + builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator); + break; + default: + builder = appendArrayField(builder, schema, fieldId, itemType, seperator); + } + } catch (Exception e) { + LOGGER.warn("No item type resolvable for {}", fieldId); + } + } + checkNotNull(builder); + return builder; + } + + private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) { + // safe to append nothing + checkNotNull(builder); + checkNotNull(fieldId); + builder.append(cqlEscape(fieldId)); + builder.append(seperator); + builder.append("list<"+cqlType(fieldType)+">"); + checkNotNull(builder); + return builder; + } + + private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, Character seperator) { + // safe to append nothing + checkNotNull(builder); + String schemaSymbol = schemaSymbol(schema); + if( !Strings.isNullOrEmpty(fieldId) && schemaSymbol != null ) { + builder.append(cqlEscape(fieldId)); + builder.append(seperator); + builder.append("list<" + schemaSymbol + ">"); + builder.append(LS); + } + checkNotNull(builder); + return builder; + } + + private StringBuilder appendSchemaField(StringBuilder builder, Schema schema, String fieldId, Character seperator) { + // safe to append nothing + checkNotNull(builder); + String schemaSymbol = schemaSymbol(schema); + if( !Strings.isNullOrEmpty(fieldId) && schemaSymbol != null ) { + builder.append(cqlEscape(fieldId)); + builder.append(seperator); + builder.append(schemaSymbol); + } + checkNotNull(builder); + return builder; + } + + /* + can this be moved to streams-schemas if schemastore available in scope? + maybe an interface? + lot of boilerplate / reuse between plugins + however treatment is way different when resolving a type symbol vs resolving and listing fields . + */ + private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) { + checkNotNull(builder); + checkNotNull(propertiesNode); + Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields(); + Joiner joiner = Joiner.on(","+LS).skipNulls(); + List<String> fieldStrings = Lists.newArrayList(); + for( ; fields.hasNext(); ) { + Map.Entry<String, JsonNode> field = fields.next(); + String fieldId = field.getKey(); + if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) { + ObjectNode fieldNode = (ObjectNode) field.getValue(); + FieldType fieldType = FieldUtil.determineFieldType(fieldNode); + if (fieldType != null ) { + switch (fieldType) { + case ARRAY: + ObjectNode itemsNode = (ObjectNode) fieldNode.get("items"); + if( currentDepth <= config.getMaxDepth()) { + StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator); + if( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) { + fieldStrings.add(arrayItemsBuilder.toString()); + } + } + break; + case OBJECT: + Schema objectSchema = null; + URI parentURI = null; + if( fieldNode.has("$ref") || fieldNode.has("extends") ) { + JsonNode refNode = fieldNode.get("$ref"); + JsonNode extendsNode = fieldNode.get("extends"); + if (refNode != null && refNode.isValueNode()) + parentURI = URI.create(refNode.asText()); + else if (extendsNode != null && extendsNode.isObject()) + parentURI = URI.create(extendsNode.get("$ref").asText()); + URI absoluteURI; + if (parentURI.isAbsolute()) + absoluteURI = parentURI; + else { + absoluteURI = schema.getURI().resolve(parentURI); + if (!absoluteURI.isAbsolute() || (absoluteURI.isAbsolute() && !schemaStore.getByUri(absoluteURI).isPresent() )) + absoluteURI = schema.getParentURI().resolve(parentURI); + } + if (absoluteURI != null && absoluteURI.isAbsolute()) { + Optional<Schema> schemaLookup = schemaStore.getByUri(absoluteURI); + if (schemaLookup.isPresent()) { + objectSchema = schemaLookup.get(); + } + } + } + //ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId); + if( currentDepth < config.getMaxDepth()) { + StringBuilder structFieldBuilder = appendSchemaField(new StringBuilder(), objectSchema, fieldId, seperator); + if( !Strings.isNullOrEmpty(structFieldBuilder.toString())) { + fieldStrings.add(structFieldBuilder.toString()); + } + } + break; + default: + StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator); + if( !Strings.isNullOrEmpty(valueFieldBuilder.toString())) { + fieldStrings.add(valueFieldBuilder.toString()); + } + } + } + } + } + builder.append(joiner.join(fieldStrings)).append(LS); + Preconditions.checkNotNull(builder); + return builder; + } + + private static String cqlEscape( String fieldId ) { + return "`"+fieldId+"`"; + } + + private static String cqlType( FieldType fieldType ) { + switch( fieldType ) { + case STRING: + return "text"; + case INTEGER: + return "int"; + case NUMBER: + return "double"; + case OBJECT: + return "tuple"; + case ARRAY: + return "list"; + default: + return fieldType.name().toUpperCase(); + } + } + + private String schemaSymbol( Schema schema ) { + if (schema == null) return null; + // this needs to return whatever + if (schema.getURI().getScheme().equals("file")) { + String inputFile = schema.getURI().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + return dropExtension(resourcePath).replace("/", "_"); + } else { + return "IDK"; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java new file mode 100644 index 0000000..6969051 --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java @@ -0,0 +1,93 @@ +package org.apache.streams.plugins.elasticsearch; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.Execute; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; + +@Mojo( name = "elasticsearch", + defaultPhase = LifecyclePhase.GENERATE_RESOURCES +) +@Execute( goal = "elasticsearch", + phase = LifecyclePhase.GENERATE_RESOURCES +) +public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class); + + private volatile MojoFailureException mojoFailureException; + + @Component + private MavenProject project; + +// @Component +// private Settings settings; +// +// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) +// protected ArtifactRepository localRepository; +// +// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only +// private PluginDescriptor plugin; +// + @Parameter( defaultValue = "${project.basedir}", readonly = true ) + private File basedir; + + @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only + public String sourceDirectory; + + @Parameter( readonly = true ) // Maven 3 only + public List<String> sourcePaths; + + @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true) + public String targetDirectory; + + public void execute() throws MojoExecutionException, MojoFailureException { + + //addProjectDependenciesToClasspath(); + + StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); + + if( sourcePaths != null && sourcePaths.size() > 0) + config.setSourcePaths(sourcePaths); + else + config.setSourceDirectory(sourceDirectory); + config.setTargetDirectory(targetDirectory); + + StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); + + Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread th, Throwable ex) { + LOGGER.error("Exception", ex); + mojoFailureException = new MojoFailureException("Exception", ex); + } + }; + Thread.setDefaultUncaughtExceptionHandler(h); + Thread thread = new Thread(streamsElasticsearchResourceGenerator); + thread.setUncaughtExceptionHandler(h); + try { + thread.start(); + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + mojoFailureException = new MojoFailureException("Exception", e); + } + + if( mojoFailureException != null ) + throw mojoFailureException; + + return; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md new file mode 100644 index 0000000..9e80a3e --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md @@ -0,0 +1,22 @@ +org.apache.streams.plugins:streams-plugin-elasticsearch +======================================================= + +streams-plugin-elasticsearch generates resources from json schemas to assist with indexing of json data using Elasticsearch. + +#### Usage + +Run within a module containing a src/main/jsonschema directory + + mvn org.apache.streams.plugins:streams-plugin-elasticsearch:0.3-incubating-SNAPSHOT:elasticsearch + +Output will be placed in target/generated-resources/elasticsearch by default + +#### Example + +[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml") + +#### Documentation + +[JavaDocs](apidocs/index.html "JavaDocs") + +###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java new file mode 100644 index 0000000..72f4675 --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoTest.java @@ -0,0 +1,50 @@ +package org.apache.streams.plugins.elasticsearch.test; + +import com.google.common.collect.Lists; +import junit.framework.TestCase; +import org.apache.maven.it.Verifier; +import org.apache.maven.it.util.ResourceExtractor; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + * Tests that streams-plugin-hive running via maven generates hql resources + */ +public class StreamsElasticsearchResourceGeneratorMojoTest extends TestCase { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoTest.class); + + protected void setUp() throws Exception + { + // required for mojo lookups to work + super.setUp(); + } + + + @Test + public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception { + + File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" ); + + Verifier verifier; + + verifier = new Verifier( testDir.getAbsolutePath() ); + + List cliOptions = new ArrayList(); + cliOptions.add( "-N" ); + verifier.executeGoals( Lists.<String>newArrayList( + "clean", + "dependency:unpack-dependencies", + "generate-resources")); + + verifier.verifyErrorFreeLog(); + + verifier.resetStreams(); + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java new file mode 100644 index 0000000..20bb561 --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java @@ -0,0 +1,139 @@ +package org.apache.streams.plugins.elasticsearch.test; + +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig; +import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Iterator; + +import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; + +/** + * Test that Elasticsearch resources are generated. + */ +public class StreamsElasticsearchResourceGeneratorTest { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class); + + /** + * Test that Elasticsearch resources are generated + * + * @throws Exception + */ + @Test + public void StreamsElasticsearchResourceGenerator() throws Exception { + + StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); + + String sourceDirectory = "target/test-classes/streams-schemas"; + + config.setSourceDirectory(sourceDirectory); + + config.setTargetDirectory("target/generated-sources/test"); + + config.setExclusions(Sets.newHashSet("attachments")); + + config.setMaxDepth(2); + + StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); + Thread thread = new Thread(streamsElasticsearchResourceGenerator); + thread.start(); + try { + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + } + + File testOutput = new File( "./target/generated-sources/test"); + Predicate<File> jsonFilter = new Predicate<File>() { + @Override + public boolean apply(@Nullable File file) { + if( file.getName().endsWith(".json") ) + return true; + else return false; + } + }; + + assert( testOutput != null ); + assert( testOutput.exists() == true ); + assert( testOutput.isDirectory() == true ); + + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(jsonFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert( outputCollection.size() == 133 ); + + String expectedDirectory = "target/test-classes/expected"; + File testExpected = new File( expectedDirectory ); + + Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) + .filter(jsonFilter); + Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); + + int fails = 0; + + Iterator<File> iterator = expectedCollection.iterator(); + while( iterator.hasNext() ) { + File objectExpected = iterator.next(); + String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); + File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); + LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); + assert( objectActual.exists()); + if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { + LOGGER.info("Exact Match!"); + } else { + LOGGER.info("No Match!"); + fails++; + } + } + if( fails > 0 ) { + LOGGER.info("Fails: {}", fails); + Assert.fail(); + } + +// String expectedDirectory = "target/test-classes/expected"; +// File testExpected = new File( expectedDirectory ); +// +// Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) +// .filter(cqlFilter); +// Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); +// +// int fails = 0; +// +// Iterator<File> iterator = expectedCollection.iterator(); +// while( iterator.hasNext() ) { +// File objectExpected = iterator.next(); +// String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); +// File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); +// LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); +// assert( objectActual.exists()); +// if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { +// LOGGER.info("Exact Match!"); +// } else { +// LOGGER.info("No Match!"); +// fails++; +// } +// } +// if( fails > 0 ) { +// LOGGER.info("Fails: {}", fails); +// Assert.fail(); +// } + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml new file mode 100644 index 0000000..2aaa037 --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml @@ -0,0 +1,75 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-elasticsearch-test</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <packaging>jar</packaging> + <name>Test StreamsElasticsearchResourceGeneratorMojo</name> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schemas</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-elasticsearch</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <configuration> + <sourcePaths> + <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath> + <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath> + <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath> + <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath> + <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath> + <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath> + </sourcePaths> + <targetDirectory>target/generated-resources/test-mojo</targetDirectory> + </configuration> + <executions> + <execution> + <goals> + <goal>elasticsearch</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.10</version> + <configuration> + <includes>**/*.json</includes> + <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <includeGroupIds>org.apache.streams</includeGroupIds> + <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeTypes>jar,test-jar</includeTypes> + </configuration> + <executions> + <execution> + <id>unpack-schemas</id> + <phase>process-resources</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/pom.xml new file mode 100644 index 0000000..94a0115 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/pom.xml @@ -0,0 +1,244 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-hbase</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <packaging>maven-plugin</packaging> + + <parent> + <groupId>org.apache.streams</groupId> + <artifactId>streams-plugins</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <properties> + <hbase.version>3.0</hbase.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-config</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schemas</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>0.9.9</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-aether-provider</artifactId> + <version>3.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-artifact</artifactId> + <version>3.3.3</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-compat</artifactId> + <version>3.3.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <version>3.3.3</version> + <exclusions> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>3.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.shared</groupId> + <artifactId>maven-verifier</artifactId> + <version>1.6</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.7</version> + <exclusions> + <exclusion> + <groupId>org.apache.maven</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + <version>3.0.15</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.4</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-tools-api</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-tools-generators</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-testing</groupId> + <artifactId>maven-plugin-testing-harness</artifactId> + <version>3.3.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.6</version> + </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>2.2</version> + </dependency> + <dependency> + <groupId>org.joda</groupId> + <artifactId>joda-convert</artifactId> + <version>1.8.1</version> + </dependency> + </dependencies> + + <build> + <sourceDirectory>src/main/java</sourceDirectory> + <testSourceDirectory>src/test/java</testSourceDirectory> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + </resources> + <testResources> + <testResource> + <directory>src/test/resources</directory> + </testResource> + </testResources> + <plugins> + <plugin> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.4</version> + <configuration> + <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound> + </configuration> + <executions> + <execution> + <id>mojo-descriptor</id> + <phase>process-classes</phase> + <goals> + <goal>descriptor</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.reflections</groupId> + <artifactId>reflections-maven</artifactId> + <executions> + <execution> + <goals> + <goal>reflections</goal> + </goals> + <phase>process-classes</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <id>resource-dependencies</id> + <phase>process-test-resources</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeGroupIds>org.apache.streams</includeGroupIds> + <includeArtifactIds>streams-schemas</includeArtifactIds> + <includes>**/*.json</includes> + <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java new file mode 100644 index 0000000..2612ec2 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java @@ -0,0 +1,90 @@ +package org.apache.streams.plugins.hbase; + +import org.apache.streams.schema.GenerationConfig; +import org.jsonschema2pojo.DefaultGenerationConfig; +import org.jsonschema2pojo.util.URLUtil; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Configures StreamsHiveResourceGenerator + * + * + */ +public class StreamsHbaseGenerationConfig extends DefaultGenerationConfig implements GenerationConfig { + + public String getSourceDirectory() { + return sourceDirectory; + } + + public List<String> getSourcePaths() { + return sourcePaths; + } + + private String columnFamily; + private String sourceDirectory; + private List<String> sourcePaths = new ArrayList<String>(); + private String targetDirectory; + private int maxDepth = 1; + + public Set<String> getExclusions() { + return exclusions; + } + + public void setExclusions(Set<String> exclusions) { + this.exclusions = exclusions; + } + + private Set<String> exclusions = new HashSet<String>(); + + public int getMaxDepth() { + return maxDepth; + } + + public void setSourceDirectory(String sourceDirectory) { + this.sourceDirectory = sourceDirectory; + } + + public void setSourcePaths(List<String> sourcePaths) { + this.sourcePaths = sourcePaths; + } + + public void setTargetDirectory(String targetDirectory) { + this.targetDirectory = targetDirectory; + } + + public File getTargetDirectory() { + return new File(targetDirectory); + } + + public Iterator<URL> getSource() { + if (null != sourceDirectory) { + return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator(); + } + List<URL> sourceURLs = new ArrayList<URL>(); + if( sourcePaths != null && sourcePaths.size() > 0) + for (String source : sourcePaths) { + sourceURLs.add(URLUtil.parseURL(source)); + } + return sourceURLs.iterator(); + } + + public void setMaxDepth(int maxDepth) { + this.maxDepth = maxDepth; + } + + public String getColumnFamily() { + return columnFamily; + } + + public void setColumnFamily(String columnFamily) { + this.columnFamily = columnFamily; + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java new file mode 100644 index 0000000..649bdd9 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java @@ -0,0 +1,204 @@ +package org.apache.streams.plugins.hbase; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import org.apache.streams.schema.FieldType; +import org.apache.streams.schema.FieldUtil; +import org.apache.streams.schema.GenerationConfig; +import org.apache.streams.schema.Schema; +import org.apache.streams.schema.SchemaStore; +import org.jsonschema2pojo.util.URLUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.streams.schema.FileUtil.dropExtension; +import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.schema.FileUtil.resolveRecursive; +import static org.apache.streams.schema.FileUtil.swapExtension; +import static org.apache.streams.schema.FileUtil.writeFile; + +/** + * Created by sblackmon on 5/3/16. + */ +public class StreamsHbaseResourceGenerator implements Runnable { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class); + + private final static String LS = System.getProperty("line.separator"); + + private StreamsHbaseGenerationConfig config; + + private SchemaStore schemaStore = new SchemaStore(); + + private int currentDepth = 0; + + public static void main(String[] args) { + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + + String sourceDirectory = "./target/test-classes/activities"; + String targetDirectory = "./target/generated-resources/hbase"; + + if( args.length > 0 ) + sourceDirectory = args[0]; + if( args.length > 1 ) + targetDirectory = args[1]; + + config.setSourceDirectory(sourceDirectory); + config.setTargetDirectory(targetDirectory); + + StreamsHbaseResourceGenerator StreamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + Thread thread = new Thread(StreamsHbaseResourceGenerator); + thread.start(); + try { + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + } + return; + } + + public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) { + this.config = config; + } + + public void run() { + + checkNotNull(config); + + generate(config); + + } + + public void generate(StreamsHbaseGenerationConfig config) { + + LinkedList<File> sourceFiles = new LinkedList<File>(); + + for (Iterator<URL> sources = config.getSource(); sources.hasNext();) { + URL source = sources.next(); + sourceFiles.add(URLUtil.getFileFromURL(source)); + } + + LOGGER.info("Seeded with {} source paths:", sourceFiles.size()); + + resolveRecursive((GenerationConfig)config, sourceFiles); + + LOGGER.info("Resolved {} schema files:", sourceFiles.size()); + + for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) { + File item = iterator.next(); + schemaStore.create(item.toURI()); + } + + LOGGER.info("Identified {} objects:", schemaStore.getSize()); + + for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { + Schema schema = schemaIterator.next(); + currentDepth = 0; + if( schema.getURI().getScheme().equals("file")) { + String inputFile = schema.getURI().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt"); + + LOGGER.info("Processing {}:", resourcePath); + + String resourceId = dropExtension(resourcePath).replace("/", "_"); + + String resourceContent = generateResource(schema, resourceId); + + writeFile(outputFile, resourceContent); + + LOGGER.info("Wrote {}:", outputFile); + } + } + + } + + public String generateResource(Schema schema, String resourceId) { + StringBuilder resourceBuilder = new StringBuilder(); + resourceBuilder.append("CREATE "); + resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId); + return resourceBuilder.toString(); + } + + public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) { + checkNotNull(builder); + ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId); + if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) { + + List<String> fieldStrings = Lists.newArrayList(); + + // table + fieldStrings.add(hbaseEscape(schemaSymbol(schema))); + + // column family + fieldStrings.add(hbaseEscape(schemaSymbol(schema))); + + // parent column family + if( schema.getParent() != null ) + fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent()))); + + // sub-object column families + if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) { + + Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields(); + Joiner joiner = Joiner.on(", ").skipNulls(); + for( ; fields.hasNext(); ) { + Map.Entry<String, JsonNode> field = fields.next(); + String fieldId = field.getKey(); + if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) { + ObjectNode fieldNode = (ObjectNode) field.getValue(); + FieldType fieldType = FieldUtil.determineFieldType(fieldNode); + if (fieldType != null ) { + switch (fieldType) { + case OBJECT: + fieldStrings.add(hbaseEscape(fieldId)); + } + } + } + } + builder.append(joiner.join(fieldStrings)); + + } + } + checkNotNull(builder); + return builder; + } + + private static String hbaseEscape( String fieldId ) { + return "'"+fieldId+"'"; + } + + private String schemaSymbol( Schema schema ) { + if (schema == null) return null; + if (schema.getURI().getScheme().equals("file")) { + String inputFile = schema.getURI().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + return dropExtension(resourcePath).replace("/", "_"); + } else { + return "IDK"; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java new file mode 100644 index 0000000..0db68f0 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java @@ -0,0 +1,93 @@ +package org.apache.streams.plugins.hbase; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.Execute; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.List; + +@Mojo( name = "hbase", + defaultPhase = LifecyclePhase.GENERATE_RESOURCES +) +@Execute( goal = "hbase", + phase = LifecyclePhase.GENERATE_RESOURCES +) +public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class); + + private volatile MojoFailureException mojoFailureException; + + @Component + private MavenProject project; + +// @Component +// private Settings settings; +// +// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) +// protected ArtifactRepository localRepository; +// +// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only +// private PluginDescriptor plugin; +// + @Parameter( defaultValue = "${project.basedir}", readonly = true ) + private File basedir; + + @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only + public String sourceDirectory; + + @Parameter( readonly = true ) // Maven 3 only + public List<String> sourcePaths; + + @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true) + public String targetDirectory; + + public void execute() throws MojoExecutionException, MojoFailureException { + + //addProjectDependenciesToClasspath(); + + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + + if( sourcePaths != null && sourcePaths.size() > 0) + config.setSourcePaths(sourcePaths); + else + config.setSourceDirectory(sourceDirectory); + config.setTargetDirectory(targetDirectory); + + StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + + Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread th, Throwable ex) { + LOGGER.error("Exception", ex); + mojoFailureException = new MojoFailureException("Exception", ex); + } + }; + Thread.setDefaultUncaughtExceptionHandler(h); + Thread thread = new Thread(streamsHbaseResourceGenerator); + thread.setUncaughtExceptionHandler(h); + try { + thread.start(); + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + mojoFailureException = new MojoFailureException("Exception", e); + } + + if( mojoFailureException != null ) + throw mojoFailureException; + + return; + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md new file mode 100644 index 0000000..858cb85 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md @@ -0,0 +1,22 @@ +org.apache.streams.plugins:streams-plugin-hbase +=============================================== + +streams-plugin-hbase generates resources from json schemas to assist with indexing of json data using Apache HBase. + +#### Usage + +Run within a module containing a src/main/jsonschema directory + + mvn org.apache.streams.plugins:streams-plugin-hbase:0.3-incubating-SNAPSHOT:hbase + +Output will be placed in target/generated-resources/hive by default + +#### Example + +[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml") + +#### Documentation + +[JavaDocs](apidocs/index.html "JavaDocs") + +###### Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java new file mode 100644 index 0000000..290d601 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java @@ -0,0 +1,41 @@ +package org.apache.streams.plugins.test; + +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.File; +import java.util.Collection; +import java.util.List; + +import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter; + +/** + * Created by sblackmon on 5/5/16. + */ +public class StreamsHbaseResourceGeneratorCLITest { + + @Test + public void testStreamsHiveResourceGeneratorCLI() throws Exception { + + String sourceDirectory = "target/test-classes/streams-schemas"; + String targetDirectory = "target/generated-resources/hbase-cli"; + + List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); + StreamsHbaseResourceGenerator.main(argsList.toArray(new String[0])); + + File testOutput = new File(targetDirectory); + + assert( testOutput != null ); + assert( testOutput.exists() == true ); + assert( testOutput.isDirectory() == true ); + + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(txtFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert( outputCollection.size() == 133 ); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java new file mode 100644 index 0000000..6293021 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java @@ -0,0 +1,65 @@ +package org.apache.streams.plugins.test; + +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import junit.framework.TestCase; +import org.apache.maven.it.Verifier; +import org.apache.maven.it.util.ResourceExtractor; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter; + +/** + * Tests that streams-plugin-hive running via maven generates hql resources + */ +public class StreamsHbaseResourceGeneratorMojoTest extends TestCase { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoTest.class); + + protected void setUp() throws Exception + { + // required for mojo lookups to work + super.setUp(); + } + + + @Test + public void testStreamsHbaseResourceGeneratorMojo() throws Exception { + + File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" ); + + Verifier verifier; + + verifier = new Verifier( testDir.getAbsolutePath() ); + + List cliOptions = new ArrayList(); + cliOptions.add( "-N" ); + verifier.executeGoals( Lists.<String>newArrayList( + "clean", + "dependency:unpack-dependencies", + "generate-resources")); + + verifier.verifyErrorFreeLog(); + + verifier.resetStreams(); + + File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/test-mojo"); + + assert( testOutput != null ); + assert( testOutput.exists() == true ); + assert( testOutput.isDirectory() == true ); + + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(txtFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert( outputCollection.size() == 133 ); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/95a02d71/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java new file mode 100644 index 0000000..02b0214 --- /dev/null +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java @@ -0,0 +1,125 @@ +package org.apache.streams.plugins.test; + +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig; +import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.io.File; +import java.util.Collection; +import java.util.Iterator; + +import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; + +/** + * Test that Activity beans are compatible with the example activities in the spec. + */ +public class StreamsHbaseResourceGeneratorTest { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class); + + public static final Predicate<File> txtFilter = new Predicate<File>() { + @Override + public boolean apply(@Nullable File file) { + if( file.getName().endsWith(".txt") ) + return true; + else return false; + } + }; + + /** + * Tests that all example activities can be loaded into Activity beans + * + * @throws Exception + */ + @Test + public void StreamsHbaseResourceGenerator() throws Exception { + + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + + String sourceDirectory = "target/test-classes/streams-schemas"; + + config.setSourceDirectory(sourceDirectory); + + config.setTargetDirectory("target/generated-resources/test"); + + config.setExclusions(Sets.newHashSet("attachments")); + + config.setColumnFamily("cf"); + config.setMaxDepth(2); + + StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + Thread thread = new Thread(streamsHbaseResourceGenerator); + thread.start(); + try { + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + } + + File testOutput = config.getTargetDirectory(); + + + assert( testOutput != null ); + assert( testOutput.exists() == true ); + assert( testOutput.isDirectory() == true ); + + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(txtFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert( outputCollection.size() == 133 ); + + String expectedDirectory = "target/test-classes/expected"; + File testExpected = new File( expectedDirectory ); + + Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) + .filter(txtFilter); + Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); + + int fails = 0; + + Iterator<File> iterator = expectedCollection.iterator(); + while( iterator.hasNext() ) { + File objectExpected = iterator.next(); + String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); + File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); + LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); + assert( objectActual.exists()); + if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { + LOGGER.info("Exact Match!"); + } else { + LOGGER.info("No Match!"); + fails++; + } + } + if( fails > 0 ) { + LOGGER.info("Fails: {}", fails); + Assert.fail(); + } + + +// assert( new File(testOutput + "/traits").exists() == true ); +// assert( new File(testOutput + "/traits").isDirectory() == true ); +// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null ); +// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 ); +// assert( new File(testOutput + "/objectTypes").exists() == true ); +// assert( new File(testOutput + "/objectTypes").isDirectory() == true ); +// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null ); +// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 ); +// assert( new File(testOutput + "/verbs").exists() == true ); +// assert( new File(testOutput + "/verbs").isDirectory() == true ); +// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null ); +// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 ); + } +} \ No newline at end of file
