http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java index d46eaa6..daf85ae 100644 --- a/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java +++ b/streams-plugins/streams-plugin-cassandra/src/test/java/org/apache/streams/plugins/cassandra/test/StreamsCassandraResourceGeneratorTest.java @@ -1,15 +1,12 @@ package org.apache.streams.plugins.cassandra.test; -import com.google.common.base.Charsets; import com.google.common.base.Predicate; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.io.Files; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.streams.plugins.cassandra.StreamsCassandraGenerationConfig; import org.apache.streams.plugins.cassandra.StreamsCassandraResourceGenerator; -import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,10 +16,6 @@ import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; -import java.util.Iterator; -import java.util.List; - -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; /** * Test that cassandra resources are generated. @@ -50,28 +43,20 @@ public class StreamsCassandraResourceGeneratorTest { StreamsCassandraGenerationConfig config = new StreamsCassandraGenerationConfig(); - String sourceDirectory = "target/test-classes/streams-schemas"; + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory("target/generated-sources/test"); + config.setTargetDirectory("target/generated-resources/cassandra"); config.setExclusions(Sets.newHashSet("attachments")); config.setMaxDepth(2); StreamsCassandraResourceGenerator streamsCassandraResourceGenerator = new StreamsCassandraResourceGenerator(config); - Thread thread = new Thread(streamsCassandraResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } + streamsCassandraResourceGenerator.run(); - File testOutput = new File( "./target/generated-sources/test"); + File testOutput = config.getTargetDirectory(); assert( testOutput != null ); assert( testOutput.exists() == true ); @@ -82,43 +67,14 @@ public class StreamsCassandraResourceGeneratorTest { Collection<File> outputCollection = Lists.newArrayList(outputIterator); assert( outputCollection.size() == 1 ); - Path path = Paths.get("./target/generated-sources/test/types.cql"); + Path path = Paths.get(testOutput.getAbsolutePath()).resolve("types.cql"); + + assert( path.toFile().exists() ); String typesCqlBytes = new String( java.nio.file.Files.readAllBytes(path)); assert( StringUtils.countMatches(typesCqlBytes, "CREATE TYPE") == 133 ); - assert( !typesCqlBytes.contains("IDK")); - - -// String expectedDirectory = "target/test-classes/expected"; -// File testExpected = new File( expectedDirectory ); -// -// Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) -// .filter(cqlFilter); -// Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); -// -// int fails = 0; -// -// Iterator<File> iterator = expectedCollection.iterator(); -// while( iterator.hasNext() ) { -// File objectExpected = iterator.next(); -// String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); -// File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); -// LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); -// assert( objectActual.exists()); -// if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { -// LOGGER.info("Exact Match!"); -// } else { -// LOGGER.info("No Match!"); -// fails++; -// } -// } -// if( fails > 0 ) { -// LOGGER.info("Fails: {}", fails); -// Assert.fail(); -// } - } } \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml b/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml index 064ea52..2a69ec7 100644 --- a/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml +++ b/streams-plugins/streams-plugin-cassandra/src/test/resources/streams-plugin-cassandra/pom.xml @@ -18,8 +18,9 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> <version>${project.version}</version> + <type>test-jar</type> <scope>test</scope> </dependency> </dependencies> @@ -32,19 +33,19 @@ <version>0.3-incubating-SNAPSHOT</version> <configuration> <sourcePaths> - <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath> - <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath> </sourcePaths> <targetDirectory>target/generated-resources/test-mojo</targetDirectory> </configuration> <executions> <execution> <goals> - <goal>cassandra</goal> + <goal>generate-resources</goal> </goals> </execution> </executions> @@ -55,9 +56,9 @@ <version>2.10</version> <configuration> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includeTypes>jar,test-jar</includeTypes> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/pom.xml index 4f0ed4a..07d7fc1 100644 --- a/streams-plugins/streams-plugin-elasticsearch/pom.xml +++ b/streams-plugins/streams-plugin-elasticsearch/pom.xml @@ -55,10 +55,17 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-util</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schema-activitystreams</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> @@ -231,9 +238,9 @@ </goals> <configuration> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> </configuration> </execution> </executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java index ef5b9ef..71bbdf8 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchGenerationConfig.java @@ -1,6 +1,6 @@ package org.apache.streams.plugins.elasticsearch; -import org.apache.streams.schema.GenerationConfig; +import org.apache.streams.util.schema.GenerationConfig; import org.jsonschema2pojo.DefaultGenerationConfig; import org.jsonschema2pojo.util.URLUtil; http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java index 0f506ca..4d16f7e 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGenerator.java @@ -1,17 +1,21 @@ package org.apache.streams.plugins.elasticsearch; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import org.apache.streams.schema.FieldType; -import org.apache.streams.schema.FieldUtil; -import org.apache.streams.schema.GenerationConfig; -import org.apache.streams.schema.Schema; -import org.apache.streams.schema.SchemaStore; +import org.apache.streams.jackson.StreamsJacksonMapper; +import org.apache.streams.util.schema.FieldType; +import org.apache.streams.util.schema.FieldUtil; +import org.apache.streams.util.schema.GenerationConfig; +import org.apache.streams.util.schema.Schema; +import org.apache.streams.util.schema.SchemaStore; +import org.apache.streams.util.schema.SchemaStoreImpl; import org.jsonschema2pojo.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,10 +29,11 @@ import java.util.List; import java.util.Map; import static com.google.common.base.Preconditions.checkNotNull; -import static org.apache.streams.schema.FileUtil.dropExtension; -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; -import static org.apache.streams.schema.FileUtil.resolveRecursive; -import static org.apache.streams.schema.FileUtil.writeFile; +import static org.apache.streams.util.schema.FileUtil.dropExtension; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.resolveRecursive; +import static org.apache.streams.util.schema.FileUtil.swapExtension; +import static org.apache.streams.util.schema.FileUtil.writeFile; /** * Created by sblackmon on 5/3/16. @@ -37,15 +42,17 @@ public class StreamsElasticsearchResourceGenerator implements Runnable { private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGenerator.class); + ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(); + private final static String LS = System.getProperty("line.separator"); private StreamsElasticsearchGenerationConfig config; - private SchemaStore schemaStore = new SchemaStore(); + private SchemaStore schemaStore = new SchemaStoreImpl(); private int currentDepth = 0; - public void main(String[] args) { + public static void main(String[] args) { StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); String sourceDirectory = "target/test-classes/activities"; @@ -59,17 +66,9 @@ public class StreamsElasticsearchResourceGenerator implements Runnable { config.setSourceDirectory(sourceDirectory); config.setTargetDirectory(targetDirectory); - StreamsElasticsearchResourceGenerator StreamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); - Thread thread = new Thread(StreamsElasticsearchResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } - return; + StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); + streamsElasticsearchResourceGenerator.run(); + } public StreamsElasticsearchResourceGenerator(StreamsElasticsearchGenerationConfig config) { @@ -106,7 +105,6 @@ public class StreamsElasticsearchResourceGenerator implements Runnable { LOGGER.info("Identified {} objects:", schemaStore.getSize()); - String outputFile = config.getTargetDirectory() + "/" + "types.cql"; StringBuilder typesContent = new StringBuilder(); for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { @@ -118,6 +116,7 @@ public class StreamsElasticsearchResourceGenerator implements Runnable { for (String sourcePath : config.getSourcePaths()) { resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); } + String outputFile = config.getTargetDirectory() + "/" + resourcePath; LOGGER.info("Processing {}:", resourcePath); @@ -125,25 +124,33 @@ public class StreamsElasticsearchResourceGenerator implements Runnable { String resourceContent = generateResource(schema, resourceId); - typesContent.append(resourceContent); + if( !Strings.isNullOrEmpty(resourceContent)) + writeFile(outputFile, resourceContent); - LOGGER.info("Added {}:", resourceId); + LOGGER.info("Wrote {}:", outputFile); } } - writeFile(outputFile, typesContent.toString()); - } public String generateResource(Schema schema, String resourceId) { StringBuilder resourceBuilder = new StringBuilder(); - resourceBuilder.append("CREATE TYPE "); - resourceBuilder.append(resourceId); - resourceBuilder.append(" IF NOT EXISTS ("); - resourceBuilder.append(LS); - resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' '); - resourceBuilder.append(");"); - resourceBuilder.append(LS); + + ObjectNode rootNode = (ObjectNode) schema.getContent(); + + // remove java* + // remove description + // resolve all $ref + // replace format: date with type: date + // replace format: date-time with type: date + // replace array of primitive with just primitive + + try { + String objectString = MAPPER.writeValueAsString(rootNode); + resourceBuilder.append(objectString); + } catch (JsonProcessingException e) { + LOGGER.error("{}: {}", e.getClass().getName(), e); + } return resourceBuilder.toString(); } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java index 6969051..ed280ff 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java @@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.util.List; -@Mojo( name = "elasticsearch", +@Mojo( name = "generate-resources", defaultPhase = LifecyclePhase.GENERATE_RESOURCES ) -@Execute( goal = "elasticsearch", +@Execute( goal = "generate-resources", phase = LifecyclePhase.GENERATE_RESOURCES ) public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo { @@ -65,29 +65,8 @@ public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo { StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); - Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread th, Throwable ex) { - LOGGER.error("Exception", ex); - mojoFailureException = new MojoFailureException("Exception", ex); - } - }; - Thread.setDefaultUncaughtExceptionHandler(h); - Thread thread = new Thread(streamsElasticsearchResourceGenerator); - thread.setUncaughtExceptionHandler(h); - try { - thread.start(); - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - mojoFailureException = new MojoFailureException("Exception", e); - } - - if( mojoFailureException != null ) - throw mojoFailureException; - - return; + streamsElasticsearchResourceGenerator.run(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md index 9e80a3e..f65433b 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md +++ b/streams-plugins/streams-plugin-elasticsearch/src/site/markdown/index.md @@ -3,17 +3,33 @@ org.apache.streams.plugins:streams-plugin-elasticsearch streams-plugin-elasticsearch generates resources from json schemas to assist with indexing of json data using Elasticsearch. -#### Usage +### Usage + +Output will be placed in target/generated-resources/elasticsearch by default + +##### Maven Run within a module containing a src/main/jsonschema directory mvn org.apache.streams.plugins:streams-plugin-elasticsearch:0.3-incubating-SNAPSHOT:elasticsearch -Output will be placed in target/generated-resources/elasticsearch by default +[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml") -#### Example +##### SDK -[streams-plugin-elasticsearch/pom.xml](streams-plugin-elasticsearch/pom.xml "streams-plugin-elasticsearch/pom.xml") +Embed within your own java code + + StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); + config.setSourceDirectory("src/main/jsonschema"); + config.setTargetDirectory("target/generated-resources"); + StreamsElasticsearchResourceGenerator generator = new StreamsElasticsearchResourceGenerator(config); + generator.run(); + +##### CLI + +Run from CLI without Maven + + java -jar streams-plugin-elasticsearch-jar-with-dependencies.jar StreamsElasticsearchResourceGenerator src/main/jsonschema target/generated-resources #### Documentation http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java new file mode 100644 index 0000000..3805894 --- /dev/null +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java @@ -0,0 +1,39 @@ +package org.apache.streams.plugins.elasticsearch.test; + +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; +import org.junit.Test; + +import java.io.File; +import java.util.Collection; +import java.util.List; + +import static org.apache.streams.plugins.elasticsearch.test.StreamsElasticsearchResourceGeneratorTest.jsonFilter; + +/** + * Created by sblackmon on 5/5/16. + */ +public class StreamsElasticsearchResourceGeneratorCLITest { + + @Test + public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception { + + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; + String targetDirectory = "target/generated-resources/elasticsearch-cli"; + + List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); + StreamsElasticsearchResourceGenerator.main(argsList.toArray(new String[0])); + + File testOutput = new File(targetDirectory); + + assert( testOutput != null ); + assert( testOutput.exists() == true ); + assert( testOutput.isDirectory() == true ); + + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(jsonFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert( outputCollection.size() == 133 ); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java index 20bb561..7844442 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java @@ -5,7 +5,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig; import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; import org.junit.Assert; @@ -15,12 +14,10 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Collection; import java.util.Iterator; -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; /** * Test that Elasticsearch resources are generated. @@ -29,6 +26,15 @@ public class StreamsElasticsearchResourceGeneratorTest { private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class); + public static final Predicate<File> jsonFilter = new Predicate<File>() { + @Override + public boolean apply(@Nullable File file) { + if( file.getName().endsWith(".json") ) + return true; + else return false; + } + }; + /** * Test that Elasticsearch resources are generated * @@ -39,28 +45,21 @@ public class StreamsElasticsearchResourceGeneratorTest { StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); - String sourceDirectory = "target/test-classes/streams-schemas"; + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory("target/generated-sources/test"); + config.setTargetDirectory("target/generated-resources/elasticsearch"); config.setExclusions(Sets.newHashSet("attachments")); config.setMaxDepth(2); StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); - Thread thread = new Thread(streamsElasticsearchResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } + streamsElasticsearchResourceGenerator.run(); + + File testOutput = config.getTargetDirectory(); - File testOutput = new File( "./target/generated-sources/test"); Predicate<File> jsonFilter = new Predicate<File>() { @Override public boolean apply(@Nullable File file) { @@ -107,33 +106,5 @@ public class StreamsElasticsearchResourceGeneratorTest { Assert.fail(); } -// String expectedDirectory = "target/test-classes/expected"; -// File testExpected = new File( expectedDirectory ); -// -// Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) -// .filter(cqlFilter); -// Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); -// -// int fails = 0; -// -// Iterator<File> iterator = expectedCollection.iterator(); -// while( iterator.hasNext() ) { -// File objectExpected = iterator.next(); -// String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); -// File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); -// LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); -// assert( objectActual.exists()); -// if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { -// LOGGER.info("Exact Match!"); -// } else { -// LOGGER.info("No Match!"); -// fails++; -// } -// } -// if( fails > 0 ) { -// LOGGER.info("Fails: {}", fails); -// Assert.fail(); -// } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml index 2aaa037..de74014 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/resources/streams-plugin-elasticsearch/pom.xml @@ -18,9 +18,10 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> <version>${project.version}</version> <scope>test</scope> + <type>test-jar</type> </dependency> </dependencies> @@ -32,19 +33,19 @@ <version>0.3-incubating-SNAPSHOT</version> <configuration> <sourcePaths> - <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath> - <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath> </sourcePaths> - <targetDirectory>target/generated-resources/test-mojo</targetDirectory> + <targetDirectory>target/generated-resources/elasticsearch-mojo</targetDirectory> </configuration> <executions> <execution> <goals> - <goal>elasticsearch</goal> + <goal>generate-resources</goal> </goals> </execution> </executions> @@ -55,9 +56,9 @@ <version>2.10</version> <configuration> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includeTypes>jar,test-jar</includeTypes> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/pom.xml index 94a0115..b29bd38 100644 --- a/streams-plugins/streams-plugin-hbase/pom.xml +++ b/streams-plugins/streams-plugin-hbase/pom.xml @@ -54,10 +54,17 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-util</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schema-activitystreams</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> @@ -230,9 +237,9 @@ </goals> <configuration> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> </configuration> </execution> </executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java index 2612ec2..3972a54 100644 --- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java @@ -1,6 +1,6 @@ package org.apache.streams.plugins.hbase; -import org.apache.streams.schema.GenerationConfig; +import org.apache.streams.util.schema.GenerationConfig; import org.jsonschema2pojo.DefaultGenerationConfig; import org.jsonschema2pojo.util.URLUtil; http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java index 649bdd9..71a8705 100644 --- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java @@ -3,21 +3,18 @@ package org.apache.streams.plugins.hbase; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Lists; -import org.apache.streams.schema.FieldType; -import org.apache.streams.schema.FieldUtil; -import org.apache.streams.schema.GenerationConfig; -import org.apache.streams.schema.Schema; -import org.apache.streams.schema.SchemaStore; +import org.apache.streams.util.schema.FieldType; +import org.apache.streams.util.schema.FieldUtil; +import org.apache.streams.util.schema.GenerationConfig; +import org.apache.streams.util.schema.Schema; +import org.apache.streams.util.schema.SchemaStore; +import org.apache.streams.util.schema.SchemaStoreImpl; import org.jsonschema2pojo.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.net.URI; import java.net.URL; import java.util.Iterator; import java.util.LinkedList; @@ -25,11 +22,11 @@ import java.util.List; import java.util.Map; import static com.google.common.base.Preconditions.checkNotNull; -import static org.apache.streams.schema.FileUtil.dropExtension; -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; -import static org.apache.streams.schema.FileUtil.resolveRecursive; -import static org.apache.streams.schema.FileUtil.swapExtension; -import static org.apache.streams.schema.FileUtil.writeFile; +import static org.apache.streams.util.schema.FileUtil.dropExtension; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.resolveRecursive; +import static org.apache.streams.util.schema.FileUtil.swapExtension; +import static org.apache.streams.util.schema.FileUtil.writeFile; /** * Created by sblackmon on 5/3/16. @@ -42,7 +39,7 @@ public class StreamsHbaseResourceGenerator implements Runnable { private StreamsHbaseGenerationConfig config; - private SchemaStore schemaStore = new SchemaStore(); + private SchemaStore schemaStore = new SchemaStoreImpl(); private int currentDepth = 0; @@ -60,17 +57,9 @@ public class StreamsHbaseResourceGenerator implements Runnable { config.setSourceDirectory(sourceDirectory); config.setTargetDirectory(targetDirectory); - StreamsHbaseResourceGenerator StreamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); - Thread thread = new Thread(StreamsHbaseResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } - return; + StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + streamsHbaseResourceGenerator.run(); + } public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) { http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java index 0db68f0..6bad40b 100644 --- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java @@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.util.List; -@Mojo( name = "hbase", +@Mojo( name = "generate-resources", defaultPhase = LifecyclePhase.GENERATE_RESOURCES ) -@Execute( goal = "hbase", +@Execute( goal = "generate-resources", phase = LifecyclePhase.GENERATE_RESOURCES ) public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo { @@ -64,30 +64,8 @@ public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo { config.setTargetDirectory(targetDirectory); StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + streamsHbaseResourceGenerator.run(); - Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread th, Throwable ex) { - LOGGER.error("Exception", ex); - mojoFailureException = new MojoFailureException("Exception", ex); - } - }; - Thread.setDefaultUncaughtExceptionHandler(h); - Thread thread = new Thread(streamsHbaseResourceGenerator); - thread.setUncaughtExceptionHandler(h); - try { - thread.start(); - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - mojoFailureException = new MojoFailureException("Exception", e); - } - - if( mojoFailureException != null ) - throw mojoFailureException; - - return; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md index 858cb85..0a0c308 100644 --- a/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md +++ b/streams-plugins/streams-plugin-hbase/src/site/markdown/index.md @@ -3,17 +3,35 @@ org.apache.streams.plugins:streams-plugin-hbase streams-plugin-hbase generates resources from json schemas to assist with indexing of json data using Apache HBase. -#### Usage +### Usage + +Output will be placed in target/generated-resources/hbase by default + +[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml") + +##### Maven Run within a module containing a src/main/jsonschema directory mvn org.apache.streams.plugins:streams-plugin-hbase:0.3-incubating-SNAPSHOT:hbase -Output will be placed in target/generated-resources/hive by default +##### SDK -#### Example +Embed within your own java code -[streams-plugin-hbase/pom.xml](streams-plugin-hbase/pom.xml "streams-plugin-hbase/pom.xml") + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + config.setSourceDirectory("src/main/jsonschema"); + config.setTargetDirectory("target/generated-resources"); + StreamsHbaseResourceGenerator generator = new StreamsHbaseResourceGenerator(config); + generator.run(); + +##### CLI + +Run from CLI without Maven + + java -jar streams-plugin-hbase-jar-with-dependencies.jar StreamsHbaseResourceGenerator src/main/jsonschema target/generated-resources + +#### Example #### Documentation http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java index 290d601..24eba24 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java @@ -21,7 +21,7 @@ public class StreamsHbaseResourceGeneratorCLITest { @Test public void testStreamsHiveResourceGeneratorCLI() throws Exception { - String sourceDirectory = "target/test-classes/streams-schemas"; + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; String targetDirectory = "target/generated-resources/hbase-cli"; List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java index 6293021..fc8c77a 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoTest.java @@ -51,7 +51,7 @@ public class StreamsHbaseResourceGeneratorMojoTest extends TestCase { verifier.resetStreams(); - File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/test-mojo"); + File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo"); assert( testOutput != null ); assert( testOutput.exists() == true ); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java index 02b0214..91a1a05 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java @@ -8,7 +8,6 @@ import org.apache.commons.io.FileUtils; import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig; import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +17,7 @@ import java.io.File; import java.util.Collection; import java.util.Iterator; -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; /** * Test that Activity beans are compatible with the example activities in the spec. @@ -46,11 +45,11 @@ public class StreamsHbaseResourceGeneratorTest { StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); - String sourceDirectory = "target/test-classes/streams-schemas"; + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory("target/generated-resources/test"); + config.setTargetDirectory("target/generated-resources/hbase"); config.setExclusions(Sets.newHashSet("attachments")); @@ -58,19 +57,10 @@ public class StreamsHbaseResourceGeneratorTest { config.setMaxDepth(2); StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); - Thread thread = new Thread(streamsHbaseResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } + streamsHbaseResourceGenerator.run(); File testOutput = config.getTargetDirectory(); - assert( testOutput != null ); assert( testOutput.exists() == true ); assert( testOutput.isDirectory() == true ); @@ -108,18 +98,5 @@ public class StreamsHbaseResourceGeneratorTest { Assert.fail(); } - -// assert( new File(testOutput + "/traits").exists() == true ); -// assert( new File(testOutput + "/traits").isDirectory() == true ); -// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null ); -// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 ); -// assert( new File(testOutput + "/objectTypes").exists() == true ); -// assert( new File(testOutput + "/objectTypes").isDirectory() == true ); -// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null ); -// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 ); -// assert( new File(testOutput + "/verbs").exists() == true ); -// assert( new File(testOutput + "/verbs").isDirectory() == true ); -// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null ); -// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 ); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml b/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml index 9aa7d8d..a3fbeeb 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml +++ b/streams-plugins/streams-plugin-hbase/src/test/resources/streams-plugin-hbase/pom.xml @@ -18,9 +18,10 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> <version>${project.version}</version> <scope>test</scope> + <type>test-jar</type> </dependency> </dependencies> @@ -32,20 +33,20 @@ <version>0.3-incubating-SNAPSHOT</version> <configuration> <sourcePaths> - <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath> - <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath> </sourcePaths> - <targetDirectory>target/generated-resources/test-mojo</targetDirectory> + <targetDirectory>target/generated-resources/hbase-mojo</targetDirectory> <columnFamily>cf</columnFamily> </configuration> <executions> <execution> <goals> - <goal>hbase</goal> + <goal>generate-resources</goal> </goals> </execution> </executions> @@ -56,9 +57,9 @@ <version>2.10</version> <configuration> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includeTypes>jar,test-jar</includeTypes> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/pom.xml index 515314a..38c69c1 100644 --- a/streams-plugins/streams-plugin-hive/pom.xml +++ b/streams-plugins/streams-plugin-hive/pom.xml @@ -54,10 +54,17 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-util</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schema-activitystreams</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> @@ -230,9 +237,9 @@ </goals> <configuration> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> </configuration> </execution> </executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java index 7e3bf35..f957757 100644 --- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java @@ -1,11 +1,10 @@ package org.apache.streams.plugins.hive; -import org.apache.streams.schema.GenerationConfig; +import org.apache.streams.util.schema.GenerationConfig; import org.jsonschema2pojo.DefaultGenerationConfig; import org.jsonschema2pojo.util.URLUtil; import java.io.File; -import java.io.FileFilter; import java.net.URL; import java.util.ArrayList; import java.util.Collections; http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java index c09f7dd..60c59ca 100644 --- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java @@ -1,33 +1,29 @@ package org.apache.streams.plugins.hive; -import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import org.apache.streams.schema.FieldType; -import org.apache.streams.schema.FieldUtil; -import org.apache.streams.schema.FileUtil; -import org.apache.streams.schema.GenerationConfig; -import org.apache.streams.schema.Schema; -import org.apache.streams.schema.SchemaStore; -import org.apache.streams.schema.SchemaUtil; -import org.apache.streams.schema.URIUtil; +import org.apache.streams.util.schema.FieldType; +import org.apache.streams.util.schema.FieldUtil; +import org.apache.streams.util.schema.FileUtil; +import org.apache.streams.util.schema.GenerationConfig; +import org.apache.streams.util.schema.Schema; +import org.apache.streams.util.schema.SchemaStore; +import org.apache.streams.util.schema.SchemaStoreImpl; import org.jsonschema2pojo.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.net.URI; import java.net.URL; import java.util.*; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.commons.lang3.StringUtils.defaultString; -import static org.apache.streams.schema.FileUtil.*; +import static org.apache.streams.util.schema.FileUtil.*; /** * Generates hive table definitions for using org.openx.data.jsonserde.JsonSerDe on new-line delimited json documents. @@ -42,7 +38,7 @@ public class StreamsHiveResourceGenerator implements Runnable { private StreamsHiveGenerationConfig config; - private SchemaStore schemaStore = new SchemaStore(); + private SchemaStore schemaStore = new SchemaStoreImpl(); private int currentDepth = 0; @@ -60,17 +56,8 @@ public class StreamsHiveResourceGenerator implements Runnable { config.setSourceDirectory(sourceDirectory); config.setTargetDirectory(targetDirectory); - StreamsHiveResourceGenerator streamsPojoSourceGenerator = new StreamsHiveResourceGenerator(config); - Thread thread = new Thread(streamsPojoSourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } - return; + StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config); + streamsHiveResourceGenerator.run(); } public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) { http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java index 16466cf..76e3a18 100644 --- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGeneratorMojo.java @@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.util.List; -@Mojo( name = "hive", +@Mojo( name = "generate-resources", defaultPhase = LifecyclePhase.GENERATE_RESOURCES ) -@Execute( goal = "hive", +@Execute( goal = "generate-resources", phase = LifecyclePhase.GENERATE_RESOURCES ) public class StreamsHiveResourceGeneratorMojo extends AbstractMojo { @@ -30,15 +30,6 @@ public class StreamsHiveResourceGeneratorMojo extends AbstractMojo { @Component private MavenProject project; -// @Component -// private Settings settings; -// -// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) -// protected ArtifactRepository localRepository; -// -// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only -// private PluginDescriptor plugin; -// @Parameter( defaultValue = "${project.basedir}", readonly = true ) private File basedir; @@ -65,29 +56,8 @@ public class StreamsHiveResourceGeneratorMojo extends AbstractMojo { StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config); - Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread th, Throwable ex) { - LOGGER.error("Exception", ex); - mojoFailureException = new MojoFailureException("Exception", ex); - } - }; - Thread.setDefaultUncaughtExceptionHandler(h); - Thread thread = new Thread(streamsHiveResourceGenerator); - thread.setUncaughtExceptionHandler(h); - try { - thread.start(); - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - mojoFailureException = new MojoFailureException("Exception", e); - } - - if( mojoFailureException != null ) - throw mojoFailureException; - - return; + streamsHiveResourceGenerator.run(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/site/markdown/index.md b/streams-plugins/streams-plugin-hive/src/site/markdown/index.md index 4df9103..12ce831 100644 --- a/streams-plugins/streams-plugin-hive/src/site/markdown/index.md +++ b/streams-plugins/streams-plugin-hive/src/site/markdown/index.md @@ -3,18 +3,36 @@ org.apache.streams.plugins:streams-plugin-hive streams-plugin-hive generates resources from json schemas to assist with analysis of json data using Apache Hive. -#### Usage +### Usage -Run within a module containing a src/main/jsonschema directory +Output will be placed in target/generated-resources/hive by default - mvn org.apache.streams.plugins:streams-plugin-hive:0.3-incubating-SNAPSHOT:hive +##### Maven -Output will be placed in target/generated-resources/hive by default +Run within a module containing a src/main/jsonschema directory -#### Example + mvn org.apache.streams.plugins:streams-plugin-hive:0.3-incubating-SNAPSHOT:generate-resources [streams-plugin-hive/pom.xml](streams-plugin-hive/pom.xml "streams-plugin-hive/pom.xml") +##### SDK + +Embed within your own java code + + StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig(); + config.setSourceDirectory("src/main/jsonschema"); + config.setTargetDirectory("target/generated-resources"); + StreamsHiveGenerationConfig generator = new StreamsHiveGenerationConfig(config); + generator.run(); + +##### CLI + +Run from CLI without Maven + + java -jar streams-plugin-hive-jar-with-dependencies.jar StreamsHiveResourceGenerator src/main/jsonschema target/generated-resources + +#### Example + #### Documentation [JavaDocs](apidocs/index.html "JavaDocs") http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java index 21c82f9..ed08c31 100644 --- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java +++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorCLITest.java @@ -21,7 +21,7 @@ public class StreamsHiveResourceGeneratorCLITest { @Test public void testStreamsHiveResourceGeneratorCLI() throws Exception { - String sourceDirectory = "target/test-classes/streams-schemas"; + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; String targetDirectory = "target/generated-resources/hive-cli"; List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java index b2f3e14..e2fbecc 100644 --- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java +++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorMojoTest.java @@ -50,7 +50,7 @@ public class StreamsHiveResourceGeneratorMojoTest extends TestCase { verifier.resetStreams(); - File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/test-mojo"); + File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hive-mojo"); assert( testOutput != null ); assert( testOutput.exists() == true ); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java index 3944ddf..9778bb6 100644 --- a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java +++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsHiveResourceGeneratorTest.java @@ -18,7 +18,7 @@ import java.io.File; import java.util.Collection; import java.util.Iterator; -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; /** * Test that Activity beans are compatible with the example activities in the spec. @@ -47,26 +47,18 @@ public class StreamsHiveResourceGeneratorTest { StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig(); - String sourceDirectory = "target/test-classes/streams-schemas"; + String sourceDirectory = "target/test-classes/streams-schema-activitystreams"; config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory("target/generated-sources/test"); + config.setTargetDirectory("target/generated-resources/test"); config.setExclusions(Sets.newHashSet("attachments")); config.setMaxDepth(2); StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config); - Thread thread = new Thread(streamsHiveResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } + streamsHiveResourceGenerator.run(); File testOutput = config.getTargetDirectory(); @@ -106,19 +98,5 @@ public class StreamsHiveResourceGeneratorTest { LOGGER.info("Fails: {}", fails); Assert.fail(); } - - -// assert( new File(testOutput + "/traits").exists() == true ); -// assert( new File(testOutput + "/traits").isDirectory() == true ); -// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null ); -// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 ); -// assert( new File(testOutput + "/objectTypes").exists() == true ); -// assert( new File(testOutput + "/objectTypes").isDirectory() == true ); -// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null ); -// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 ); -// assert( new File(testOutput + "/verbs").exists() == true ); -// assert( new File(testOutput + "/verbs").isDirectory() == true ); -// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null ); -// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 ); - } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml index ae4bb66..40931ae 100644 --- a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml +++ b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml @@ -18,9 +18,10 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> <version>${project.version}</version> <scope>test</scope> + <type>test-jar</type> </dependency> </dependencies> @@ -32,19 +33,19 @@ <version>0.3-incubating-SNAPSHOT</version> <configuration> <sourcePaths> - <sourcePath>target/test-classes/streams-schemas/activity.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/collection.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/media_link.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/object.json</sourcePath> - <sourcePath>target/test-classes/streams-schemas/objectTypes</sourcePath> - <sourcePath>target/test-classes/streams-schemas/verbs</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/activity.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/collection.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/media_link.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/object.json</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/objectTypes</sourcePath> + <sourcePath>target/test-classes/streams-schema-activitystreams/verbs</sourcePath> </sourcePaths> - <targetDirectory>target/generated-resources/test-mojo</targetDirectory> + <targetDirectory>target/generated-resources/hive-mojo</targetDirectory> </configuration> <executions> <execution> <goals> - <goal>hive</goal> + <goal>generate-resources</goal> </goals> </execution> </executions> @@ -55,9 +56,9 @@ <version>2.10</version> <configuration> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includeTypes>jar,test-jar</includeTypes> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-pig/pom.xml b/streams-plugins/streams-plugin-pig/pom.xml index ef59587..0c7bf1a 100644 --- a/streams-plugins/streams-plugin-pig/pom.xml +++ b/streams-plugins/streams-plugin-pig/pom.xml @@ -54,10 +54,17 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-schemas</artifactId> + <artifactId>streams-util</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-schema-activitystreams</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> @@ -230,9 +237,9 @@ </goals> <configuration> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-schemas</includeArtifactIds> + <includeArtifactIds>streams-schema-activitystreams</includeArtifactIds> <includes>**/*.json</includes> - <outputDirectory>${project.build.directory}/test-classes/streams-schemas</outputDirectory> + <outputDirectory>${project.build.directory}/test-classes/streams-schema-activitystreams</outputDirectory> </configuration> </execution> </executions> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java index 5aa55ed..1b346f4 100644 --- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java +++ b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigGenerationConfig.java @@ -1,11 +1,10 @@ package org.apache.streams.plugins.pig; -import org.apache.streams.schema.GenerationConfig; +import org.apache.streams.util.schema.GenerationConfig; import org.jsonschema2pojo.DefaultGenerationConfig; import org.jsonschema2pojo.util.URLUtil; import java.io.File; -import java.io.FileFilter; import java.net.URL; import java.util.ArrayList; import java.util.Collections; http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java index 072e1b6..f1f94a6 100644 --- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java +++ b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGenerator.java @@ -6,13 +6,13 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import org.apache.streams.plugins.pig.StreamsPigGenerationConfig; -import org.apache.streams.schema.FieldType; -import org.apache.streams.schema.FieldUtil; -import org.apache.streams.schema.FileUtil; -import org.apache.streams.schema.GenerationConfig; -import org.apache.streams.schema.Schema; -import org.apache.streams.schema.SchemaStore; +import org.apache.streams.util.schema.FieldType; +import org.apache.streams.util.schema.FieldUtil; +import org.apache.streams.util.schema.FileUtil; +import org.apache.streams.util.schema.GenerationConfig; +import org.apache.streams.util.schema.Schema; +import org.apache.streams.util.schema.SchemaStore; +import org.apache.streams.util.schema.SchemaStoreImpl; import org.jsonschema2pojo.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,10 +25,10 @@ import java.util.List; import java.util.Map; import static com.google.common.base.Preconditions.checkNotNull; -import static org.apache.streams.schema.FileUtil.dropExtension; -import static org.apache.streams.schema.FileUtil.dropSourcePathPrefix; -import static org.apache.streams.schema.FileUtil.swapExtension; -import static org.apache.streams.schema.FileUtil.writeFile; +import static org.apache.streams.util.schema.FileUtil.dropExtension; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.swapExtension; +import static org.apache.streams.util.schema.FileUtil.writeFile; public class StreamsPigResourceGenerator implements Runnable { @@ -38,7 +38,7 @@ public class StreamsPigResourceGenerator implements Runnable { private StreamsPigGenerationConfig config; - private SchemaStore schemaStore = new SchemaStore(); + private SchemaStore schemaStore = new SchemaStoreImpl(); private int currentDepth = 0; @@ -57,16 +57,8 @@ public class StreamsPigResourceGenerator implements Runnable { config.setTargetDirectory(targetDirectory); StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config); - Thread thread = new Thread(streamsPigResourceGenerator); - thread.start(); - try { - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - } - return; + streamsPigResourceGenerator.run(); + } public StreamsPigResourceGenerator(StreamsPigGenerationConfig config) { @@ -160,13 +152,9 @@ public class StreamsPigResourceGenerator implements Runnable { if (fieldType != null ) { switch (fieldType) { case ARRAY: - ObjectNode itemsNode = FieldUtil.resolveItemsNode(fieldNode); - if( itemsNode == null ) { - ObjectNode resolvedItems = schemaStore.resolveProperties(schema, fieldNode, fieldId); - itemsNode = FieldUtil.resolveItemsNode(); - } - if( itemsNode != null && currentDepth <= config.getMaxDepth()) { - StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator); + ObjectNode resolvedItems = schemaStore.resolveItems(schema, fieldNode, fieldId); + if( resolvedItems != null && currentDepth <= config.getMaxDepth()) { + StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, resolvedItems, seperator); if( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) { fieldStrings.add(arrayItemsBuilder.toString()); } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6187265f/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java index 4763b60..571c75a 100644 --- a/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java +++ b/streams-plugins/streams-plugin-pig/src/main/java/org/apache/streams/plugins/pig/StreamsPigResourceGeneratorMojo.java @@ -15,10 +15,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.util.List; -@Mojo( name = "pig", +@Mojo( name = "generate-resources", defaultPhase = LifecyclePhase.GENERATE_RESOURCES ) -@Execute( goal = "pig", +@Execute( goal = "generate-resources", phase = LifecyclePhase.GENERATE_RESOURCES ) public class StreamsPigResourceGeneratorMojo extends AbstractMojo { @@ -30,15 +30,6 @@ public class StreamsPigResourceGeneratorMojo extends AbstractMojo { @Component private MavenProject project; -// @Component -// private Settings settings; -// -// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) -// protected ArtifactRepository localRepository; -// -// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only -// private PluginDescriptor plugin; -// @Parameter( defaultValue = "${project.basedir}", readonly = true ) private File basedir; @@ -64,30 +55,7 @@ public class StreamsPigResourceGeneratorMojo extends AbstractMojo { config.setTargetDirectory(targetDirectory); StreamsPigResourceGenerator streamsPigResourceGenerator = new StreamsPigResourceGenerator(config); - - Thread.UncaughtExceptionHandler h = new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread th, Throwable ex) { - LOGGER.error("Exception", ex); - mojoFailureException = new MojoFailureException("Exception", ex); - } - }; - Thread.setDefaultUncaughtExceptionHandler(h); - Thread thread = new Thread(streamsPigResourceGenerator); - thread.setUncaughtExceptionHandler(h); - try { - thread.start(); - thread.join(); - } catch (InterruptedException e) { - LOGGER.error("InterruptedException", e); - } catch (Exception e) { - LOGGER.error("Exception", e); - mojoFailureException = new MojoFailureException("Exception", e); - } - - if( mojoFailureException != null ) - throw mojoFailureException; - - return; + streamsPigResourceGenerator.run(); } } \ No newline at end of file