Repository: kudu Updated Branches: refs/heads/master 978cf3912 -> f62bbdd64
[Java] Fix the use of resource files in tests This is preliminary work needed to run the java tests on dist-test. Our tests currently reference files/resources that are not available in the output test artifacts. This means our tests work when they are run from source in the expected location but fail when they are run from the compiled artifacts. This fixes those failing tests by discovering the correct location via Class.getResource and ensuring all needed files are packaged in the resources directory. Change-Id: Ia3a90bac67652b58f2eef802c509983c68d8a299 Reviewed-on: http://gerrit.cloudera.org:8080/10924 Reviewed-by: Andrew Wong <aw...@cloudera.com> Tested-by: Grant Henke <granthe...@apache.org> Reviewed-by: Mike Percy <mpe...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fff3cf09 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fff3cf09 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fff3cf09 Branch: refs/heads/master Commit: fff3cf0923782cab8a43e081caeccc88a53f11f9 Parents: 978cf39 Author: Grant Henke <granthe...@apache.org> Authored: Wed Jul 11 14:17:04 2018 -0500 Committer: Grant Henke <granthe...@apache.org> Committed: Thu Jul 12 17:14:53 2018 +0000 ---------------------------------------------------------------------- java/kudu-flume-sink/build.gradle | 6 ++++++ java/kudu-flume-sink/pom.xml | 3 +++ .../avro/testAvroKuduOperationsProducer.avsc | 14 ------------- .../sink/AvroKuduOperationsProducerTest.java | 22 ++++++++++---------- .../testAvroKuduOperationsProducer.avsc | 14 +++++++++++++ .../spark/tools/TestImportExportFiles.scala | 10 +++++---- 6 files changed, 40 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/build.gradle ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/build.gradle b/java/kudu-flume-sink/build.gradle index 5767103..e09f0ae 100644 --- a/java/kudu-flume-sink/build.gradle +++ b/java/kudu-flume-sink/build.gradle @@ -33,4 +33,10 @@ dependencies { testCompile project(path: ":kudu-client", configuration: "shadowTest") testCompile libs.junit +} + +// Configure the Avro plugin to compile the schemas in the +// resource directory so the schemas are available in tests. +generateTestAvroJava { + source("src/test/resources") } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/pom.xml ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml index aa8a859..7b664fb 100644 --- a/java/kudu-flume-sink/pom.xml +++ b/java/kudu-flume-sink/pom.xml @@ -126,6 +126,9 @@ <goal>schema</goal> </goals> <configuration> + <!-- Configure the Avro plugin to compile the schemas in the + resource directory so the schemas are available in tests. --> + <testSourceDirectory>${basedir}/src/test/resources</testSourceDirectory> <enableDecimalLogicalType>true</enableDecimalLogicalType> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc b/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc deleted file mode 100644 index 6bcf6d2..0000000 --- a/java/kudu-flume-sink/src/test/avro/testAvroKuduOperationsProducer.avsc +++ /dev/null @@ -1,14 +0,0 @@ -{"namespace": "org.apache.kudu.flume.sink", - "type": "record", - "name": "AvroKuduOperationsProducerTestRecord", - "fields": [ - {"name": "key", "type": "int"}, - {"name": "longField", "type": "long"}, - {"name": "doubleField", "type": "double"}, - {"name": "nullableField", "type": ["string", "null"]}, - {"name": "stringField", "type": "string"}, - {"name": "decimalField", "type": { - "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1} - } - ] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java index 8e3c8dc..42c94cd 100644 --- a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java +++ b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/AvroKuduOperationsProducerTest.java @@ -33,8 +33,9 @@ import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.IOException; import java.math.BigDecimal; +import java.net.URL; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -51,15 +52,12 @@ import org.apache.avro.specific.SpecificDatumWriter; import org.apache.flume.Channel; import org.apache.flume.Context; import org.apache.flume.Event; -import org.apache.flume.FlumeException; import org.apache.flume.Sink; import org.apache.flume.Transaction; import org.apache.flume.channel.MemoryChannel; import org.apache.flume.conf.Configurables; import org.apache.flume.event.EventBuilder; import org.apache.kudu.util.DecimalUtil; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.apache.kudu.ColumnSchema; @@ -70,13 +68,17 @@ import org.apache.kudu.client.CreateTableOptions; import org.apache.kudu.client.KuduTable; public class AvroKuduOperationsProducerTest extends BaseKuduTest { - private static final String schemaPath = "src/test/avro/testAvroKuduOperationsProducer.avsc"; + private static String schemaUriString; private static String schemaLiteral; static { try { - schemaLiteral = Files.toString(new File(schemaPath), UTF_8); - } catch (IOException e) { + String schemaPath = "/testAvroKuduOperationsProducer.avsc"; + URL schemaUrl = AvroKuduOperationsProducerTest.class.getResource(schemaPath); + File schemaFile = Paths.get(schemaUrl.toURI()).toFile(); + schemaUriString = schemaFile.getAbsoluteFile().toURI().toString(); + schemaLiteral = Files.asCharSource(schemaFile, UTF_8).read(); + } catch (Exception e) { throw new RuntimeException(e); } } @@ -115,9 +117,8 @@ public class AvroKuduOperationsProducerTest extends BaseKuduTest { KuduTable table = createNewTable( String.format("test%sevents%s", eventCount, schemaLocation)); String tableName = table.getName(); - String schemaURI = new File(schemaPath).getAbsoluteFile().toURI().toString(); Context ctx = schemaLocation != SchemaLocation.GLOBAL ? new Context() - : new Context(ImmutableMap.of(PRODUCER_PREFIX + SCHEMA_PROP, schemaURI)); + : new Context(ImmutableMap.of(PRODUCER_PREFIX + SCHEMA_PROP, schemaUriString)); KuduSink sink = createSink(tableName, ctx); Channel channel = new MemoryChannel(); @@ -190,8 +191,7 @@ public class AvroKuduOperationsProducerTest extends BaseKuduTest { encoder.flush(); Event e = EventBuilder.withBody(out.toByteArray()); if (schemaLocation == SchemaLocation.URL) { - String schemaURI = new File(schemaPath).getAbsoluteFile().toURI().toString(); - e.setHeaders(ImmutableMap.of(SCHEMA_URL_HEADER, schemaURI)); + e.setHeaders(ImmutableMap.of(SCHEMA_URL_HEADER, schemaUriString)); } else if (schemaLocation == SchemaLocation.LITERAL) { e.setHeaders(ImmutableMap.of(SCHEMA_LITERAL_HEADER, schemaLiteral)); } http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc ---------------------------------------------------------------------- diff --git a/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc b/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc new file mode 100644 index 0000000..6bcf6d2 --- /dev/null +++ b/java/kudu-flume-sink/src/test/resources/testAvroKuduOperationsProducer.avsc @@ -0,0 +1,14 @@ +{"namespace": "org.apache.kudu.flume.sink", + "type": "record", + "name": "AvroKuduOperationsProducerTestRecord", + "fields": [ + {"name": "key", "type": "int"}, + {"name": "longField", "type": "long"}, + {"name": "doubleField", "type": "double"}, + {"name": "nullableField", "type": ["string", "null"]}, + {"name": "stringField", "type": "string"}, + {"name": "decimalField", "type": { + "type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 1} + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/fff3cf09/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala ---------------------------------------------------------------------- diff --git a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala index e470ddc..ceca8d1 100644 --- a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala +++ b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/TestImportExportFiles.scala @@ -17,7 +17,7 @@ package org.apache.kudu.spark.tools -import java.io.File +import java.nio.file.Paths import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder import org.apache.kudu.{Schema, Type} @@ -35,7 +35,7 @@ import scala.collection.JavaConverters._ class TestImportExportFiles extends FunSuite with TestContext with Matchers { private val TABLE_NAME: String = "TestImportExportFiles" - private val TABLE_DATA_PATH: String = "src/test/resources/TestImportExportFiles.csv" + private val TABLE_DATA_PATH: String = "/TestImportExportFiles.csv" test("Spark Import Export") { val schema: Schema = { @@ -51,12 +51,14 @@ class TestImportExportFiles extends FunSuite with TestContext with Matchers { .setNumReplicas(1) kuduClient.createTable(TABLE_NAME, schema, tableOptions) - val dataPath = new File(TABLE_DATA_PATH).getAbsolutePath + // Get the absolute path of the resource file. + val schemaResource = classOf[TestImportExportFiles].getResource(TABLE_DATA_PATH) + val dataPath = Paths.get(schemaResource.toURI).toAbsolutePath ImportExportFiles.testMain(Array("--operation=import", "--format=csv", s"--master-addrs=${miniCluster.getMasterAddresses}", - s"--path=$TABLE_DATA_PATH", + s"--path=$dataPath", s"--table-name=$TABLE_NAME", "--delimiter=,", "--header=true",
