This is an automated email from the ASF dual-hosted git repository. vinoth pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push: new d65efe6 [HUDI-780] Migrate test cases to Junit 5 (#1504) d65efe6 is described below commit d65efe659d36dfb80be16a6389c5aba2f8701c39 Author: Raymond Xu <2701446+xushi...@users.noreply.github.com> AuthorDate: Wed Apr 15 12:35:01 2020 -0700 [HUDI-780] Migrate test cases to Junit 5 (#1504) --- hudi-cli/pom.xml | 31 +++ hudi-client/pom.xml | 24 ++ .../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +- .../org/apache/hudi/metrics/TestHoodieMetrics.java | 15 +- hudi-common/pom.xml | 24 ++ .../org/apache/hudi/avro/TestHoodieAvroUtils.java | 28 +- .../hudi/avro/TestHoodieAvroWriteSupport.java | 21 +- hudi-hadoop-mr/pom.xml | 24 ++ .../apache/hudi/hadoop/InputPathHandlerTest.java | 28 +- .../org/apache/hudi/hadoop/TestAnnotation.java | 4 +- .../hudi/hadoop/TestRecordReaderValueIterator.java | 13 +- hudi-hive-sync/pom.xml | 26 +- .../org/apache/hudi/hive/TestHiveSyncTool.java | 297 ++++++++++----------- .../test/java/org/apache/hudi/hive/TestUtil.java | 2 +- hudi-integ-test/pom.xml | 24 +- .../java/org/apache/hudi/integ/ITTestBase.java | 18 +- .../org/apache/hudi/integ/ITTestHoodieDemo.java | 20 +- .../org/apache/hudi/integ/ITTestHoodieSanity.java | 33 +-- hudi-spark/pom.xml | 29 +- hudi-spark/src/test/java/DataSourceUtilsTest.java | 6 +- hudi-spark/src/test/scala/TestDataSource.scala | 21 +- .../src/test/scala/TestDataSourceDefaults.scala | 18 +- hudi-timeline-service/pom.xml | 22 +- hudi-utilities/pom.xml | 24 ++ .../transform/TestChainedTransformer.java | 14 +- pom.xml | 34 ++- 26 files changed, 522 insertions(+), 282 deletions(-) diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml index 6aecd3b..2f0f4b4 100644 --- a/hudi-cli/pom.xml +++ b/hudi-cli/pom.xml @@ -248,5 +248,36 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> </dependency> + + <!-- Test --> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> diff --git a/hudi-client/pom.xml b/hudi-client/pom.xml index 7650555..9c716f2 100644 --- a/hudi-client/pom.xml +++ b/hudi-client/pom.xml @@ -240,6 +240,30 @@ </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> diff --git a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java index fa0f1da..063f64e 100644 --- a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java +++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java @@ -20,10 +20,10 @@ package org.apache.hudi.metrics; import org.apache.hudi.config.HoodieWriteConfig; -import org.junit.Test; +import org.junit.jupiter.api.Test; import static org.apache.hudi.metrics.Metrics.registerGauge; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java index 9716dd6..41842b1 100755 --- a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java +++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java @@ -22,22 +22,23 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.config.HoodieWriteConfig; import com.codahale.metrics.Timer; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Random; +import java.util.stream.Stream; import static org.apache.hudi.metrics.Metrics.registerGauge; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TestHoodieMetrics { + private HoodieMetrics metrics; - @Before + @BeforeEach public void start() { HoodieWriteConfig config = mock(HoodieWriteConfig.class); when(config.isMetricsOn()).thenReturn(true); @@ -97,7 +98,7 @@ public class TestHoodieMetrics { assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesFinalized); // Commit / deltacommit / compaction metrics - Arrays.asList("commit", "deltacommit", "compaction").stream().forEach(action -> { + Stream.of("commit", "deltacommit", "compaction").forEach(action -> { Timer.Context commitTimer = action.equals("commit") ? metrics.getCommitCtx() : action.equals("deltacommit") ? metrics.getDeltaCommitCtx() : metrics.getCompactionCtx(); diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index 71904aa..424f6c1 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -154,6 +154,30 @@ </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java index edcb37d..e2c1266 100644 --- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java @@ -21,11 +21,15 @@ package org.apache.hudi.avro; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + /** * Tests hoodie avro utilities. */ @@ -59,18 +63,18 @@ public class TestHoodieAvroUtils { continue; } - Assert.assertNotNull("field name is null", field.name()); + assertNotNull(field.name(), "field name is null"); Map<String, Object> props = field.getObjectProps(); - Assert.assertNotNull("The property is null", props); + assertNotNull(props, "The property is null"); if (field.name().equals("pii_col")) { piiPresent = true; - Assert.assertTrue("sensitivity_level is removed in field 'pii_col'", props.containsKey("column_category")); + assertTrue(props.containsKey("column_category"), "sensitivity_level is removed in field 'pii_col'"); } else { - Assert.assertEquals("The property shows up but not set", 0, props.size()); + assertEquals(0, props.size(), "The property shows up but not set"); } } - Assert.assertTrue("column pii_col doesn't show up", piiPresent); + assertTrue(piiPresent, "column pii_col doesn't show up"); } @Test @@ -81,8 +85,8 @@ public class TestHoodieAvroUtils { rec.put("pii_col", "val2"); rec.put("timestamp", 3.5); GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, new Schema.Parser().parse(EVOLVED_SCHEMA)); - Assert.assertEquals(rec1.get("new_col1"), "dummy_val"); - Assert.assertNull(rec1.get("new_col2")); + assertEquals(rec1.get("new_col1"), "dummy_val"); + assertNull(rec1.get("new_col2")); } @Test @@ -93,8 +97,8 @@ public class TestHoodieAvroUtils { rec.put("pii_col", "val2"); rec.put("timestamp", 3.5); GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, new Schema.Parser().parse(EVOLVED_SCHEMA)); - Assert.assertEquals(rec1.get("new_col1"), "dummy_val"); - Assert.assertNull(rec1.get("new_col2")); + assertEquals(rec1.get("new_col1"), "dummy_val"); + assertNull(rec1.get("new_col2")); } @Test @@ -105,6 +109,6 @@ public class TestHoodieAvroUtils { rec.put("pii_col", "val2"); rec.put("timestamp", 3.5); GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, new Schema.Parser().parse(SCHEMA_WITH_METADATA_FIELD)); - Assert.assertNull(rec1.get("_hoodie_commit_time")); + assertNull(rec1.get("_hoodie_commit_time")); } } diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java index 434807d..72f1453 100644 --- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java +++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java @@ -18,20 +18,20 @@ package org.apache.hudi.avro; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; -import org.apache.hadoop.fs.Path; import org.apache.hudi.common.bloom.BloomFilter; import org.apache.hudi.common.bloom.BloomFilterFactory; import org.apache.hudi.common.bloom.BloomFilterTypeCode; import org.apache.hudi.common.model.HoodieRecord; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroSchemaConverter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; import java.util.ArrayList; @@ -40,16 +40,13 @@ import java.util.UUID; public class TestHoodieAvroWriteSupport { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - @Test - public void testAddKey() throws IOException { + public void testAddKey(@TempDir java.nio.file.Path tempDir) throws IOException { List<String> rowKeys = new ArrayList<>(); for (int i = 0; i < 1000; i++) { rowKeys.add(UUID.randomUUID().toString()); } - String filePath = folder.getRoot() + "/test.parquet"; + String filePath = tempDir.resolve("test.parquet").toAbsolutePath().toString(); Schema schema = HoodieAvroUtils.getRecordKeySchema(); BloomFilter filter = BloomFilterFactory.createBloomFilter( 1000, 0.0001, 10000, diff --git a/hudi-hadoop-mr/pom.xml b/hudi-hadoop-mr/pom.xml index 578b9a0..8983ca5 100644 --- a/hudi-hadoop-mr/pom.xml +++ b/hudi-hadoop-mr/pom.xml @@ -103,6 +103,30 @@ </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputPathHandlerTest.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputPathHandlerTest.java index 2c927f8..5a364f7 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputPathHandlerTest.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputPathHandlerTest.java @@ -29,10 +29,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; import java.util.ArrayList; @@ -40,7 +40,7 @@ import java.util.List; import java.util.Properties; import java.util.stream.Collectors; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; public class InputPathHandlerTest { @@ -54,6 +54,9 @@ public class InputPathHandlerTest { // non Hoodie table public static final String TRIPS_STATS_TEST_NAME = "trips_stats"; + @TempDir + static java.nio.file.Path parentPath; + private static MiniDFSCluster dfsCluster; private static DistributedFileSystem dfs; private static HdfsTestService hdfsTestService; @@ -68,7 +71,7 @@ public class InputPathHandlerTest { private static List<Path> nonHoodiePaths; private static List<Path> inputPaths; - @BeforeClass + @BeforeAll public static void setUpDFS() throws IOException { // Need to closeAll to clear FileSystem.Cache, required because DFS and LocalFS used in the // same JVM @@ -86,7 +89,7 @@ public class InputPathHandlerTest { initTables(); } - @AfterClass + @AfterAll public static void cleanUp() throws Exception { if (hdfsTestService != null) { hdfsTestService.stop(); @@ -101,13 +104,10 @@ public class InputPathHandlerTest { } static void initTables() throws IOException { - // Create a temp folder as the base path - TemporaryFolder parentFolder = new TemporaryFolder(); - parentFolder.create(); - basePathTable1 = parentFolder.newFolder(RAW_TRIPS_TEST_NAME).getAbsolutePath(); - basePathTable2 = parentFolder.newFolder(MODEL_TRIPS_TEST_NAME).getAbsolutePath(); - basePathTable3 = parentFolder.newFolder(ETL_TRIPS_TEST_NAME).getAbsolutePath(); - basePathTable4 = parentFolder.newFolder(TRIPS_STATS_TEST_NAME).getAbsolutePath(); + basePathTable1 = parentPath.resolve(RAW_TRIPS_TEST_NAME).toAbsolutePath().toString(); + basePathTable2 = parentPath.resolve(MODEL_TRIPS_TEST_NAME).toAbsolutePath().toString(); + basePathTable3 = parentPath.resolve(ETL_TRIPS_TEST_NAME).toAbsolutePath().toString(); + basePathTable4 = parentPath.resolve(TRIPS_STATS_TEST_NAME).toAbsolutePath().toString(); dfs.mkdirs(new Path(basePathTable1)); initTableType(dfs.getConf(), basePathTable1, RAW_TRIPS_TEST_NAME, HoodieTableType.MERGE_ON_READ); diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java index 3bdaa64..15d0a6c 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestAnnotation.java @@ -18,11 +18,11 @@ package org.apache.hudi.hadoop; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.lang.annotation.Annotation; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestAnnotation { diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java index cd20d19..8e86281 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java @@ -23,13 +23,16 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.RecordReader; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class TestRecordReaderValueIterator { @Test @@ -40,11 +43,11 @@ public class TestRecordReaderValueIterator { TestRecordReader reader = new TestRecordReader(entries); RecordReaderValueIterator<IntWritable, Text> itr = new RecordReaderValueIterator<IntWritable, Text>(reader); for (int i = 0; i < values.length; i++) { - Assert.assertTrue(itr.hasNext()); + assertTrue(itr.hasNext()); Text val = itr.next(); - Assert.assertEquals(values[i], val.toString()); + assertEquals(values[i], val.toString()); } - Assert.assertFalse(itr.hasNext()); + assertFalse(itr.hasNext()); } /** diff --git a/hudi-hive-sync/pom.xml b/hudi-hive-sync/pom.xml index a47a14e..3b13915 100644 --- a/hudi-hive-sync/pom.xml +++ b/hudi-hive-sync/pom.xml @@ -142,14 +142,32 @@ </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> diff --git a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java index f394314..d92c9d9 100644 --- a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java +++ b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java @@ -30,55 +30,47 @@ import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Types; import org.joda.time.DateTime; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.List; +import java.util.stream.Stream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -@RunWith(Parameterized.class) public class TestHiveSyncTool { - // Test sync tool using both jdbc and metastore client - private boolean useJdbc; - - public TestHiveSyncTool(Boolean useJdbc) { - this.useJdbc = useJdbc; - } - - @Parameterized.Parameters(name = "UseJdbc") - public static Collection<Boolean[]> data() { - return Arrays.asList(new Boolean[][] {{false}, {true}}); + private static Stream<Boolean> useJdbc() { + return Stream.of(false, true); } - @Before + @BeforeEach public void setUp() throws IOException, InterruptedException { TestUtil.setUp(); } - @After + @AfterEach public void teardown() throws IOException { TestUtil.clear(); } - @AfterClass + @AfterAll public static void cleanUpClass() { TestUtil.shutdown(); } /** - * Testing converting array types to Hive field declaration strings. According to the Parquet-113 spec: - * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists + * Testing converting array types to Hive field declaration strings. + * <p> + * Refer to the Parquet-113 spec: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists */ @Test public void testSchemaConvertArray() throws IOException { @@ -153,44 +145,45 @@ public class TestHiveSyncTool { assertEquals("`map_list` ARRAY< MAP< string, int>>", schemaString); } - @Test - public void testBasicSync() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testBasicSync(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; TestUtil.createCOWTable(instantTime, 5); HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse("Table " + TestUtil.hiveSyncConfig.tableName + " should not exist initially", - hiveClient.doesTableExist(TestUtil.hiveSyncConfig.tableName)); + assertFalse(hiveClient.doesTableExist(TestUtil.hiveSyncConfig.tableName), + "Table " + TestUtil.hiveSyncConfig.tableName + " should not exist initially"); // Lets do the sync HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue("Table " + TestUtil.hiveSyncConfig.tableName + " should exist after sync completes", - hiveClient.doesTableExist(TestUtil.hiveSyncConfig.tableName)); - assertEquals("Hive Schema should match the table schema + partition field", - hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size(), - hiveClient.getDataSchema().getColumns().size() + 1); - assertEquals("Table partitions should match the number of partitions we wrote", 5, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); - assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", instantTime, - hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get()); + assertTrue(hiveClient.doesTableExist(TestUtil.hiveSyncConfig.tableName), + "Table " + TestUtil.hiveSyncConfig.tableName + " should exist after sync completes"); + assertEquals(hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size(), + hiveClient.getDataSchema().getColumns().size() + 1, + "Hive Schema should match the table schema + partition field"); + assertEquals(5, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "Table partitions should match the number of partitions we wrote"); + assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + "The last commit that was sycned should be updated in the TBLPROPERTIES"); // Adding of new partitions List<String> newPartition = Arrays.asList("2050/01/01"); hiveClient.addPartitionsToTable(TestUtil.hiveSyncConfig.tableName, Arrays.asList()); - assertEquals("No new partition should be added", 5, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); + assertEquals(5, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "No new partition should be added"); hiveClient.addPartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition); - assertEquals("New partition should be added", 6, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); + assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "New partition should be added"); // Update partitions hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, Arrays.asList()); - assertEquals("Partition count should remain the same", 6, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); - hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition); - assertEquals("Partition count should remain the same", 6, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); + assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "Partition count should remain the same"); + hiveClient.updatePartitionsToTable(TestUtil.hiveSyncConfig.tableName, newPartition); + assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "Partition count should remain the same"); // Alter partitions // Manually change a hive partition location to check if the sync will detect @@ -203,23 +196,23 @@ public class TestHiveSyncTool { List<String> writtenPartitionsSince = hiveClient.getPartitionsWrittenToSince(Option.empty()); writtenPartitionsSince.add(newPartition.get(0)); List<PartitionEvent> partitionEvents = hiveClient.getPartitionEvents(hivePartitions, writtenPartitionsSince); - assertEquals("There should be only one paritition event", 1, partitionEvents.size()); - assertEquals("The one partition event must of type UPDATE", PartitionEventType.UPDATE, - partitionEvents.iterator().next().eventType); + assertEquals(1, partitionEvents.size(), "There should be only one paritition event"); + assertEquals(PartitionEventType.UPDATE, partitionEvents.iterator().next().eventType, + "The one partition event must of type UPDATE"); tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); // Sync should update the changed partition to correct path List<Partition> tablePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName); - assertEquals("The one partition we wrote should be added to hive", 6, tablePartitions.size()); - assertEquals("The last commit that was sycned should be 100", instantTime, - hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get()); - + assertEquals(6, tablePartitions.size(), "The one partition we wrote should be added to hive"); + assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + "The last commit that was sycned should be 100"); } - @Test - public void testSyncIncremental() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testSyncIncremental(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String commitTime1 = "100"; TestUtil.createCOWTable(commitTime1, 5); HoodieHiveClient hiveClient = @@ -227,10 +220,10 @@ public class TestHiveSyncTool { // Lets do the sync HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertEquals("Table partitions should match the number of partitions we wrote", 5, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); - assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", commitTime1, - hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get()); + assertEquals(5, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "Table partitions should match the number of partitions we wrote"); + assertEquals(commitTime1, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + "The last commit that was sycned should be updated in the TBLPROPERTIES"); // Now lets create more parititions and these are the only ones which needs to be synced DateTime dateTime = DateTime.now().plusDays(6); @@ -240,25 +233,25 @@ public class TestHiveSyncTool { // Lets do the sync hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); List<String> writtenPartitionsSince = hiveClient.getPartitionsWrittenToSince(Option.of(commitTime1)); - assertEquals("We should have one partition written after 100 commit", 1, writtenPartitionsSince.size()); + assertEquals(1, writtenPartitionsSince.size(), "We should have one partition written after 100 commit"); List<Partition> hivePartitions = hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName); List<PartitionEvent> partitionEvents = hiveClient.getPartitionEvents(hivePartitions, writtenPartitionsSince); - assertEquals("There should be only one paritition event", 1, partitionEvents.size()); - assertEquals("The one partition event must of type ADD", PartitionEventType.ADD, - partitionEvents.iterator().next().eventType); + assertEquals(1, partitionEvents.size(), "There should be only one paritition event"); + assertEquals(PartitionEventType.ADD, partitionEvents.iterator().next().eventType, "The one partition event must of type ADD"); tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); // Sync should add the one partition - assertEquals("The one partition we wrote should be added to hive", 6, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); - assertEquals("The last commit that was sycned should be 101", commitTime2, - hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get()); + assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "The one partition we wrote should be added to hive"); + assertEquals(commitTime2, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + "The last commit that was sycned should be 101"); } - @Test - public void testSyncIncrementalWithSchemaEvolution() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testSyncIncrementalWithSchemaEvolution(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String commitTime1 = "100"; TestUtil.createCOWTable(commitTime1, 5); HoodieHiveClient hiveClient = @@ -278,42 +271,42 @@ public class TestHiveSyncTool { tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertEquals("Hive Schema has evolved and should not be 3 more field", fields + 3, - hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size()); - assertEquals("Hive Schema has evolved - Field favorite_number has evolved from int to long", "BIGINT", - hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).get("favorite_number")); - assertTrue("Hive Schema has evolved - Field favorite_movie was added", - hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).containsKey("favorite_movie")); + assertEquals(fields + 3, hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).size(), + "Hive Schema has evolved and should not be 3 more field"); + assertEquals("BIGINT", hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).get("favorite_number"), + "Hive Schema has evolved - Field favorite_number has evolved from int to long"); + assertTrue(hiveClient.getTableSchema(TestUtil.hiveSyncConfig.tableName).containsKey("favorite_movie"), + "Hive Schema has evolved - Field favorite_movie was added"); // Sync should add the one partition - assertEquals("The one partition we wrote should be added to hive", 6, - hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size()); - assertEquals("The last commit that was sycned should be 101", commitTime2, - hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get()); + assertEquals(6, hiveClient.scanTablePartitions(TestUtil.hiveSyncConfig.tableName).size(), + "The one partition we wrote should be added to hive"); + assertEquals(commitTime2, hiveClient.getLastCommitTimeSynced(TestUtil.hiveSyncConfig.tableName).get(), + "The last commit that was sycned should be 101"); } - @Test - public void testSyncMergeOnRead() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testSyncMergeOnRead(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; String deltaCommitTime = "101"; TestUtil.createMORTable(instantTime, deltaCommitTime, 5, true); String roTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE; HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse("Table " + TestUtil.hiveSyncConfig.tableName + " should not exist initially", hiveClient.doesTableExist(roTableName)); + assertFalse(hiveClient.doesTableExist(roTableName), "Table " + TestUtil.hiveSyncConfig.tableName + " should not exist initially"); // Lets do the sync HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue("Table " + roTableName + " should exist after sync completes", - hiveClient.doesTableExist(roTableName)); - assertEquals("Hive Schema should match the table schema + partition field", hiveClient.getTableSchema(roTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + 1); - assertEquals("Table partitions should match the number of partitions we wrote", 5, - hiveClient.scanTablePartitions(roTableName).size()); - assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", deltaCommitTime, - hiveClient.getLastCommitTimeSynced(roTableName).get()); + assertTrue(hiveClient.doesTableExist(roTableName), "Table " + roTableName + " should exist after sync completes"); + assertEquals(hiveClient.getTableSchema(roTableName).size(), SchemaTestUtil.getSimpleSchema().getFields().size() + 1, + "Hive Schema should match the table schema + partition field"); + assertEquals(5, hiveClient.scanTablePartitions(roTableName).size(), + "Table partitions should match the number of partitions we wrote"); + assertEquals(deltaCommitTime, hiveClient.getLastCommitTimeSynced(roTableName).get(), + "The last commit that was sycned should be updated in the TBLPROPERTIES"); // Now lets create more partitions and these are the only ones which needs to be synced DateTime dateTime = DateTime.now().plusDays(6); @@ -327,17 +320,19 @@ public class TestHiveSyncTool { tool.syncHoodieTable(); hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertEquals("Hive Schema should match the evolved table schema + partition field", - hiveClient.getTableSchema(roTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1); + assertEquals(hiveClient.getTableSchema(roTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1, + "Hive Schema should match the evolved table schema + partition field"); // Sync should add the one partition - assertEquals("The 2 partitions we wrote should be added to hive", 6, hiveClient.scanTablePartitions(roTableName).size()); - assertEquals("The last commit that was synced should be 103", deltaCommitTime2, - hiveClient.getLastCommitTimeSynced(roTableName).get()); + assertEquals(6, hiveClient.scanTablePartitions(roTableName).size(), + "The 2 partitions we wrote should be added to hive"); + assertEquals(deltaCommitTime2, hiveClient.getLastCommitTimeSynced(roTableName).get(), + "The last commit that was synced should be 103"); } - @Test - public void testSyncMergeOnReadRT() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testSyncMergeOnReadRT(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; String deltaCommitTime = "101"; String snapshotTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; @@ -345,22 +340,24 @@ public class TestHiveSyncTool { HoodieHiveClient hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse("Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE - + " should not exist initially", hiveClientRT.doesTableExist(snapshotTableName)); + assertFalse(hiveClientRT.doesTableExist(snapshotTableName), + "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + + " should not exist initially"); // Lets do the sync HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue("Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE - + " should exist after sync completes", hiveClientRT.doesTableExist(snapshotTableName)); + assertTrue(hiveClientRT.doesTableExist(snapshotTableName), + "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + + " should exist after sync completes"); - assertEquals("Hive Schema should match the table schema + partition field", hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + 1); - assertEquals("Table partitions should match the number of partitions we wrote", 5, - hiveClientRT.scanTablePartitions(snapshotTableName).size()); - assertEquals("The last commit that was synced should be updated in the TBLPROPERTIES", deltaCommitTime, - hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get()); + assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getSimpleSchema().getFields().size() + 1, + "Hive Schema should match the table schema + partition field"); + assertEquals(5, hiveClientRT.scanTablePartitions(snapshotTableName).size(), + "Table partitions should match the number of partitions we wrote"); + assertEquals(deltaCommitTime, hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get(), + "The last commit that was synced should be updated in the TBLPROPERTIES"); // Now lets create more parititions and these are the only ones which needs to be synced DateTime dateTime = DateTime.now().plusDays(6); @@ -374,17 +371,19 @@ public class TestHiveSyncTool { tool.syncHoodieTable(); hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertEquals("Hive Schema should match the evolved table schema + partition field", - hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1); + assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1, + "Hive Schema should match the evolved table schema + partition field"); // Sync should add the one partition - assertEquals("The 2 partitions we wrote should be added to hive", 6, hiveClientRT.scanTablePartitions(snapshotTableName).size()); - assertEquals("The last commit that was sycned should be 103", deltaCommitTime2, - hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get()); + assertEquals(6, hiveClientRT.scanTablePartitions(snapshotTableName).size(), + "The 2 partitions we wrote should be added to hive"); + assertEquals(deltaCommitTime2, hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get(), + "The last commit that was sycned should be 103"); } - @Test - public void testMultiPartitionKeySync() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testMultiPartitionKeySync(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String instantTime = "100"; TestUtil.createCOWTable(instantTime, 5); @@ -395,46 +394,46 @@ public class TestHiveSyncTool { TestUtil.getCreatedTablesSet().add(hiveSyncConfig.databaseName + "." + hiveSyncConfig.tableName); HoodieHiveClient hiveClient = new HoodieHiveClient(hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse("Table " + hiveSyncConfig.tableName + " should not exist initially", - hiveClient.doesTableExist(hiveSyncConfig.tableName)); + assertFalse(hiveClient.doesTableExist(hiveSyncConfig.tableName), + "Table " + hiveSyncConfig.tableName + " should not exist initially"); // Lets do the sync HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue("Table " + hiveSyncConfig.tableName + " should exist after sync completes", - hiveClient.doesTableExist(hiveSyncConfig.tableName)); - assertEquals("Hive Schema should match the table schema + partition fields", - hiveClient.getTableSchema(hiveSyncConfig.tableName).size(), - hiveClient.getDataSchema().getColumns().size() + 3); - assertEquals("Table partitions should match the number of partitions we wrote", 5, - hiveClient.scanTablePartitions(hiveSyncConfig.tableName).size()); - assertEquals("The last commit that was sycned should be updated in the TBLPROPERTIES", instantTime, - hiveClient.getLastCommitTimeSynced(hiveSyncConfig.tableName).get()); + assertTrue(hiveClient.doesTableExist(hiveSyncConfig.tableName), + "Table " + hiveSyncConfig.tableName + " should exist after sync completes"); + assertEquals(hiveClient.getTableSchema(hiveSyncConfig.tableName).size(), + hiveClient.getDataSchema().getColumns().size() + 3, + "Hive Schema should match the table schema + partition fields"); + assertEquals(5, hiveClient.scanTablePartitions(hiveSyncConfig.tableName).size(), + "Table partitions should match the number of partitions we wrote"); + assertEquals(instantTime, hiveClient.getLastCommitTimeSynced(hiveSyncConfig.tableName).get(), + "The last commit that was sycned should be updated in the TBLPROPERTIES"); } - @Test - public void testReadSchemaForMOR() throws Exception { - TestUtil.hiveSyncConfig.useJdbc = this.useJdbc; + @ParameterizedTest + @MethodSource("useJdbc") + public void testReadSchemaForMOR(boolean useJdbc) throws Exception { + TestUtil.hiveSyncConfig.useJdbc = useJdbc; String commitTime = "100"; String snapshotTableName = TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE; TestUtil.createMORTable(commitTime, "", 5, false); HoodieHiveClient hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); - assertFalse("Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE - + " should not exist initially", hiveClientRT.doesTableExist(snapshotTableName)); + assertFalse(hiveClientRT.doesTableExist(snapshotTableName), "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + + " should not exist initially"); // Lets do the sync HiveSyncTool tool = new HiveSyncTool(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); tool.syncHoodieTable(); - assertTrue("Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE - + " should exist after sync completes", hiveClientRT.doesTableExist(snapshotTableName)); + assertTrue(hiveClientRT.doesTableExist(snapshotTableName), "Table " + TestUtil.hiveSyncConfig.tableName + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE + + " should exist after sync completes"); // Schema being read from compacted base files - assertEquals("Hive Schema should match the table schema + partition field", hiveClientRT.getTableSchema(snapshotTableName).size(), - SchemaTestUtil.getSimpleSchema().getFields().size() + 1); - assertEquals("Table partitions should match the number of partitions we wrote", 5, - hiveClientRT.scanTablePartitions(snapshotTableName).size()); + assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getSimpleSchema().getFields().size() + 1, + "Hive Schema should match the table schema + partition field"); + assertEquals(5, hiveClientRT.scanTablePartitions(snapshotTableName).size(), "Table partitions should match the number of partitions we wrote"); // Now lets create more partitions and these are the only ones which needs to be synced DateTime dateTime = DateTime.now().plusDays(6); @@ -448,12 +447,12 @@ public class TestHiveSyncTool { hiveClientRT = new HoodieHiveClient(TestUtil.hiveSyncConfig, TestUtil.getHiveConf(), TestUtil.fileSystem); // Schema being read from the log files - assertEquals("Hive Schema should match the evolved table schema + partition field", - hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1); + assertEquals(hiveClientRT.getTableSchema(snapshotTableName).size(), SchemaTestUtil.getEvolvedSchema().getFields().size() + 1, + "Hive Schema should match the evolved table schema + partition field"); // Sync should add the one partition - assertEquals("The 1 partition we wrote should be added to hive", 6, hiveClientRT.scanTablePartitions(snapshotTableName).size()); - assertEquals("The last commit that was sycned should be 103", deltaCommitTime2, - hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get()); + assertEquals(6, hiveClientRT.scanTablePartitions(snapshotTableName).size(), "The 1 partition we wrote should be added to hive"); + assertEquals(deltaCommitTime2, hiveClientRT.getLastCommitTimeSynced(snapshotTableName).get(), + "The last commit that was sycned should be 103"); } } diff --git a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestUtil.java b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestUtil.java index cee1330..90c3995 100644 --- a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestUtil.java +++ b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestUtil.java @@ -75,7 +75,7 @@ import java.util.Map.Entry; import java.util.Set; import java.util.UUID; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; @SuppressWarnings("SameParameterValue") public class TestUtil { diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml index 7fa1ae2..53b1233 100644 --- a/hudi-integ-test/pom.xml +++ b/hudi-integ-test/pom.xml @@ -115,11 +115,29 @@ </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>${junit.version}</version> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + </dependencies> <properties> diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index ee3b51e..cf87273 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -33,8 +33,7 @@ import com.github.dockerjava.core.command.ExecStartResultCallback; import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import org.junit.Assert; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -46,6 +45,8 @@ import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; public abstract class ITTestBase { @@ -113,12 +114,12 @@ public abstract class ITTestBase { static String getPrestoConsoleCommand(String commandFile) { StringBuilder builder = new StringBuilder().append("presto --server " + PRESTO_COORDINATOR_URL) .append(" --catalog hive --schema default") - .append(" -f " + commandFile ); + .append(" -f " + commandFile); System.out.println("Presto comamnd " + builder.toString()); return builder.toString(); } - @Before + @BeforeEach public void init() { String dockerHost = (OVERRIDDEN_DOCKER_HOST != null) ? OVERRIDDEN_DOCKER_HOST : DEFAULT_DOCKER_HOST; // Assuming insecure docker engine @@ -165,10 +166,9 @@ public abstract class ITTestBase { LOG.error("\n\n ###### Stderr #######\n" + callback.getStderr().toString()); if (expectedToSucceed) { - Assert.assertEquals("Command (" + Arrays.toString(command) + ") expected to succeed. Exit (" + exitCode + ")", 0, exitCode); + assertEquals(0, exitCode, "Command (" + Arrays.toString(command) + ") expected to succeed. Exit (" + exitCode + ")"); } else { - Assert.assertTrue("Command (" + Arrays.toString(command) + ") expected to fail. Exit (" + exitCode + ")", - exitCode != 0); + assertNotEquals(0, exitCode, "Command (" + Arrays.toString(command) + ") expected to fail. Exit (" + exitCode + ")"); } cmd.close(); return callback; @@ -224,7 +224,7 @@ public abstract class ITTestBase { return Pair.of(callback.getStdout().toString().trim(), callback.getStderr().toString().trim()); } - void executePrestoCopyCommand(String fromFile, String remotePath){ + void executePrestoCopyCommand(String fromFile, String remotePath) { Container sparkWorkerContainer = runningContainers.get(PRESTO_COORDINATOR); dockerClient.copyArchiveToContainerCmd(sparkWorkerContainer.getId()) .withHostResource(fromFile) @@ -268,7 +268,7 @@ public abstract class ITTestBase { saveUpLogs(); } - Assert.assertEquals("Did not find output the expected number of times", times, count); + assertEquals(times, count, "Did not find output the expected number of times"); } public class TestExecStartResultCallback extends ExecStartResultCallback { diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java index bb08323..ad18eae 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java @@ -21,7 +21,7 @@ package org.apache.hudi.integ; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.collection.Pair; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.List; @@ -102,9 +102,9 @@ public class ITTestHoodieDemo extends ITTestBase { private void setupDemo() throws Exception { List<String> cmds = CollectionUtils.createImmutableList("hdfs dfsadmin -safemode wait", - "hdfs dfs -mkdir -p " + HDFS_DATA_DIR, - "hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH1 + " " + HDFS_BATCH_PATH1, - "/bin/bash " + DEMO_CONTAINER_SCRIPT); + "hdfs dfs -mkdir -p " + HDFS_DATA_DIR, + "hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH1 + " " + HDFS_BATCH_PATH1, + "/bin/bash " + DEMO_CONTAINER_SCRIPT); executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds); @@ -113,21 +113,21 @@ public class ITTestHoodieDemo extends ITTestBase { executeCommandStringsInDocker(PRESTO_COORDINATOR, cmds); // copy presto sql files to presto coordinator - executePrestoCopyCommand( System.getProperty("user.dir") + "/.." + PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR); - executePrestoCopyCommand( System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR); - executePrestoCopyCommand( System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR); + executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_TABLE_CHECK_RELATIVE_PATH, HDFS_DATA_DIR); + executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH1_RELATIVE_PATH, HDFS_DATA_DIR); + executePrestoCopyCommand(System.getProperty("user.dir") + "/.." + PRESTO_INPUT_BATCH2_RELATIVE_PATH, HDFS_DATA_DIR); } private void ingestFirstBatchAndHiveSync() throws Exception { List<String> cmds = CollectionUtils.createImmutableList( - "spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + "spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + " --table-type COPY_ON_WRITE " + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts " + " --target-base-path " + COW_BASE_PATH + " --target-table " + COW_TABLE_NAME + " --props /var/demo/config/dfs-source.properties" + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider " + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME), - ("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + ("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + " --table-type MERGE_ON_READ " + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts " + " --target-base-path " + MOR_BASE_PATH + " --target-table " + MOR_TABLE_NAME @@ -286,7 +286,7 @@ public class ITTestHoodieDemo extends ITTestBase { testIncrementalHiveQuery(MIN_COMMIT_TIME_COW_SCRIPT, HIVE_INCREMENTAL_COW_COMMANDS, expectedOutput, 1); // verify that 10:59 is NOT present in RO table because of pending compaction - testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 0); + testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 0); // verify that 10:59 is present in RT table even with pending compaction testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RT_COMMANDS, expectedOutput, 1); diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java index 4b6d89f..1d1ef8d 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java @@ -18,12 +18,15 @@ package org.apache.hudi.integ; +import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; -import org.apache.hudi.common.model.HoodieTableType; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + /** * Smoke tests to run as part of verification. @@ -129,7 +132,7 @@ public class ITTestHoodieSanity extends ITTestBase { // Ensure table does not exist Pair<String, String> stdOutErr = executeHiveCommand("show tables like '" + hiveTableName + "'"); - Assert.assertTrue("Dropped table " + hiveTableName + " exists!", stdOutErr.getLeft().isEmpty()); + assertTrue(stdOutErr.getLeft().isEmpty(), "Dropped table " + hiveTableName + " exists!"); // Run Hoodie Java App String cmd; @@ -145,24 +148,24 @@ public class ITTestHoodieSanity extends ITTestBase { } executeCommandStringInDocker(ADHOC_1_CONTAINER, cmd, true); - String snapshotTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name()) ? - hiveTableName + "_rt" : hiveTableName; - Option<String> roTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name()) ? - Option.of(hiveTableName +"_ro") : Option.empty(); + String snapshotTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name()) + ? hiveTableName + "_rt" : hiveTableName; + Option<String> roTableName = tableType.equals(HoodieTableType.MERGE_ON_READ.name()) + ? Option.of(hiveTableName + "_ro") : Option.empty(); // Ensure table does exist stdOutErr = executeHiveCommand("show tables like '" + snapshotTableName + "'"); - Assert.assertEquals("Table exists", snapshotTableName, stdOutErr.getLeft()); + assertEquals(snapshotTableName, stdOutErr.getLeft(), "Table exists"); // Ensure row count is 80 (without duplicates) (100 - 20 deleted) stdOutErr = executeHiveCommand("select count(1) from " + snapshotTableName); - Assert.assertEquals("Expecting 80 rows to be present in the snapshot table", 80, - Integer.parseInt(stdOutErr.getLeft().trim())); + assertEquals(80, Integer.parseInt(stdOutErr.getLeft().trim()), + "Expecting 80 rows to be present in the snapshot table"); if (roTableName.isPresent()) { stdOutErr = executeHiveCommand("select count(1) from " + roTableName.get()); - Assert.assertEquals("Expecting 80 rows to be present in the snapshot table", 80, - Integer.parseInt(stdOutErr.getLeft().trim())); + assertEquals(80, Integer.parseInt(stdOutErr.getLeft().trim()), + "Expecting 80 rows to be present in the snapshot table"); } // Make the HDFS dataset non-hoodie and run the same query; Checks for interoperability with non-hoodie tables @@ -175,8 +178,8 @@ public class ITTestHoodieSanity extends ITTestBase { } else { stdOutErr = executeHiveCommand("select count(1) from " + snapshotTableName); } - Assert.assertEquals("Expecting 280 rows to be present in the new table", 280, - Integer.parseInt(stdOutErr.getLeft().trim())); + assertEquals(280, Integer.parseInt(stdOutErr.getLeft().trim()), + "Expecting 280 rows to be present in the new table"); } private void dropHiveTables(String hiveTableName, String tableType) throws Exception { diff --git a/hudi-spark/pom.xml b/hudi-spark/pom.xml index b3231af..49e883c 100644 --- a/hudi-spark/pom.xml +++ b/hudi-spark/pom.xml @@ -315,9 +315,34 @@ </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + </dependencies> </project> diff --git a/hudi-spark/src/test/java/DataSourceUtilsTest.java b/hudi-spark/src/test/java/DataSourceUtilsTest.java index bcb652c..4bacb7c 100644 --- a/hudi-spark/src/test/java/DataSourceUtilsTest.java +++ b/hudi-spark/src/test/java/DataSourceUtilsTest.java @@ -21,11 +21,11 @@ import org.apache.hudi.DataSourceUtils; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.time.LocalDate; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class DataSourceUtilsTest { @@ -34,7 +34,7 @@ public class DataSourceUtilsTest { // There are fields event_date1, event_date2, event_date3 with logical type as Date. event_date1 & event_date3 are // of UNION schema type, which is a union of null and date type in different orders. event_date2 is non-union // date type - String avroSchemaString = "{\"type\": \"record\"," + "\"name\": \"events\"," + "\"fields\": [ " + String avroSchemaString = "{\"type\": \"record\"," + "\"name\": \"events\"," + "\"fields\": [ " + "{\"name\": \"event_date1\", \"type\" : [{\"type\" : \"int\", \"logicalType\" : \"date\"}, \"null\"]}," + "{\"name\": \"event_date2\", \"type\" : {\"type\": \"int\", \"logicalType\" : \"date\"}}," + "{\"name\": \"event_date3\", \"type\" : [\"null\", {\"type\" : \"int\", \"logicalType\" : \"date\"}]}," diff --git a/hudi-spark/src/test/scala/TestDataSource.scala b/hudi-spark/src/test/scala/TestDataSource.scala index fd0677d..900d61c 100644 --- a/hudi-spark/src/test/scala/TestDataSource.scala +++ b/hudi-spark/src/test/scala/TestDataSource.scala @@ -21,12 +21,11 @@ import org.apache.hudi.common.fs.FSUtils import org.apache.hudi.config.HoodieWriteConfig import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.sql._ -import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime} import org.apache.spark.sql.functions.col -import org.junit.Assert._ -import org.junit.rules.TemporaryFolder -import org.junit.{Before, Test} -import org.scalatest.junit.AssertionsForJUnit +import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime} +import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.api.{BeforeEach, Test} import scala.collection.JavaConversions._ import scala.concurrent.ExecutionContext.Implicits.global @@ -34,9 +33,9 @@ import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} /** - * Basic tests on the spark datasource - */ -class TestDataSource extends AssertionsForJUnit { + * Basic tests on the spark datasource + */ +class TestDataSource { var spark: SparkSession = null var dataGen: HoodieTestDataGenerator = null @@ -51,16 +50,14 @@ class TestDataSource extends AssertionsForJUnit { var basePath: String = null var fs: FileSystem = null - @Before def initialize() { + @BeforeEach def initialize(@TempDir tempDir: java.nio.file.Path) { spark = SparkSession.builder .appName("Hoodie Datasource test") .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .getOrCreate dataGen = new HoodieTestDataGenerator() - val folder = new TemporaryFolder - folder.create - basePath = folder.getRoot.getAbsolutePath + basePath = tempDir.toAbsolutePath.toString fs = FSUtils.getFs(basePath, spark.sparkContext.hadoopConfiguration) } diff --git a/hudi-spark/src/test/scala/TestDataSourceDefaults.scala b/hudi-spark/src/test/scala/TestDataSourceDefaults.scala index 7a8008f..0e5c9f0 100644 --- a/hudi-spark/src/test/scala/TestDataSourceDefaults.scala +++ b/hudi-spark/src/test/scala/TestDataSourceDefaults.scala @@ -16,25 +16,25 @@ */ import org.apache.avro.generic.GenericRecord +import org.apache.hudi.DataSourceWriteOptions +import org.apache.hudi.common.config.TypedProperties import org.apache.hudi.common.model.{EmptyHoodieRecordPayload, OverwriteWithLatestAvroPayload} import org.apache.hudi.common.util.{Option, SchemaTestUtil} import org.apache.hudi.exception.{HoodieException, HoodieKeyException} import org.apache.hudi.keygen.{ComplexKeyGenerator, GlobalDeleteKeyGenerator, SimpleKeyGenerator} -import org.apache.hudi.DataSourceWriteOptions -import org.apache.hudi.common.config.TypedProperties -import org.junit.Assert._ -import org.junit.{Before, Test} -import org.scalatest.junit.AssertionsForJUnit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.{BeforeEach, Test} +import org.scalatest.Assertions.fail /** - * Tests on the default key generator, payload classes. - */ -class TestDataSourceDefaults extends AssertionsForJUnit { + * Tests on the default key generator, payload classes. + */ +class TestDataSourceDefaults { val schema = SchemaTestUtil.getComplexEvolvedSchema var baseRecord: GenericRecord = _ - @Before def initialize(): Unit = { + @BeforeEach def initialize(): Unit = { baseRecord = SchemaTestUtil .generateAvroRecordFromJson(schema, 1, "001", "f1") } diff --git a/hudi-timeline-service/pom.xml b/hudi-timeline-service/pom.xml index 8061f78..81323dd 100644 --- a/hudi-timeline-service/pom.xml +++ b/hudi-timeline-service/pom.xml @@ -184,8 +184,26 @@ </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> <scope>test</scope> </dependency> diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml index 2324ae8..827a056 100644 --- a/hudi-utilities/pom.xml +++ b/hudi-utilities/pom.xml @@ -364,6 +364,30 @@ </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestChainedTransformer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestChainedTransformer.java index dd5b8b9..1a7aa0e 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestChainedTransformer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/transform/TestChainedTransformer.java @@ -29,9 +29,9 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; @@ -39,21 +39,21 @@ import java.util.List; import static org.apache.spark.sql.types.DataTypes.IntegerType; import static org.apache.spark.sql.types.DataTypes.StringType; import static org.apache.spark.sql.types.DataTypes.createStructField; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; public class TestChainedTransformer { private JavaSparkContext jsc; private SparkSession sparkSession; - @Before + @BeforeEach public void setUp() { jsc = UtilHelpers.buildSparkContext(this.getClass().getName() + "-hoodie", "local[2]"); sparkSession = SparkSession.builder().config(jsc.getConf()).getOrCreate(); } - @After + @AfterEach public void tearDown() { jsc.stop(); } diff --git a/pom.xml b/pom.xml index 7d711ad..e725b74 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ <properties> <maven-jar-plugin.version>2.6</maven-jar-plugin.version> - <maven-surefire-plugin.version>2.19.1</maven-surefire-plugin.version> + <maven-surefire-plugin.version>3.0.0-M4</maven-surefire-plugin.version> <maven-shade-plugin.version>3.1.1</maven-shade-plugin.version> <maven-javadoc-plugin.version>3.1.1</maven-javadoc-plugin.version> <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> @@ -82,7 +82,9 @@ <kafka.version>2.0.0</kafka.version> <glassfish.version>2.17</glassfish.version> <parquet.version>1.10.1</parquet.version> - <junit.version>4.11</junit.version> + <junit.version>4.12</junit.version> + <junit.jupiter.version>5.6.1</junit.jupiter.version> + <junit.vintage.version>5.6.1</junit.vintage.version> <mockito.version>1.10.19</mockito.version> <log4j.version>1.2.17</log4j.version> <slf4j.version>1.7.5</slf4j.version> @@ -821,6 +823,34 @@ </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>${junit.jupiter.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.jupiter.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.vintage.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-params</artifactId> + <version>${junit.jupiter.version}</version> + <scope>test</scope> + </dependency> + + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope>