vinothchandar commented on a change in pull request #2768:
URL: https://github.com/apache/hudi/pull/2768#discussion_r773575945
##########
File path:
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java
##########
@@ -41,4 +67,84 @@ public void testInitHiveIncrementalPuller() {
}
+ private HiveIncrementalPuller.Config getHivePullerConfig(String
incrementalSql) throws IOException {
+ config.hiveJDBCUrl = hiveSyncConfig.jdbcUrl;
+ config.hiveUsername = hiveSyncConfig.hiveUser;
+ config.hivePassword = hiveSyncConfig.hivePass;
+ config.hoodieTmpDir =
Files.createTempDirectory("hivePullerTest").toUri().toString();
+ config.sourceDb = hiveSyncConfig.databaseName;
+ config.sourceTable = hiveSyncConfig.tableName;
+ config.targetDb = "tgtdb";
+ config.targetTable = "test2";
+ config.tmpDb = "tmp_db";
+ config.fromCommitTime = "100";
+ createIncrementalSqlFile(incrementalSql, config);
+ return config;
+ }
+
+ private void createIncrementalSqlFile(String text,
HiveIncrementalPuller.Config cfg) throws IOException {
+ java.nio.file.Path path = Paths.get(cfg.hoodieTmpDir +
"/incremental_pull.txt");
+ Files.createDirectories(path.getParent());
+ Files.createFile(path);
+ try (FileWriter fr = new FileWriter(new File(path.toUri()))) {
+ fr.write(text);
+ } catch (Exception e) {
+ // no-op
+ }
+ cfg.incrementalSQLFile = path.toString();
+ }
+
+ private void createSourceTable() throws IOException, URISyntaxException {
+ String instantTime = "101";
+ HiveTestUtil.createCOWTable(instantTime, 5, true);
+ hiveSyncConfig.syncMode = "jdbc";
+ HiveTestUtil.hiveSyncConfig.batchSyncNum = 3;
+ HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig,
HiveTestUtil.getHiveConf(), fileSystem);
+ tool.syncHoodieTable();
+ }
+
+ private void createTargetTable() throws IOException, URISyntaxException {
+ String instantTime = "100";
+ targetBasePath = Files.createTempDirectory("hivesynctest1" +
Instant.now().toEpochMilli()).toUri().toString();
+ HiveTestUtil.createCOWTable(instantTime, 5, true,
+ targetBasePath, "tgtdb", "test2");
+ HiveSyncTool tool = new
HiveSyncTool(getTargetHiveSyncConfig(targetBasePath),
HiveTestUtil.getHiveConf(), fileSystem);
+ tool.syncHoodieTable();
+ }
+
+ private HiveSyncConfig getTargetHiveSyncConfig(String basePath) {
+ HiveSyncConfig config = HiveSyncConfig.copy(hiveSyncConfig);
+ config.databaseName = "tgtdb";
+ config.tableName = "test2";
+ config.basePath = basePath;
+ config.batchSyncNum = 3;
+ config.syncMode = "jdbc";
+ return config;
+ }
+
+ private void createTables() throws IOException, URISyntaxException {
+ createSourceTable();
+ createTargetTable();
+ }
+
+ @Test
+ public void testPullerWithoutIncrementalClause() throws IOException,
URISyntaxException {
Review comment:
these tests only seem to be testing some failure scenarios? not the
happy path?
##########
File path:
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java
##########
@@ -73,7 +73,7 @@
}
@Test
- public void testInvalidHiveSyncProps() throws IOException {
+ public void testInvalidHiveSyncProps() {
Review comment:
can we avoid unrelated code changes in PRs like this one
##########
File path:
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java
##########
@@ -126,6 +126,16 @@ public static void setUp() throws IOException,
InterruptedException, HiveExcepti
clear();
}
+ public static void clearIncrementalPullSetup(String path1, String path2)
throws IOException, HiveException, MetaException {
Review comment:
are these paths that pass in `null`? wondering if we can directly call
`fs.delete()` here. and avoid the exists check as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]