xushiyan commented on code in PR #5854:
URL: https://github.com/apache/hudi/pull/5854#discussion_r912320024
##########
hudi-sync/hudi-adb-sync/src/main/java/org/apache/hudi/sync/adb/HoodieAdbJdbcClient.java:
##########
@@ -405,26 +423,69 @@ private String constructShowCreateDatabaseSql(String
databaseName) {
private String constructUpdateTblPropertiesSql(String tableName, String
lastCommitSynced) {
return String.format("alter table `%s`.`%s` set tblproperties('%s' =
'%s')",
- adbSyncConfig.databaseName, tableName, HOODIE_LAST_COMMIT_TIME_SYNC,
lastCommitSynced);
+ databaseName, tableName, HOODIE_LAST_COMMIT_TIME_SYNC,
lastCommitSynced);
}
private String constructAddColumnSql(String tableName, String columnName,
String columnType) {
return String.format("alter table `%s`.`%s` add columns(`%s` %s)",
- adbSyncConfig.databaseName, tableName, columnName, columnType);
+ databaseName, tableName, columnName, columnType);
}
private String constructChangeColumnSql(String tableName, String columnName,
String columnType) {
return String.format("alter table `%s`.`%s` change `%s` `%s` %s",
- adbSyncConfig.databaseName, tableName, columnName, columnName,
columnType);
+ databaseName, tableName, columnName, columnName, columnType);
+ }
+
+ /**
+ * TODO align with {@link HoodieSyncClient#getPartitionEvents}
+ */
+ public List<PartitionEvent> getPartitionEvents(Map<List<String>, String>
tablePartitions, List<String> partitionStoragePartitions) {
+ Map<String, String> paths = new HashMap<>();
+
+ for (Map.Entry<List<String>, String> entry : tablePartitions.entrySet()) {
+ List<String> partitionValues = entry.getKey();
+ String fullTablePartitionPath = entry.getValue();
+ paths.put(String.join(", ", partitionValues), fullTablePartitionPath);
+ }
+ List<PartitionEvent> events = new ArrayList<>();
+ for (String storagePartition : partitionStoragePartitions) {
+ Path storagePartitionPath =
FSUtils.getPartitionPath(config.getString(META_SYNC_BASE_PATH),
storagePartition);
+ String fullStoragePartitionPath =
Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
+ // Check if the partition values or if hdfs path is the same
+ List<String> storagePartitionValues =
partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
+ if (config.getBoolean(ADB_SYNC_USE_HIVE_STYLE_PARTITIONING)) {
+ String partition = String.join("/", storagePartitionValues);
+ storagePartitionPath =
FSUtils.getPartitionPath(config.getString(META_SYNC_BASE_PATH), partition);
+ fullStoragePartitionPath =
Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
+ }
+ if (!storagePartitionValues.isEmpty()) {
+ String storageValue = String.join(", ", storagePartitionValues);
+ if (!paths.containsKey(storageValue)) {
+ events.add(PartitionEvent.newPartitionAddEvent(storagePartition));
+ } else if (!paths.get(storageValue).equals(fullStoragePartitionPath)) {
+ events.add(PartitionEvent.newPartitionUpdateEvent(storagePartition));
+ }
+ }
+ }
+ return events;
}
- private HiveSyncConfig getHiveSyncConfig() {
- HiveSyncConfig hiveSyncConfig = new HiveSyncConfig();
- hiveSyncConfig.partitionFields = adbSyncConfig.partitionFields;
- hiveSyncConfig.databaseName = adbSyncConfig.databaseName;
- Path basePath = new Path(adbSyncConfig.basePath);
- hiveSyncConfig.basePath = generateAbsolutePathStr(basePath);
- return hiveSyncConfig;
+ public void closeQuietly(ResultSet resultSet, Statement stmt) {
Review Comment:
this comes from existing APIs. just to keep as is. also looks like it's
closing quietly for sql stmt error but keep connection open for jdbc. the
close() API is used for closing connection.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
