vaibhavk1992 commented on code in PR #659:
URL: https://github.com/apache/incubator-xtable/pull/659#discussion_r1989941203
##########
xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java:
##########
@@ -142,63 +221,58 @@ public static void main(String[] args) throws IOException
{
ConversionSourceProvider<?> conversionSourceProvider =
ReflectionUtils.createInstanceOfClass(sourceProviderClass);
conversionSourceProvider.init(hadoopConf);
+ return conversionSourceProvider;
+ }
+ public static List<String> getTableFormatList(DatasetConfig datasetConfig)
throws IOException {
+ // Retrieve table format list
List<String> tableFormatList = datasetConfig.getTargetFormats();
- ConversionController conversionController = new
ConversionController(hadoopConf);
- for (DatasetConfig.Table table : datasetConfig.getDatasets()) {
- log.info(
- "Running sync for basePath {} for following table formats {}",
- table.getTableBasePath(),
- tableFormatList);
- Properties sourceProperties = new Properties();
- if (table.getPartitionSpec() != null) {
- sourceProperties.put(
- HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG,
table.getPartitionSpec());
- }
- SourceTable sourceTable =
- SourceTable.builder()
- .name(table.getTableName())
- .basePath(table.getTableBasePath())
- .namespace(table.getNamespace() == null ? null :
table.getNamespace().split("\\."))
- .dataPath(table.getTableDataPath())
- .catalogConfig(icebergCatalogConfig)
- .additionalProperties(sourceProperties)
- .formatName(sourceFormat)
- .build();
- List<TargetTable> targetTables =
- tableFormatList.stream()
- .map(
- tableFormat ->
- TargetTable.builder()
- .name(table.getTableName())
- .basePath(table.getTableBasePath())
- .namespace(
- table.getNamespace() == null
- ? null
- : table.getNamespace().split("\\."))
- .catalogConfig(icebergCatalogConfig)
- .formatName(tableFormat)
- .build())
- .collect(Collectors.toList());
+ return tableFormatList;
+ }
- ConversionConfig conversionConfig =
- ConversionConfig.builder()
- .sourceTable(sourceTable)
- .targetTables(targetTables)
- .syncMode(SyncMode.INCREMENTAL)
- .build();
- try {
- conversionController.sync(conversionConfig, conversionSourceProvider);
- } catch (Exception e) {
- log.error("Error running sync for {}", table.getTableBasePath(), e);
- }
+ public static CommandLine CommandParser(String[] args) {
+ CommandLineParser parser = new DefaultParser();
+
+ CommandLine cmd;
+ try {
+ cmd = parser.parse(OPTIONS, args);
+ } catch (ParseException e) {
+ new HelpFormatter().printHelp("xtable.jar", OPTIONS, true);
+ return null;
+ }
+
+ if (cmd.hasOption(HELP_OPTION)) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("RunSync", OPTIONS);
+ return null;
}
+ return cmd;
+ }
+
+ public static String getValueFromConfig(CommandLine cmd, String configFlag) {
+ return cmd.getOptionValue(configFlag);
+ }
+
+ public static void main(String[] args) throws IOException {
+ CommandLine cmd = CommandParser(args);
+ String datasetConfigpath = getValueFromConfig(cmd, DATASET_CONFIG_OPTION);
+ String icebergCatalogConfigpath = getValueFromConfig(cmd,
ICEBERG_CATALOG_CONFIG_PATH);
+ String hadoopConfigpath = getValueFromConfig(cmd, HADOOP_CONFIG_PATH);
+ String conversionProviderConfigpath = getValueFromConfig(cmd,
CONVERTERS_CONFIG_PATH);
+ DatasetConfig datasetConfig = getDatasetConfig(datasetConfigpath);
+ IcebergCatalogConfig icebergCatalogConfig =
getIcebergCatalogConfig(icebergCatalogConfigpath);
+ Configuration hadoopConf = gethadoopConf(hadoopConfigpath);
+ ConversionSourceProvider conversionSourceProvider =
+ getConversionSourceProvider(conversionProviderConfigpath,
datasetConfig, hadoopConf);
+ List<String> tableFormatList = getTableFormatList(datasetConfig);
+ formatConvertor(
+ datasetConfig, tableFormatList, icebergCatalogConfig, hadoopConf,
conversionSourceProvider);
}
- static byte[] getCustomConfigurations(CommandLine cmd, String option) throws
IOException {
+ static byte[] getCustomConfigurations(String cmd) throws IOException {
Review Comment:
Changes have been done.
##########
xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java:
##########
@@ -142,63 +221,58 @@ public static void main(String[] args) throws IOException
{
ConversionSourceProvider<?> conversionSourceProvider =
ReflectionUtils.createInstanceOfClass(sourceProviderClass);
conversionSourceProvider.init(hadoopConf);
+ return conversionSourceProvider;
+ }
+ public static List<String> getTableFormatList(DatasetConfig datasetConfig)
throws IOException {
+ // Retrieve table format list
List<String> tableFormatList = datasetConfig.getTargetFormats();
Review Comment:
Changes have been done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
