This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20766-8b38911c0edaf02d23dbe301b2b2fbd7e1003a84 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit d97f98013b5adc56a3b23cf985e194b7a71b7319 Author: voonhous <[email protected]> AuthorDate: Fri Nov 20 02:36:50 2026 +0000 refactor(client): use Lombok @Slf4j for loggers (#20766) * refactor(client): use Lombok @Slf4j for loggers Replace hand-declared static SLF4J loggers in hudi-client (hudi-client-common, hudi-spark-client, hudi-flink-client, hudi-java-client) with Lombok @Slf4j: add the annotation, drop the field and the org.slf4j imports, and rename LOG/LOGGER to log. One remaining string-concatenated warn in OneToTwoUpgradeHandler now uses a {} placeholder. Topic preserved (logger category is a supertype): - FlinkSizeBasedClusteringPlanStrategyRecently keeps FlinkSizeBasedClusteringPlanStrategy. Category changes (logger was copied from an unrelated class, now logs under its own name): - RegexRuleEngine (was PartitionBucketIndexCalculator) - SparkHoodieBackedTableMetadataWriterTableVersionSix (was SparkHoodieBackedTableMetadataWriter) - BaseBootstrapMetadataHandler (was ParquetBootstrapMetadataHandler) - SparkPartitionTTLActionExecutor (was ConsistentBucketBulkInsertDataInternalWriterHelper) - TestHoodieSparkEngineDynamicRepartition (was HoodieBackedTestDelayedTableMetadata) Subclasses that used an inherited non-private LOG get their own @Slf4j, so those lines now log under the subclass name: - DirectMarkerTransactionManager (was TransactionManager) - HoodieBackedTableMetadataWriterTableVersionSix (was HoodieBackedTableMetadataWriter) - hudi-spark-datasource/hudi-spark tests TestHoodieClientMultiWriter, TestClusteringTaskBinaryPayloads and TestDataValidationCheckForLogCompactionActions (was HoodieClientTestBase); cross-module touch. Skipped: - RunsTableService: interface field, @Slf4j is not legal there. - BucketIndexBulkInsertPartitioner, RDDBucketIndexPartitioner: log4j LogManager loggers, not SLF4J. - HoodieSparkKryoRegistrar: Scala. * refactor(client): move bucket-index partitioners to @Slf4j BucketIndexBulkInsertPartitioner and RDDBucketIndexPartitioner were the only main-source classes logging through the log4j2 API directly (LogManager.getLogger) instead of SLF4J. Switch both to Lombok @Slf4j; the logger category is unchanged. --- .../bootstrap/FullRecordBootstrapDataProvider.java | 6 +- .../DirectMarkerTransactionManager.java | 11 ++- .../client/transaction/TransactionManager.java | 15 ++-- .../client/validator/PreWriteValidatorUtils.java | 24 +++--- .../FileMetadataWriteStatusConverter.java | 7 +- .../index/bucket/partition/RegexRuleEngine.java | 11 ++- .../hudi/io/ExternalFileClusteringWriteHandle.java | 9 +-- .../java/org/apache/hudi/keygen/KeyGenUtils.java | 15 ++-- .../metadata/HoodieBackedTableMetadataWriter.java | 92 +++++++++++----------- ...ieBackedTableMetadataWriterTableVersionSix.java | 27 ++++--- .../table/BucketIndexBulkInsertPartitioner.java | 6 +- .../CommitBasedClusteringPlanStrategy.java | 15 ++-- .../hudi/table/upgrade/OneToTwoUpgradeHandler.java | 13 ++- .../table/upgrade/TenToNineDowngradeHandler.java | 8 +- ...inkSizeBasedClusteringPlanStrategyRecently.java | 13 ++- .../java/org/apache/hudi/util/FlinkClientUtil.java | 9 +-- .../testutils/HoodieFlinkClientTestHarness.java | 9 +-- .../common/TestMultipleHoodieJavaWriteClient.java | 15 ++-- ...arkExternalFileClusteringExecutionStrategy.java | 10 +-- .../hudi/client/utils/SparkValidatorUtils.java | 26 +++--- .../bulkinsert/RDDBucketIndexPartitioner.java | 8 +- .../storage/row/HoodieRowParquetWriteSupport.java | 7 +- ...ieBackedTableMetadataWriterTableVersionSix.java | 8 +- .../apache/hudi/metrics/DistributedRegistry.java | 10 +-- .../hudi/metrics/RecordIndexLookupMetrics.java | 10 +-- .../bootstrap/BaseBootstrapMetadataHandler.java | 7 +- .../commit/SparkPartitionTTLActionExecutor.java | 10 +-- .../ClusteringIdentityTestExecutionStrategy.java | 6 +- .../TestHoodieSparkEngineDynamicRepartition.java | 15 ++-- .../hudi/testutils/HoodieClientTestBase.java | 6 +- .../hudi/client/TestHoodieClientMultiWriter.java | 14 ++-- .../strategy/TestClusteringTaskBinaryPayloads.java | 4 +- ...DataValidationCheckForLogCompactionActions.java | 10 ++- 33 files changed, 211 insertions(+), 245 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/FullRecordBootstrapDataProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/FullRecordBootstrapDataProvider.java index c658e13a265e..68264f0a8236 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/FullRecordBootstrapDataProvider.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/FullRecordBootstrapDataProvider.java @@ -24,8 +24,7 @@ import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; import java.util.List; @@ -33,10 +32,9 @@ import java.util.List; /** * Creates Hoodie Records with complete record data, given a list of partitions to be bootstrapped. */ +@Slf4j public abstract class FullRecordBootstrapDataProvider<I> implements Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(FullRecordBootstrapDataProvider.class); - protected final TypedProperties props; protected final transient HoodieEngineContext context; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java index 90c5e963da1e..1aeaf7cb48d8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/DirectMarkerTransactionManager.java @@ -30,6 +30,8 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.storage.HoodieStorage; +import lombok.extern.slf4j.Slf4j; + import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING; /** @@ -38,6 +40,7 @@ import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING; * detection is enabled. Anything done between a start and end transaction is guaranteed to be * atomic. */ +@Slf4j public class DirectMarkerTransactionManager extends TransactionManager { private final String filePath; @@ -48,20 +51,20 @@ public class DirectMarkerTransactionManager extends TransactionManager { public void beginTransaction(String newTxnOwnerInstantTime, InstantGenerator instantGenerator) { if (isLockRequired) { - LOG.info("Transaction starting for {} and {}", newTxnOwnerInstantTime, filePath); + log.info("Transaction starting for {} and {}", newTxnOwnerInstantTime, filePath); lockManager.lock(); reset(changeActionInstant, Option.of(getInstant(newTxnOwnerInstantTime, instantGenerator)), Option.empty()); - LOG.info("Transaction started for {} and {}", newTxnOwnerInstantTime, filePath); + log.info("Transaction started for {} and {}", newTxnOwnerInstantTime, filePath); } } public void endTransaction(String currentTxnOwnerInstantTime, InstantGenerator instantGenerator) { if (isLockRequired) { - LOG.info("Transaction ending with transaction owner {} for {}", currentTxnOwnerInstantTime, filePath); + log.info("Transaction ending with transaction owner {} for {}", currentTxnOwnerInstantTime, filePath); if (reset(Option.of(getInstant(currentTxnOwnerInstantTime, instantGenerator)), Option.empty(), Option.empty())) { lockManager.unlock(); - LOG.info("Transaction ended with transaction owner {} for {}", currentTxnOwnerInstantTime, filePath); + log.info("Transaction ended with transaction owner {} for {}", currentTxnOwnerInstantTime, filePath); } } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java index 7b1d4b4fe6f9..32d6abf08216 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/TransactionManager.java @@ -25,8 +25,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.storage.HoodieStorage; import lombok.Getter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; @@ -34,9 +33,9 @@ import java.io.Serializable; * This class allows clients to start and end transactions. Anything done between a start and end transaction is * guaranteed to be atomic. */ +@Slf4j public class TransactionManager implements Serializable, AutoCloseable { - protected static final Logger LOG = LoggerFactory.getLogger(TransactionManager.class); @Getter protected final LockManager lockManager; @Getter @@ -56,21 +55,21 @@ public class TransactionManager implements Serializable, AutoCloseable { public void beginStateChange(Option<HoodieInstant> changeActionInstant, Option<HoodieInstant> lastCompletedActionInstant) { if (isLockRequired) { - LOG.info("State change starting for {} with latest completed action instant {}", + log.info("State change starting for {} with latest completed action instant {}", changeActionInstant, lastCompletedActionInstant); lockManager.lock(); reset(this.changeActionInstant, changeActionInstant, lastCompletedActionInstant); - LOG.info("State change started for {} with latest completed action instant {}", + log.info("State change started for {} with latest completed action instant {}", changeActionInstant, lastCompletedActionInstant); } } public void endStateChange(Option<HoodieInstant> changeActionInstant) { if (isLockRequired) { - LOG.info("State change ending for action instant {}", changeActionInstant); + log.info("State change ending for action instant {}", changeActionInstant); if (reset(changeActionInstant, Option.empty(), Option.empty())) { lockManager.unlock(); - LOG.info("State change ended for action instant {}", changeActionInstant); + log.info("State change ended for action instant {}", changeActionInstant); } } } @@ -90,7 +89,7 @@ public class TransactionManager implements Serializable, AutoCloseable { public void close() { if (isLockRequired) { lockManager.close(); - LOG.debug("Transaction manager closed"); + log.debug("Transaction manager closed"); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/PreWriteValidatorUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/PreWriteValidatorUtils.java index e6d596938287..f20796f5d983 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/PreWriteValidatorUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/validator/PreWriteValidatorUtils.java @@ -30,8 +30,7 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieValidationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.Arrays; import java.util.List; @@ -42,9 +41,8 @@ import java.util.stream.Stream; /** * Utility class for running pre-write validators. */ +@Slf4j public class PreWriteValidatorUtils { - private static final Logger LOG = LoggerFactory.getLogger(PreWriteValidatorUtils.class); - /** * Run all configured pre-write validators. * Validators are run asynchronously in parallel, and if any validator fails, @@ -69,7 +67,7 @@ public class PreWriteValidatorUtils { String validatorClassNames = config.getPreWriteValidators(); if (StringUtils.isNullOrEmpty(validatorClassNames)) { - LOG.debug("No pre-write validators configured."); + log.debug("No pre-write validators configured."); return; } @@ -79,7 +77,7 @@ public class PreWriteValidatorUtils { .filter(className -> !className.isEmpty()) .map(className -> (PreWriteValidator) ReflectionUtils.loadClass(className)); - LOG.info("Running pre-write validators for instant {}", instantTime); + log.info("Running pre-write validators for instant {}", instantTime); // Collect all futures first to ensure parallel execution List<CompletableFuture<Boolean>> futures = validators @@ -92,12 +90,12 @@ public class PreWriteValidatorUtils { .reduce(true, Boolean::logicalAnd); long duration = timer.endTimer(); - LOG.info("Pre-write validation completed in {} ms", duration); + log.info("Pre-write validation completed in {} ms", duration); if (allSuccess) { - LOG.info("All pre-write validations succeeded"); + log.info("All pre-write validations succeeded"); } else { - LOG.error("At least one pre-write validation failed"); + log.error("At least one pre-write validation failed"); throw new HoodieValidationException("At least one pre-write validation failed"); } } @@ -116,19 +114,19 @@ public class PreWriteValidatorUtils { Option<HoodieData<HoodieRecord<T>>> recordsOpt) { return CompletableFuture.supplyAsync(() -> { String validatorName = validator.getName(); - LOG.info("Running pre-write validator: {}", validatorName); + log.info("Running pre-write validator: {}", validatorName); try { HoodieTimer timer = HoodieTimer.start(); validator.validate(instantTime, writeOperationType, metaClient, writeConfig, engineContext, recordsOpt); long duration = timer.endTimer(); - LOG.info("Pre-write validator {} completed successfully in {} ms", validatorName, duration); + log.info("Pre-write validator {} completed successfully in {} ms", validatorName, duration); return true; } catch (HoodieValidationException e) { - LOG.error("Pre-write validation failed for validator {}: {}", validatorName, e.getMessage(), e); + log.error("Pre-write validation failed for validator {}: {}", validatorName, e.getMessage(), e); return false; } catch (Exception e) { - LOG.error("Unexpected error running pre-write validator {}: {}", validatorName, e.getMessage(), e); + log.error("Unexpected error running pre-write validator {}: {}", validatorName, e.getMessage(), e); return false; } }); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java index ccdee0bdc032..45b3536562c5 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/FileMetadataWriteStatusConverter.java @@ -30,8 +30,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.Map; @@ -39,9 +38,9 @@ import java.util.Map; /** * This class is mainly used by the SparkExternalFileClusteringExecutionStrategy to generate WriteStatus classes. */ +@Slf4j public class FileMetadataWriteStatusConverter<T extends HoodieRecordPayload, I, K, O> { - private static final Logger LOG = LoggerFactory.getLogger(FileMetadataWriteStatusConverter.class); private final HoodieTable<T, I, K, O> hoodieTable; private final HoodieWriteConfig writeConfig; private final HoodieStorage storage; @@ -61,7 +60,7 @@ public class FileMetadataWriteStatusConverter<T extends HoodieRecordPayload, I, */ public WriteStatus convert(String parquetFile, String partitionPath, Map<String, Object> executionConfigs) throws IOException { - LOG.info("Creating write status for parquet file {}", parquetFile); + log.info("Creating write status for parquet file {}", parquetFile); WriteStatus writeStatus = (WriteStatus) ReflectionUtils.loadClass(this.writeConfig.getWriteStatusClassName(), this.hoodieTable.shouldTrackSuccessRecords(), this.writeConfig.getWriteStatusFailureFraction(), this.hoodieTable.isMetadataTable()); StoragePath parquetFilePath = new StoragePath(parquetFile); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/partition/RegexRuleEngine.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/partition/RegexRuleEngine.java index 9d6bf4a05d8a..61afee5e002d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/partition/RegexRuleEngine.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bucket/partition/RegexRuleEngine.java @@ -20,8 +20,7 @@ package org.apache.hudi.index.bucket.partition; import org.apache.hudi.exception.HoodieException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.Serializable; import java.util.ArrayList; @@ -32,9 +31,9 @@ import java.util.regex.Pattern; /** * Regex-based rule engine implementation. */ +@Slf4j public class RegexRuleEngine implements RuleEngine { - private static final Logger LOG = LoggerFactory.getLogger(PartitionBucketIndexCalculator.class); private static final long serialVersionUID = 1L; private final List<RegexRule> rules = new ArrayList<>(); @@ -91,10 +90,10 @@ public class RegexRuleEngine implements RuleEngine { } rules.add(new RegexRule(regex, bucketNumber)); - LOG.info("Added regex rule: {} with bucket number: {}", regex, bucketNumber); + log.info("Added regex rule: {} with bucket number: {}", regex, bucketNumber); } - LOG.info("Initialized {} regex rules", rules.size()); + log.info("Initialized {} regex rules", rules.size()); } @Override @@ -102,7 +101,7 @@ public class RegexRuleEngine implements RuleEngine { // Check each rule in order (priority by position) for (RegexRule rule : rules) { if (rule.matches(partitionPath)) { - LOG.debug("Partition '{}' matched regex rule: {}", partitionPath, rule); + log.debug("Partition '{}' matched regex rule: {}", partitionPath, rule); return rule.numBuckets; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java index 23fde1c5ee09..155816e94828 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/ExternalFileClusteringWriteHandle.java @@ -30,8 +30,7 @@ import org.apache.hudi.execution.FileMetadataWriteStatusConverter; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.Collections; @@ -45,9 +44,9 @@ import static org.apache.hudi.execution.FileMetadataWriteStatusConverter.TIME_TA /** * Write handle that is used to work on top of files rather than on individual records. */ +@Slf4j public class ExternalFileClusteringWriteHandle<T extends HoodieRecordPayload, I, K, O> extends HoodieWriteHandle<T, I, K, O> { - private static final Logger LOG = LoggerFactory.getLogger(ExternalFileClusteringWriteHandle.class); private final StoragePath path; private final String prevCommit; @@ -63,7 +62,7 @@ public class ExternalFileClusteringWriteHandle<T extends HoodieRecordPayload, I, // Create inProgress marker file createMarkerFile(partitionPath, path.getName()); - LOG.info("New ExternalFileClusteringWriteHandle for partition :{} with fileId {}", partitionPath, fileId); + log.info("New ExternalFileClusteringWriteHandle for partition :{} with fileId {}", partitionPath, fileId); } /** @@ -82,7 +81,7 @@ public class ExternalFileClusteringWriteHandle<T extends HoodieRecordPayload, I, executionConfigs.put(TIME_TAKEN, timer.endTimer()); this.writeStatus = generateWriteStatus(path.toString(), partitionPath, executionConfigs); - LOG.info(String.format("ExternalFileClusteringWriteHandle for partitionPath %s fileID %s, took %d ms.", + log.info(String.format("ExternalFileClusteringWriteHandle for partitionPath %s fileID %s, took %d ms.", writeStatus.getStat().getPartitionPath(), writeStatus.getStat().getFileId(), writeStatus.getStat().getRuntimeStats().getTotalCreateTime())); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java index c2087d6a56a1..3223ef2f14ca 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java @@ -56,9 +56,8 @@ import org.apache.hudi.keygen.parser.BaseHoodieDateTimeParser; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -77,6 +76,7 @@ import static org.apache.hudi.keygen.KeyGenerator.EMPTY_RECORDKEY_PLACEHOLDER; import static org.apache.hudi.keygen.KeyGenerator.NULL_RECORDKEY_PLACEHOLDER; import static org.apache.hudi.keygen.KeyGenerator.constructRecordKey; +@Slf4j public class KeyGenUtils { /** @@ -84,7 +84,6 @@ public class KeyGenUtils { * that a table whose latest commits wrote no data files does not turn the upgrade into a full timeline scan. */ private static final int MAX_INSTANTS_SCANNED_FOR_ENCODING = 20; - private static final Logger LOG = LoggerFactory.getLogger(KeyGenUtils.class); protected static final String HUDI_DEFAULT_PARTITION_PATH = PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH; public static final String DEFAULT_PARTITION_PATH_SEPARATOR = "/"; @@ -586,7 +585,7 @@ public class KeyGenUtils { props.setProperty(HoodieTableConfig.COMPLEX_KEYGEN_ENCODING.key(), encoding.name()); HoodieTableConfig.update(metaClient.getStorage(), metaClient.getMetaPath(), props); metaClient.reloadTableConfig(); - LOG.info("Recorded complex keygen record key encoding {} on table {}", encoding, metaClient.getBasePath()); + log.info("Recorded complex keygen record key encoding {} on table {}", encoding, metaClient.getBasePath()); } finally { transactionManager.endStateChange(Option.empty()); } @@ -631,7 +630,7 @@ public class KeyGenUtils { } ComplexKeyGenEncoding configured = ComplexKeyGenEncoding.fromUseNewEncoding(config.getBooleanOrDefault(COMPLEX_KEYGEN_NEW_ENCODING)); - LOG.warn("Could not determine the record key encoding of table {} from its data; recording the configured {} " + log.warn("Could not determine the record key encoding of table {} from its data; recording the configured {} " + "because {} is disabled. If the existing keys are not {}, fix {} and rerun, or the records written " + "from now on will not match the existing ones.", metaClient.getBasePath(), configured, ENABLE_COMPLEX_KEYGEN_VALIDATION.key(), configured, @@ -671,13 +670,13 @@ public class KeyGenUtils { if (recordKey.isPresent()) { ComplexKeyGenEncoding encoding = ComplexKeyGenEncoding.fromUseNewEncoding(!recordKey.get().startsWith(expectedPrefix)); - LOG.info("Deduced complex keygen record key encoding {} of table {} from {}", + log.info("Deduced complex keygen record key encoding {} of table {} from {}", encoding, metaClient.getBasePath(), path); return Option.of(encoding); } } } - LOG.warn("The most recent {} commit(s) of table {} yielded no data file with a readable record key, so the " + log.warn("The most recent {} commit(s) of table {} yielded no data file with a readable record key, so the " + "complex keygen record key encoding cannot be deduced from the data.", instantsToScan.size(), metaClient.getBasePath()); return Option.empty(); @@ -752,7 +751,7 @@ public class KeyGenUtils { return keys.hasNext() ? Option.of(keys.next().getRecordKey()) : Option.empty(); } } catch (Exception e) { - LOG.warn("Could not read a record key from {} to deduce the complex keygen record key encoding", path, e); + log.warn("Could not read a record key from {} to deduce the complex keygen record key encoding", path, e); return Option.empty(); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index e36f9b14b709..d1d321d822fc 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -90,8 +90,7 @@ import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.table.BulkInsertPartitioner; import lombok.Getter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.FileNotFoundException; import java.io.IOException; @@ -137,10 +136,9 @@ import static org.apache.hudi.metadata.MetadataPartitionType.fromPartitionPath; * * @param <I> Type of input for the write client */ +@Slf4j public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTableMetadataWriter<I, O>, RunsTableService { - static final Logger LOG = LoggerFactory.getLogger(HoodieBackedTableMetadataWriter.class); - // Virtual keys support for metadata table. This Field is // from the metadata payload schema. private static final String RECORD_KEY_FIELD_NAME = HoodieMetadataPayload.KEY_FIELD_NAME; @@ -208,7 +206,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab initRegistry(); initialized = initializeIfNeeded(dataMetaClient, inflightInstantTimestamp); } catch (IOException e) { - LOG.error("Failed to initialize metadata table", e); + log.error("Failed to initialize metadata table", e); } } ValidationUtils.checkArgument(!initialized || this.metadata != null, "MDT Reader should have been opened post initialization"); @@ -276,7 +274,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab // NOTE: It needs to be guarded by async index config because if that is enabled then initialization happens through the index scheduler. if (!dataWriteConfig.isMetadataAsyncIndex()) { Set<String> completedPartitions = dataMetaClient.getTableConfig().getMetadataPartitions(); - LOG.info("Async metadata indexing disabled and following partitions already initialized: {}", completedPartitions); + log.info("Async metadata indexing disabled and following partitions already initialized: {}", completedPartitions); this.enabledIndexerMap.entrySet().stream() .filter(e -> !completedPartitions.contains(e.getKey().getPartitionPath()) && FILES != e.getKey()) .forEach(e -> indexerMapForPartitionsToInit.put(e.getKey(), e.getValue())); @@ -293,13 +291,13 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab // Otherwise, we use the timestamp of the latest completed action. String dataTableInstantTime = dataMetaClient.getActiveTimeline().filterCompletedInstants().lastInstant().map(HoodieInstant::requestedTime).orElse(SOLO_COMMIT_TIMESTAMP); if (!initializeFromFilesystem(dataTableInstantTime, indexerMapForPartitionsToInit, inflightInstantTimestamp)) { - LOG.error("Failed to initialize MDT from filesystem"); + log.error("Failed to initialize MDT from filesystem"); return false; } metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.INITIALIZE_STR, timer.endTimer())); return true; } catch (IOException | HoodieIOException e) { - LOG.error("Failed to initialize metadata table. Disabling the writer.", e); + log.error("Failed to initialize metadata table. Disabling the writer.", e); return false; } } @@ -317,7 +315,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab .setBasePath(metadataWriteConfig.getBasePath()) .setTimeGeneratorConfig(dataWriteConfig.getTimeGeneratorConfig()).build(); if (DEFAULT_METADATA_POPULATE_META_FIELDS != metadataMetaClient.getTableConfig().populateMetaFields()) { - LOG.info("Re-initiating metadata table properties since populate meta fields have changed"); + log.info("Re-initiating metadata table properties since populate meta fields have changed"); metadataMetaClient = initializeMetaClient(); } } catch (TableNotFoundException e) { @@ -331,7 +329,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab if (reInitialize) { metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.REBOOTSTRAP_STR, 1)); - LOG.info("Deleting Metadata Table directory so that it can be re-initialized"); + log.info("Deleting Metadata Table directory so that it can be re-initialized"); HoodieTableMetadataUtil.deleteMetadataTable(dataMetaClient, engineContext, false); exists = false; } @@ -352,7 +350,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab */ private boolean isBootstrapNeeded(Option<HoodieInstant> latestMetadataInstant) { if (!latestMetadataInstant.isPresent()) { - LOG.warn("Metadata Table will need to be re-initialized as no instants were found"); + log.warn("Metadata Table will need to be re-initialized as no instants were found"); return true; } return false; @@ -449,7 +447,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab Lazy.lazily(() -> HoodieTableMetadataUtil.tryResolveSchemaForTable(dataMetaClient))); initializationList = indexer.buildInitialization(initializationContext); if (initializationList.isEmpty()) { - LOG.info("Skip building {} index in metadata table", partitionType.getPartitionPath()); + log.info("Skip building {} index in metadata table", partitionType.getPartitionPath()); return; } @@ -460,7 +458,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab IndexInitializationPlan initializationPlan = initializationList.get(0); final int numFileGroup = initializationPlan.totalFileGroups(); String relativePartitionPath = initializationPlan.indexPartitionName(); - LOG.info("Initializing {} index with {} file groups", relativePartitionPath, numFileGroup); + log.info("Initializing {} index with {} file groups", relativePartitionPath, numFileGroup); HoodieTimer partitionInitTimer = HoodieTimer.start(); clearExistingMetadataPartition(relativePartitionPath); @@ -481,13 +479,13 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab // initialize metadata reader initMetadataReader(); long totalInitTime = partitionInitTimer.endTimer(); - LOG.info("Initializing {} index in metadata table took {} in ms", partitionType, totalInitTime); + log.info("Initializing {} index in metadata table took {} in ms", partitionType, totalInitTime); } catch (Exception e) { String metricKey = partitionType.getPartitionPath() + "_" + HoodieMetadataMetrics.BOOTSTRAP_ERR_STR; metrics.ifPresent(m -> m.setMetric(metricKey, 1)); String errMsg = String.format("Bootstrap on %s partition failed for %s", partitionType.getPartitionPath(), metadataMetaClient.getBasePath()); - LOG.error(errMsg, e); + log.error(errMsg, e); throw new HoodieMetadataException(errMsg, e); } hasPartitionsStateChanged = true; @@ -637,7 +635,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab if (!dirFilterRegex.isEmpty()) { final String relativePath = dirInfo.getRelativePath(); if (!relativePath.isEmpty() && relativePath.matches(dirFilterRegex)) { - LOG.info("Ignoring directory {} which matches the filter regex {}", relativePath, dirFilterRegex); + log.info("Ignoring directory {} which matches the filter regex {}", relativePath, dirFilterRegex); continue; } } @@ -702,7 +700,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab // the bulkInsert. final String msg = String.format("Creating %d file groups for partition %s with base fileId %s at instant time %s", fileGroupCount, relativePartitionPath, metadataPartition.getFileIdPrefix(), instantTime); - LOG.info(msg); + log.info(msg); final List<String> fileGroupFileIds = IntStream.range(0, fileGroupCount) .mapToObj(i -> HoodieTableMetadataUtil.getFileIDForFileGroup(metadataPartition, i, relativePartitionPath, dataPartitionName)) .collect(Collectors.toList()); @@ -731,7 +729,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab try { final List<StoragePathInfo> existingFiles = storage.listDirectEntries(partitionPath); if (!existingFiles.isEmpty()) { - LOG.info("Deleting all existing files found in MDT partition {}", relativePartitionPath); + log.info("Deleting all existing files found in MDT partition {}", relativePartitionPath); storage.deleteDirectory(partitionPath); ValidationUtils.checkState(!storage.exists(partitionPath), "Failed to delete MDT partition " + relativePartitionPath); @@ -743,11 +741,11 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab public void dropMetadataPartitions(List<String> metadataPartitions) throws IOException { for (String partitionPath : metadataPartitions) { - LOG.info("Deleting Metadata Table partition: {}", partitionPath); + log.info("Deleting Metadata Table partition: {}", partitionPath); dataMetaClient.getStorage() .deleteDirectory(new StoragePath(metadataWriteConfig.getBasePath(), partitionPath)); // delete corresponding pending indexing instant file in the timeline - LOG.info("Deleting pending indexing instant from the timeline for partition: {}", partitionPath); + log.info("Deleting pending indexing instant from the timeline for partition: {}", partitionPath); deletePendingIndexingInstant(dataMetaClient, partitionPath); } closeInternal(); @@ -770,7 +768,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab metaClient.getActiveTimeline().deleteInstantFileIfExists(instantGenerator.getIndexInflightInstant(instant.requestedTime())); } } catch (IOException e) { - LOG.error("Failed to delete the instant file corresponding to {}", instant); + log.error("Failed to delete the instant file corresponding to {}", instant); } }); } @@ -812,13 +810,13 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab return partitionsToUpdate; } // fallback to all enabled partitions if table config returned no partitions - LOG.debug("There are no partitions to update according to table config. Falling back to enabled partition types in the write config."); + log.debug("There are no partitions to update according to table config. Falling back to enabled partition types in the write config."); return enabledIndexerMap.keySet().stream().map(MetadataPartitionType::getPartitionPath).collect(Collectors.toSet()); } public void buildMetadataPartitions(HoodieEngineContext engineContext, List<HoodieIndexPartitionInfo> indexPartitionInfos, String instantTime) throws IOException { if (indexPartitionInfos.isEmpty()) { - LOG.debug("No partition to index in the plan"); + log.debug("No partition to index in the plan"); return; } String indexUptoInstantTime = indexPartitionInfos.get(0).getIndexUptoInstant(); @@ -826,7 +824,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab List<MetadataPartitionType> partitionTypes = new ArrayList<>(); indexPartitionInfos.forEach(indexPartitionInfo -> { String relativePartitionPath = indexPartitionInfo.getMetadataPartitionPath(); - LOG.info("Creating a new metadata index for partition '{}' under path {} upto instant {}", + log.info("Creating a new metadata index for partition '{}' under path {} upto instant {}", relativePartitionPath, metadataWriteConfig.getBasePath(), indexUptoInstantTime); // return early and populate enabledPartitionTypes correctly (check in initialCommit) @@ -850,9 +848,9 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab if (!metadataMetaClient.getActiveTimeline().getCommitsTimeline().containsInstant(instantTime)) { // if this is a new commit being applied to metadata for the first time - LOG.info("New commit at {} being applied to metadata table", instantTime); + log.info("New commit at {} being applied to metadata table", instantTime); } else { - LOG.error("Rolling back already inflight commit in Metadata table {} ", instantTime); + log.error("Rolling back already inflight commit in Metadata table {} ", instantTime); getWriteClient().rollback(instantTime); } @@ -979,7 +977,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab public void completeStreamingCommit(String instantTime, HoodieEngineContext context, List<HoodieWriteStat> partialWriteStats, HoodieCommitMetadata metadata) { checkClusteringKeepsRecordKeys(metadata); if (metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(instantTime)) { - LOG.info("Skipping streaming metadata commit completion for already completed instant {}", instantTime); + log.info("Skipping streaming metadata commit completion for already completed instant {}", instantTime); getWriteClient().postCommit(instantTime); return; } @@ -1140,7 +1138,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab initMetadataReader(); } } catch (IOException e) { - LOG.error("Failed to initialize newly added partitions for the partitioned record index", e); + log.error("Failed to initialize newly added partitions for the partitioned record index", e); throw new HoodieMetadataException("Failed to initialize newly added partitions for the partitioned record index", e); } } @@ -1215,7 +1213,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab throw new HoodieIOException(String.format("Deserialization of restore plan failed whose restore instant time is %s in data table", instantTime), e); } final String restoreToInstantTime = restorePlan.getSavepointToRestoreTimestamp(); - LOG.info("Triggering restore to {} in metadata table", restoreToInstantTime); + log.info("Triggering restore to {} in metadata table", restoreToInstantTime); // fetch the earliest commit to retain and ensure the base file prior to the time to restore is present List<HoodieFileGroup> filesGroups = metadata.getMetadataFileSystemView().getAllFileGroups(FILES.getPartitionPath()).collect(Collectors.toList()); @@ -1296,12 +1294,12 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab if (deltaCommitInstant.isCompleted()) { validateRollback(deltaCommitInstant); } - LOG.info("Rolling back MDT deltacommit {}", commitToRollbackInstantTime); + log.info("Rolling back MDT deltacommit {}", commitToRollbackInstantTime); if (!getWriteClient().rollback(commitToRollbackInstantTime, instantTime)) { throw new HoodieMetadataException(String.format("Failed to rollback deltacommit at %s", commitToRollbackInstantTime)); } } else { - LOG.info("Ignoring rollback of instant {} at {}. The commit to rollback is not found in MDT", + log.info("Ignoring rollback of instant {} at {}. The commit to rollback is not found in MDT", commitToRollbackInstantTime, instantTime); } closeInternal(); @@ -1382,7 +1380,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab if (!metadataMetaClient.getActiveTimeline().getCommitsTimeline().containsInstant(instantTime)) { // if this is a new commit being applied to metadata for the first time - LOG.info("New commit at {} being applied to MDT.", instantTime); + log.info("New commit at {} being applied to MDT.", instantTime); } else { // this code path refers to a re-attempted commit that: // 1. got committed to metadata table, but failed in datatable. @@ -1394,7 +1392,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab // already part of completed commit. So, we have to manually rollback the completed instant and proceed. Option<HoodieInstant> alreadyCompletedInstant = metadataMetaClient.getActiveTimeline().filterCompletedInstants().filter(entry -> entry.requestedTime().equals(instantTime)) .lastInstant(); - LOG.info("{} completed commit at {} being applied to MDT.", + log.info("{} completed commit at {} being applied to MDT.", alreadyCompletedInstant.isPresent() ? "Already" : "Partially", instantTime); // Rollback the previous commit @@ -1583,13 +1581,13 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab // Do timeline validation before scheduling compaction/logCompaction operations. if (validateCompactionScheduling(inFlightInstantTimestamp, lastInstant.get().requestedTime())) { String latestDeltacommitTime = lastInstant.get().requestedTime(); - LOG.info("Latest deltacommit time found is {}, running compaction operations.", latestDeltacommitTime); + log.info("Latest deltacommit time found is {}, running compaction operations.", latestDeltacommitTime); compactIfNecessary(writeClient, Option.of(latestDeltacommitTime)); } writeClient.archive(); - LOG.info("All the table services operations on MDT completed successfully"); + log.info("All the table services operations on MDT completed successfully"); } catch (Exception e) { - LOG.error("Exception in running table services on metadata table", e); + log.error("Exception in running table services on metadata table", e); allTableServicesExecutedSuccessfullyOrSkipped = false; if (dataWriteConfig.getMetadataConfig().shouldFailOnTableServiceFailures()) { throw e; @@ -1623,7 +1621,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab boolean ranServices = false; if (activeTimeline.filterPendingCompactionTimeline().countInstants() > 0) { if (writeClient.shouldDelegateToTableServiceManager(writeClient.getConfig(), ActionType.compaction)) { - LOG.info("Skipping pending compactions on MDT as they are delegated to table service manager."); + log.info("Skipping pending compactions on MDT as they are delegated to table service manager."); } else { writeClient.runAnyPendingCompactions(); ranServices = true; @@ -1631,7 +1629,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab } if (activeTimeline.filterPendingLogCompactionTimeline().countInstants() > 0) { if (writeClient.shouldDelegateToTableServiceManager(writeClient.getConfig(), ActionType.logcompaction)) { - LOG.info("Skipping pending log compactions on MDT as they are delegated to table service manager."); + log.info("Skipping pending log compactions on MDT as they are delegated to table service manager."); } else { writeClient.runAnyPendingLogCompactions(); ranServices = true; @@ -1675,31 +1673,31 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab boolean skipCompactions = metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(compactionInstantTime); try { if (skipCompactions) { - LOG.info("Compaction with same {} time is already present in the timeline.", compactionInstantTime); + log.info("Compaction with same {} time is already present in the timeline.", compactionInstantTime); } else if (writeClient.scheduleCompactionAtInstant(compactionInstantTime, Option.empty())) { - LOG.info("Compaction is scheduled for timestamp {}", compactionInstantTime); + log.info("Compaction is scheduled for timestamp {}", compactionInstantTime); if (shouldDelegateToTableServiceManager(metadataWriteConfig, ActionType.compaction)) { - LOG.info("Skipping execution of compaction on MDT as it is delegated to table service manager."); + log.info("Skipping execution of compaction on MDT as it is delegated to table service manager."); } else { writeClient.compact(compactionInstantTime, true); } } } catch (Exception e) { metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.COMPACTION_FAILURES, 1)); - LOG.error("Error in scheduling and executing compaction in metadata table", e); + log.error("Error in scheduling and executing compaction in metadata table", e); throw e; } try { if (skipCompactions) { - LOG.info("Compaction with same {} time is already present in the timeline.", compactionInstantTime); + log.info("Compaction with same {} time is already present in the timeline.", compactionInstantTime); } else if (metadataWriteConfig.isLogCompactionEnabled()) { // Schedule and execute log compaction with new instant time. Option<String> scheduledLogCompaction = writeClient.scheduleLogCompaction(Option.empty()); if (scheduledLogCompaction.isPresent()) { - LOG.info("Log compaction is scheduled for timestamp {}", scheduledLogCompaction.get()); + log.info("Log compaction is scheduled for timestamp {}", scheduledLogCompaction.get()); if (shouldDelegateToTableServiceManager(metadataWriteConfig, ActionType.logcompaction)) { - LOG.info("Skipping execution of log compaction on MDT as it is delegated to table service manager."); + log.info("Skipping execution of log compaction on MDT as it is delegated to table service manager."); } else { writeClient.logCompact(scheduledLogCompaction.get(), true); } @@ -1707,7 +1705,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab } } catch (Exception e) { metrics.ifPresent(m -> m.incrementMetric(HoodieMetadataMetrics.LOG_COMPACTION_FAILURES, 1)); - LOG.error("Error in scheduling and executing logcompaction in metadata table", e); + log.error("Error in scheduling and executing logcompaction in metadata table", e); throw e; } } @@ -1749,7 +1747,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab Option<HoodieInstant> pendingCompactionInstant = metadataMetaClient.getActiveTimeline().filterPendingCompactionTimeline().firstInstant(); if (pendingLogCompactionInstant.isPresent() || pendingCompactionInstant.isPresent()) { - LOG.info("Not scheduling compaction or logCompaction, since a pending compaction instant {} or logCompaction {} instant is present", + log.info("Not scheduling compaction or logCompaction, since a pending compaction instant {} or logCompaction {} instant is present", pendingCompactionInstant, pendingLogCompactionInstant); return false; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java index 9b7f3c323a60..1d3e7227f6fe 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriterTableVersionSix.java @@ -40,6 +40,8 @@ import org.apache.hudi.metadata.index.EngineIndexerSupport; import org.apache.hudi.metadata.index.model.IndexPartitionAndRecords; import org.apache.hudi.storage.StorageConfiguration; +import lombok.extern.slf4j.Slf4j; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -55,6 +57,7 @@ import static org.apache.hudi.common.table.timeline.InstantComparison.compareTim * HoodieBackedTableMetadataWriter for tables with version 6. The class derives most of the functionality from HoodieBackedTableMetadataWriter * and overrides some behaviour to make it compatible for version 6. */ +@Slf4j public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> extends HoodieBackedTableMetadataWriter<I, O> { private static final int PARTITION_INITIALIZATION_TIME_SUFFIX = 10; @@ -106,11 +109,11 @@ public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> exten if (!allBlockingInstantsBeingRolledBack) { metrics.ifPresent(m -> m.updateMetrics(HoodieMetadataMetrics.BOOTSTRAP_ERR_STR, 1)); - LOG.warn("Cannot initialize metadata table as operation(s) are in progress on the dataset: {}", + log.warn("Cannot initialize metadata table as operation(s) are in progress on the dataset: {}", Arrays.toString(blockingPendingInstants.toArray())); return false; } - LOG.info("Allowing metadata table initialization as pending instants {} are being rolled back", + log.info("Allowing metadata table initialization as pending instants {} are being rolled back", Arrays.toString(blockingPendingInstants.toArray())); } return true; @@ -166,7 +169,7 @@ public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> exten if (!pendingInstants.isEmpty()) { checkNumDeltaCommits(metadataMetaClient, dataWriteConfig.getMetadataConfig().getMaxNumDeltacommitsWhenPending()); - LOG.info( + log.info( "Cannot compact metadata table as there are {} inflight instants in data table before latest deltacommit in metadata table: {}. Inflight instants in data table: {}", pendingInstants.size(), latestDeltaCommitTimeInMetadataTable, Arrays.toString(pendingInstants.toArray())); return false; @@ -179,7 +182,7 @@ public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> exten Option<HoodieInstant> pendingCompactionInstant = metadataMetaClient.getActiveTimeline().filterPendingCompactionTimeline().firstInstant(); if (pendingLogCompactionInstant.isPresent() || pendingCompactionInstant.isPresent()) { - LOG.info("Not scheduling compaction or logCompaction, since a pending compaction instant {} or logCompaction {} instant is present", + log.info("Not scheduling compaction or logCompaction, since a pending compaction instant {} or logCompaction {} instant is present", pendingCompactionInstant, pendingLogCompactionInstant); return false; } @@ -234,12 +237,12 @@ public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> exten String rollbackInstantTime = createRollbackTimestamp(instantTime); if (metadataMetaClient.getActiveTimeline().containsInstant(deltaCommitInstant)) { - LOG.info("Rolling back MDT deltacommit {}", commitToRollbackInstantTime); + log.info("Rolling back MDT deltacommit {}", commitToRollbackInstantTime); if (!getWriteClient().rollback(commitToRollbackInstantTime, rollbackInstantTime)) { throw new HoodieMetadataException("Failed to rollback deltacommit at " + commitToRollbackInstantTime); } } else { - LOG.info("Ignoring rollback of instant {} at {}. The commit to rollback is not found in MDT", + log.info("Ignoring rollback of instant {} at {}. The commit to rollback is not found in MDT", commitToRollbackInstantTime, instantTime); } closeInternal(); @@ -300,11 +303,11 @@ public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> exten // and again w/ C6, we will re-attempt compaction at which point latest delta commit is C4 in MDT. // and so we try compaction w/ instant C4001. So, we can avoid compaction if we already have compaction w/ same instant time. if (metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(compactionInstantTime)) { - LOG.info("Compaction with same {} time is already present in the timeline.", compactionInstantTime); + log.info("Compaction with same {} time is already present in the timeline.", compactionInstantTime); } else if (writeClient.scheduleCompactionAtInstant(compactionInstantTime, Option.empty())) { - LOG.info("Compaction is scheduled for timestamp {}", compactionInstantTime); + log.info("Compaction is scheduled for timestamp {}", compactionInstantTime); if (shouldDelegateToTableServiceManager(metadataWriteConfig, ActionType.compaction)) { - LOG.info("Skipping execution of compaction on MDT as it is delegated to table service manager."); + log.info("Skipping execution of compaction on MDT as it is delegated to table service manager."); } else { writeClient.compact(compactionInstantTime, true); } @@ -314,11 +317,11 @@ public abstract class HoodieBackedTableMetadataWriterTableVersionSix<I, O> exten // metadata table. final String logCompactionInstantTime = createLogCompactionTimestamp(latestDeltaCommitTimeOpt.get()); if (metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(logCompactionInstantTime)) { - LOG.info("Log compaction with same {} time is already present in the timeline.", logCompactionInstantTime); + log.info("Log compaction with same {} time is already present in the timeline.", logCompactionInstantTime); } else if (writeClient.scheduleLogCompactionAtInstant(logCompactionInstantTime, Option.empty())) { - LOG.info("Log compaction is scheduled for timestamp {}", logCompactionInstantTime); + log.info("Log compaction is scheduled for timestamp {}", logCompactionInstantTime); if (shouldDelegateToTableServiceManager(metadataWriteConfig, ActionType.logcompaction)) { - LOG.info("Skipping execution of log compaction on MDT as it is delegated to table service manager."); + log.info("Skipping execution of log compaction on MDT as it is delegated to table service manager."); } else { writeClient.logCompact(logCompactionInstantTime, true); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java index 5fd3909f0562..d4f90c33c462 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BucketIndexBulkInsertPartitioner.java @@ -27,8 +27,7 @@ import org.apache.hudi.io.SingleFileHandleCreateFactory; import org.apache.hudi.io.WriteHandleFactory; import org.apache.hudi.keygen.KeyGenUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.List; @@ -36,10 +35,9 @@ import java.util.List; /** * Abstract of bucket index bulk_insert partitioner */ +@Slf4j public abstract class BucketIndexBulkInsertPartitioner<T> extends BucketSortBulkInsertPartitioner<T> { - public static final Logger LOG = LogManager.getLogger(BucketIndexBulkInsertPartitioner.class); - private final boolean preserveHoodieMetadata; protected final boolean consistentLogicalTimestampEnabled; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java index 7a6c3f2f44d3..99a6711f6173 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/cluster/strategy/CommitBasedClusteringPlanStrategy.java @@ -43,8 +43,7 @@ import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; import org.apache.hudi.table.HoodieTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayList; @@ -62,8 +61,8 @@ import static org.apache.hudi.config.HoodieClusteringConfig.PLAN_STRATEGY_SORT_C * This strategy can be used to cluster data based on specific commit criteria * such as commit frequency, commit size, or other commit-based metrics. */ +@Slf4j public class CommitBasedClusteringPlanStrategy<T, I, K, O> extends PartitionAwareClusteringPlanStrategy<T, I, K, O> { - private static final Logger LOG = LoggerFactory.getLogger(CommitBasedClusteringPlanStrategy.class); private String checkpoint; public static final String CLUSTERING_COMMIT_CHECKPOINT_KEY = "clustering.commit.checkpoint"; @@ -83,10 +82,10 @@ public class CommitBasedClusteringPlanStrategy<T, I, K, O> extends PartitionAwar HoodieTableMetaClient metaClient = getHoodieTable().getMetaClient(); String earliestCommit = getWriteConfig().getClusteringEarliestCommitToCluster(); if (earliestCommit == null) { - LOG.error("earliest commit to cluster is not specified"); + log.error("earliest commit to cluster is not specified"); return Option.empty(); } - LOG.info("Earliest commit to cluster (exclusive): {}", earliestCommit); + log.info("Earliest commit to cluster (exclusive): {}", earliestCommit); HoodieTimeline commitTimeline = metaClient.getCommitsTimeline().findInstantsAfter(earliestCommit).filterCompletedInstants(); // For each completed commit, invoke getFileSlicesEligibleForCommitBasedClustering @@ -153,7 +152,7 @@ public class CommitBasedClusteringPlanStrategy<T, I, K, O> extends PartitionAwar checkpoint = lastCommit; if (clusteringGroups.isEmpty()) { - LOG.warn("No data available to cluster"); + log.warn("No data available to cluster"); return Option.empty(); } @@ -247,7 +246,7 @@ public class CommitBasedClusteringPlanStrategy<T, I, K, O> extends PartitionAwar try { commitMetadata = TimelineUtils.getCommitMetadata(instant, metaClient.getActiveTimeline()); } catch (IOException e) { - LOG.error("Failed to read commit metadata for instant: {}", instant, e); + log.error("Failed to read commit metadata for instant: {}", instant, e); throw new HoodieException("Failed to read commit metadata for instant: " + instant, e); } } else { @@ -285,7 +284,7 @@ public class CommitBasedClusteringPlanStrategy<T, I, K, O> extends PartitionAwar try { pathInfo = storage.getPathInfo(path); } catch (Exception e) { - LOG.error("Could not get PathInfo for file path: {}", path, e); + log.error("Could not get PathInfo for file path: {}", path, e); throw new HoodieException("Could not get PathInfo for file path: " + path, e); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java index 0677e955dbaa..9cc6bdacc034 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java @@ -29,8 +29,7 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.keygen.constant.KeyGeneratorOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.Collections; import java.util.Hashtable; @@ -41,9 +40,9 @@ import java.util.stream.Collectors; /** * Upgrade handle to assist in upgrading hoodie table from version 1 to 2. */ +@Slf4j public class OneToTwoUpgradeHandler implements UpgradeHandler { - private static final Logger LOG = LoggerFactory.getLogger(OneToTwoUpgradeHandler.class); private static final String NESTED_FIELD_SEPARATOR = "."; @Override @@ -93,12 +92,12 @@ public class OneToTwoUpgradeHandler implements UpgradeHandler { } Option<HoodieSchema> schema = resolveSchema(config, metaClient); if (!schema.isPresent()) { - LOG.warn("Skipping the ordering fields {} while upgrading {} to table version two: no schema is available to " + log.warn("Skipping the ordering fields {} while upgrading {} to table version two: no schema is available to " + "resolve them against", orderingFields, config.getBasePath()); return Option.empty(); } if (!topLevelFields.stream().allMatch(field -> schema.get().getField(field).isPresent())) { - LOG.warn("Skipping the ordering fields {} while upgrading {} to table version two: the schema does not have " + log.warn("Skipping the ordering fields {} while upgrading {} to table version two: the schema does not have " + "all of them as top level fields", orderingFields, config.getBasePath()); return Option.empty(); } @@ -122,8 +121,8 @@ public class OneToTwoUpgradeHandler implements UpgradeHandler { } catch (Exception e) { // the upgrade gates every write, so a schema that cannot be read or parsed leaves the fields // unrecorded rather than blocking the table - LOG.warn("Failed to resolve the schema of " + config.getBasePath() - + " while upgrading to table version two, leaving the ordering fields unrecorded", e); + log.warn("Failed to resolve the schema of {} while upgrading to table version two, leaving the ordering fields unrecorded", + config.getBasePath(), e); return Option.empty(); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java index f23e76a8c535..c837a394f44b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java @@ -26,8 +26,7 @@ import org.apache.hudi.common.table.HoodieTableVersion; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieUpgradeDowngradeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.HashMap; import java.util.HashSet; @@ -62,10 +61,9 @@ import java.util.Set; * <p>Retaining the mode on a version 9 table is safe mechanically: the property carries no * {@code sinceVersion}, so {@code dropInvalidConfigs} does not strip it on load. */ +@Slf4j public class TenToNineDowngradeHandler implements DowngradeHandler { - private static final Logger LOG = LoggerFactory.getLogger(TenToNineDowngradeHandler.class); - @Override public UpgradeDowngrade.TableConfigChangeSet downgrade( HoodieWriteConfig config, @@ -106,7 +104,7 @@ public class TenToNineDowngradeHandler implements DowngradeHandler { // // Keeping it is also what makes the round trip lossless: a later re-upgrade finds the mode // intact instead of deriving NONE from the boolean. - LOG.warn("Downgrading a table on {}={} to table version 9, retaining the mode on the downgraded " + log.warn("Downgrading a table on {}={} to table version 9, retaining the mode on the downgraded " + "table (created at table version {}). Version 9 does not formally understand it, so " + "every reader of this table must honor {} rather than relying on {} alone -- which now " + "reads false and would have them treat the table as NONE.", diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java index f2d2e6e7d7ae..a0a0554b43f1 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/clustering/plan/strategy/FlinkSizeBasedClusteringPlanStrategyRecently.java @@ -37,8 +37,7 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.cluster.ClusteringPlanActionExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.ArrayList; @@ -51,8 +50,8 @@ import static org.apache.hudi.common.table.timeline.HoodieTimeline.COMMIT_ACTION /** * Only take care of partitions related to active timeline, instead of do full partition listing. */ +@Slf4j(topic = "org.apache.hudi.client.clustering.plan.strategy.FlinkSizeBasedClusteringPlanStrategy") public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBasedClusteringPlanStrategy<T> { - private static final Logger LOG = LoggerFactory.getLogger(FlinkSizeBasedClusteringPlanStrategy.class); public FlinkSizeBasedClusteringPlanStrategyRecently(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) { @@ -69,7 +68,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa } HoodieTableMetaClient metaClient = getHoodieTable().getMetaClient(); - LOG.info("Scheduling clustering for {}", metaClient.getBasePath()); + log.info("Scheduling clustering for {}", metaClient.getBasePath()); List<String> partitionPaths = getPartitionPathInActiveTimeline(hoodieTable); @@ -92,7 +91,7 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa .collect(Collectors.toList()); if (clusteringGroups.isEmpty()) { - LOG.info("No data available to cluster"); + log.info("No data available to cluster"); return Option.empty(); } @@ -124,11 +123,11 @@ public class FlinkSizeBasedClusteringPlanStrategyRecently<T> extends FlinkSizeBa partitions.addAll(metadata.getWritePartitionPaths()); } catch (IOException e) { // ignore Exception here - LOG.warn("Failed to get details from commit metadata for instant [{}].", instant, e); + log.warn("Failed to get details from commit metadata for instant [{}].", instant, e); } }); - LOG.info("Partitions related to active timeline: {}", partitions); + log.info("Partitions related to active timeline: {}", partitions); return new ArrayList<>(partitions); } } \ No newline at end of file diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/FlinkClientUtil.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/FlinkClientUtil.java index baa8fe42de7a..16712a3c798e 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/FlinkClientUtil.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/FlinkClientUtil.java @@ -27,13 +27,12 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieValidationException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.java.hadoop.mapred.utils.HadoopUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.util.Arrays; @@ -42,8 +41,8 @@ import java.util.Map; /** * Utilities for Hoodie Flink client. */ +@Slf4j public class FlinkClientUtil { - private static final Logger LOG = LoggerFactory.getLogger(FlinkClientUtil.class); public static String FLINK_HADOOP_CONFIG_PREFIX = "flink.hadoop"; public static String FLINK_CONF_DIR = "FLINK_CONF_DIR"; private static final String[] HADOOP_SITE_FILES = {"core-site.xml", "hdfs-site.xml", "yarn-site.xml", "mapred-site.xml"}; @@ -88,14 +87,14 @@ public class FlinkClientUtil { private static org.apache.hadoop.conf.Configuration applyFlinkHadoopOptions(org.apache.hadoop.conf.Configuration hadoopConf) { String confDir = System.getenv(FLINK_CONF_DIR); if (!StringUtils.isNullOrEmpty(confDir)) { - LOG.info("Flink conf dir: {}", confDir); + log.info("Flink conf dir: {}", confDir); try { Configuration configuration = GlobalConfiguration.loadConfiguration(confDir); for (Map.Entry<String, String> kv : ConfigurationUtils.getPrefixedKeyValuePairs(FLINK_HADOOP_CONFIG_PREFIX, configuration).entrySet()) { hadoopConf.set(kv.getKey(), kv.getValue()); } } catch (Exception e) { - LOG.warn("Fail to load flink configuration from path {}", confDir, e); + log.warn("Fail to load flink configuration from path {}", confDir, e); } } diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java index e6e37b7e375a..6f1a51be251d 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkClientTestHarness.java @@ -34,11 +34,10 @@ import org.apache.hudi.index.bloom.TestFlinkHoodieBloomIndex; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.table.HoodieTable; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; @@ -49,9 +48,9 @@ import static org.apache.hudi.common.util.ValidationUtils.checkState; /** * The test harness for resource initialization and cleanup. */ +@Slf4j public class HoodieFlinkClientTestHarness extends HoodieCommonTestHarness { - protected static final Logger LOG = LoggerFactory.getLogger(HoodieFlinkClientTestHarness.class); protected StorageConfiguration<Configuration> storageConf; protected FileSystem fs; protected HoodieFlinkEngineContext context; @@ -104,7 +103,7 @@ public class HoodieFlinkClientTestHarness extends HoodieCommonTestHarness { */ protected void cleanupFileSystem() throws IOException { if (fs != null) { - LOG.warn("Closing file-system instance used in previous test-run"); + log.warn("Closing file-system instance used in previous test-run"); fs.close(); fs = null; } @@ -153,7 +152,7 @@ public class HoodieFlinkClientTestHarness extends HoodieCommonTestHarness { */ protected void cleanupFlinkContexts() { if (context != null) { - LOG.info("Closing flink engine context used in previous test-case"); + log.info("Closing flink engine context used in previous test-case"); context = null; } } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java index 50e8d5683283..3f97f9cd5b07 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/common/TestMultipleHoodieJavaWriteClient.java @@ -40,6 +40,7 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.keygen.constant.KeyGeneratorType; import org.apache.hudi.storage.StorageConfiguration; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; @@ -48,8 +49,6 @@ import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Collections; @@ -64,10 +63,8 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorage /** * Tests concurrent Java client writers. */ +@Slf4j public class TestMultipleHoodieJavaWriteClient { - private static final Logger LOGGER = - LoggerFactory.getLogger(TestMultipleHoodieJavaWriteClient.class.getName()); - @TempDir protected java.nio.file.Path tablePath; @@ -197,9 +194,9 @@ public class TestMultipleHoodieJavaWriteClient { .parallel() .forEach( record -> { - LOGGER.info("Inserting record = {}", record); + log.info("Inserting record = {}", record); HoodieJavaWriteClient writer; - LOGGER.info("Entering while loop"); + log.info("Entering while loop"); try { writer = writerQueue.take(); } catch (InterruptedException e) { @@ -210,13 +207,13 @@ public class TestMultipleHoodieJavaWriteClient { try { writer.commit(startCommitTime, s); - LOGGER.info("Completed commit"); + log.info("Completed commit"); } catch (HoodieWriteConflictException e) { // Under OPTIMISTIC_CONCURRENCY_CONTROL, two writers updating overlapping file // groups conflict and one of them is aborted by design. This test validates that // concurrent writers do not deadlock, so an aborted commit is an expected outcome // and must not fail the test. - LOGGER.info("Commit {} was aborted by an expected OCC conflict", startCommitTime); + log.info("Commit {} was aborted by an expected OCC conflict", startCommitTime); } finally { // Always return the writer to the pool, even when its commit was aborted, so the // remaining records are not starved of writers. diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java index 0abcec9c9bee..193df26ebbc3 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SparkExternalFileClusteringExecutionStrategy.java @@ -33,8 +33,7 @@ import org.apache.hudi.io.ExternalFileClusteringWriteHandle; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.Map; @@ -43,11 +42,10 @@ import java.util.Map; * This class gives skeleton implementation for set of clustering execution strategy * that use external file transformation commands. */ +@Slf4j public abstract class SparkExternalFileClusteringExecutionStrategy<T extends HoodieRecordPayload<T>> extends SingleSparkJobExecutionStrategy<T> { - private static final Logger LOG = LoggerFactory.getLogger(SparkExternalFileClusteringExecutionStrategy.class); - public SparkExternalFileClusteringExecutionStrategy( HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) { super(table, engineContext, writeConfig); @@ -61,7 +59,7 @@ public abstract class SparkExternalFileClusteringExecutionStrategy<T extends Hoo HoodieSchema schema, TaskContextSupplier taskContextSupplier, String instantTime) { - LOG.info("Starting clustering operation on input file ids."); + log.info("Starting clustering operation on input file ids."); if (!preserveHoodieMetadata) { throw new HoodieClusteringException( "External file clustering strategy cannot rewrite Hudi metadata fields. preserveHoodieMetadata must be true."); @@ -87,7 +85,7 @@ public abstract class SparkExternalFileClusteringExecutionStrategy<T extends Hoo try { getHoodieTable().getStorage().deleteFile(writeHandler.getPath()); } catch (Exception deleteEx) { - LOG.warn("Failed to clean up partial output file: {}", writeHandler.getPath(), deleteEx); + log.warn("Failed to clean up partial output file: {}", writeHandler.getPath(), deleteEx); } throw new HoodieClusteringException("Failed to transform file: " + dataFilePathStr, e); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java index ae2c61b0e3db..951abcb9b320 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkValidatorUtils.java @@ -41,12 +41,11 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.util.JavaScalaConverters; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.types.StructType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -57,9 +56,8 @@ import java.util.stream.Stream; /** * Spark validator utils to verify and run any pre-commit validators configured. */ +@Slf4j public class SparkValidatorUtils { - private static final Logger LOG = LoggerFactory.getLogger(SparkValidatorUtils.class); - /** * Check configured pre-commit validators and run them. Note that this only works for COW tables * <p> @@ -71,7 +69,7 @@ public class SparkValidatorUtils { HoodieTable table, String instantTime) { if (StringUtils.isNullOrEmpty(config.getPreCommitValidators())) { - LOG.info("no validators configured."); + log.info("no validators configured."); } else { if (!writeMetadata.getWriteStats().isPresent()) { writeMetadata.setWriteStats(writeMetadata.getWriteStatuses().map(WriteStatus::getStat).collectAsList()); @@ -90,7 +88,7 @@ public class SparkValidatorUtils { try { Class<?> clazz = Class.forName(validatorClass); if (!SparkPreCommitValidator.class.isAssignableFrom(clazz)) { - LOG.warn("Skipping validator {} — it does not implement SparkPreCommitValidator. " + log.warn("Skipping validator {} — it does not implement SparkPreCommitValidator. " + "If this is a streaming offset validator (e.g. SparkKafkaOffsetValidator), " + "it will be invoked by SparkStreamerValidatorUtils instead.", validatorClass); return Stream.empty(); @@ -111,9 +109,9 @@ public class SparkValidatorUtils { .stream().reduce(true, Boolean::logicalAnd); if (allSuccess) { - LOG.info("All validations succeeded"); + log.info("All validations succeeded"); } else { - LOG.error("At least one pre-commit validation failed"); + log.error("At least one pre-commit validation failed"); throw new HoodieValidationException("At least one pre-commit validation failed"); } } @@ -128,10 +126,10 @@ public class SparkValidatorUtils { Dataset<Row> beforeState, Dataset<Row> afterState, String instantTime) { try { validator.validate(instantTime, writeMetadata, beforeState, afterState); - LOG.info("validation complete for {}", validator.getClass().getName()); + log.info("validation complete for {}", validator.getClass().getName()); return true; } catch (HoodieValidationException e) { - LOG.error("validation failed for {}", validator.getClass().getName(), e); + log.error("validation failed for {}", validator.getClass().getName(), e); return false; } } @@ -179,9 +177,9 @@ public class SparkValidatorUtils { HoodieSchemaConversionUtils.convertHoodieSchemaToStructType( new TableSchemaResolver(table.getMetaClient()).getTableSchema())); } catch (Exception e) { - LOG.warn("Could not get table schema for empty DataFrame.", e); + log.warn("Could not get table schema for empty DataFrame.", e); if (fallbackSchema.isPresent()) { - LOG.warn("Using fallback schema to create the empty Spark DataFrame: {}", fallbackSchema.get()); + log.warn("Using fallback schema to create the empty Spark DataFrame: {}", fallbackSchema.get()); return sqlContext.createDataFrame(sqlContext.emptyDataFrame().rdd(), fallbackSchema.get()); } return sqlContext.emptyDataFrame(); @@ -200,11 +198,11 @@ public class SparkValidatorUtils { HoodieSchema schema = HoodieSchema.parse(table.getConfig().getWriteSchema()); readerSchema = isPopulateMetaFieldsEnabled ? HoodieSchemaUtils.addMetadataFields(schema) : schema; } else { - LOG.warn("Schema not found from write config, defaulting to parsing schema from latest commit."); + log.warn("Schema not found from write config, defaulting to parsing schema from latest commit."); try { readerSchema = new TableSchemaResolver(table.getMetaClient()).getTableSchema(); } catch (Exception e) { - LOG.warn(String.format("Failed parsing schema from latest commit with exception %s, " + log.warn(String.format("Failed parsing schema from latest commit with exception %s, " + "defaulting to inferring schema from data files", e)); return sqlContext .read() diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java index 9eef249d6738..4e33c99a8d2f 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java @@ -28,8 +28,7 @@ import org.apache.hudi.common.util.collection.FlatLists; import org.apache.hudi.table.BucketIndexBulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.Partitioner; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.HoodieUTF8StringFactory; @@ -43,10 +42,9 @@ import scala.Tuple2; * Abstract of bucket index bulk_insert partitioner */ +@Slf4j public abstract class RDDBucketIndexPartitioner<T> extends BucketIndexBulkInsertPartitioner<JavaRDD<HoodieRecord<T>>> { - public static final Logger LOG = LogManager.getLogger(RDDBucketIndexPartitioner.class); - private final HoodieUTF8StringFactory utf8StringFactory; public RDDBucketIndexPartitioner(HoodieTable table, String sortString, boolean preserveHoodieMetadata) { @@ -119,7 +117,7 @@ public abstract class RDDBucketIndexPartitioner<T> extends BucketIndexBulkInsert */ private JavaRDD<HoodieRecord<T>> doPartitionAndSortByRecordKey(JavaRDD<HoodieRecord<T>> records, Partitioner partitioner) { if (table.getConfig().getBulkInsertSortMode() == BulkInsertSortMode.GLOBAL_SORT) { - LOG.warn("Bucket index does not support global sort mode, the sort will only be done within each data partition"); + log.warn("Bucket index does not support global sort mode, the sort will only be done within each data partition"); } Comparator<HoodieKey> comparator = table.getMetaClient().getTableConfig().isLSMTreeStorageLayout() diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java index caef37fcc0c5..587afba3a612 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java @@ -37,6 +37,7 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.ValidationUtils; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.avro.AvroWriteSupport; import org.apache.parquet.hadoop.api.WriteSupport; @@ -70,8 +71,6 @@ import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.types.YearMonthIntervalType; import org.apache.spark.unsafe.types.UTF8String; import org.apache.spark.util.VersionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; import java.util.Arrays; @@ -112,9 +111,9 @@ import static org.apache.parquet.schema.Type.Repetition.REQUIRED; * <li>The scale of the timestamps is determined by the Hudi writer schema instead of relying on Spark configuration</li> * </ul> */ +@Slf4j public class HoodieRowParquetWriteSupport extends WriteSupport<InternalRow> { - private static final Logger LOG = LoggerFactory.getLogger(HoodieRowParquetWriteSupport.class); private static final HoodieSchema MAP_KEY_SCHEMA = HoodieSchema.create(HoodieSchemaType.STRING); private static final String MAP_REPEATED_NAME = "key_value"; private static final String MAP_KEY_NAME = "key"; @@ -263,7 +262,7 @@ public class HoodieRowParquetWriteSupport extends WriteSupport<InternalRow> { continue; } } else { - LOG.warn("Variant field '{}' has no corresponding HoodieSchema; " + log.warn("Variant field '{}' has no corresponding HoodieSchema; " + "shredding will be skipped. This may indicate a schema mismatch.", field.name()); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java index 337115c4b897..4bd369ca58bf 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java @@ -46,9 +46,8 @@ import org.apache.hudi.metrics.MetricsReporterType; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.table.BulkInsertPartitioner; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; @@ -58,10 +57,9 @@ import static org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy.EAGE import static org.apache.hudi.common.table.timeline.HoodieTimeline.DELTA_COMMIT_ACTION; import static org.apache.hudi.common.table.timeline.HoodieTimeline.REPLACE_COMMIT_ACTION; +@Slf4j public class SparkHoodieBackedTableMetadataWriterTableVersionSix extends HoodieBackedTableMetadataWriterTableVersionSix<JavaRDD<HoodieRecord>, JavaRDD<WriteStatus>> { - private static final Logger LOG = LoggerFactory.getLogger(SparkHoodieBackedTableMetadataWriter.class); - public static HoodieTableMetadataWriter create(StorageConfiguration<?> conf, HoodieWriteConfig writeConfig, HoodieEngineContext context, @@ -161,7 +159,7 @@ public class SparkHoodieBackedTableMetadataWriterTableVersionSix extends HoodieB @Override public void deletePartitions(String instantTime, List<MetadataPartitionType> partitions) { List<String> partitionsToDrop = partitions.stream().map(MetadataPartitionType::getPartitionPath).collect(Collectors.toList()); - LOG.info("Deleting Metadata Table partitions: {}", partitionsToDrop); + log.info("Deleting Metadata Table partitions: {}", partitionsToDrop); SparkRDDWriteClient writeClient = (SparkRDDWriteClient) getWriteClient(); String actionType = CommitUtils.getCommitActionType(WriteOperationType.DELETE_PARTITION, HoodieTableType.MERGE_ON_READ); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java index da7d42ea6aee..5d185d57abd4 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/DistributedRegistry.java @@ -20,11 +20,10 @@ package org.apache.hudi.metrics; import org.apache.hudi.common.metrics.Registry; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.TaskContext; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.util.AccumulatorV2; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.HashMap; @@ -34,11 +33,10 @@ import java.util.concurrent.ConcurrentHashMap; /** * Lightweight Metrics Registry to track Hudi events. */ +@Slf4j public class DistributedRegistry extends AccumulatorV2<Map<String, Long>, Map<String, Long>> implements Registry, Serializable { - private static final Logger LOG = LoggerFactory.getLogger(DistributedRegistry.class); - private final String name; ConcurrentHashMap<String, Long> counters = new ConcurrentHashMap<>(); // The registered accumulator that replaced this one if it went stale, see register(). @@ -120,7 +118,7 @@ public class DistributedRegistry extends AccumulatorV2<Map<String, Long>, Map<St // an unspecified order. Driver only; executors use increment()/add(). if (TaskContext.get() != null) { // Warn rather than throw: this runs inside a task, and a metrics problem must not fail a write. - LOG.warn("DistributedRegistry.set() called from a Spark executor and ignored: it is non-commutative " + log.warn("DistributedRegistry.set() called from a Spark executor and ignored: it is non-commutative " + "under accumulator merges and would produce non-deterministic values. Use increment()/add()."); return; } @@ -136,7 +134,7 @@ public class DistributedRegistry extends AccumulatorV2<Map<String, Long>, Map<St public void release(Map<String, Long> counts) { // Driver-only for the same reason as set(): clamping and eviction are order-dependent under merges. if (TaskContext.get() != null) { - LOG.warn("DistributedRegistry.release() called from a Spark executor and ignored: clamping and eviction " + log.warn("DistributedRegistry.release() called from a Spark executor and ignored: clamping and eviction " + "are order-dependent under accumulator merges. Release at the commit boundary on the driver."); return; } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/RecordIndexLookupMetrics.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/RecordIndexLookupMetrics.java index e901c51a2554..feae5f46065b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/RecordIndexLookupMetrics.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/RecordIndexLookupMetrics.java @@ -25,8 +25,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.Collection; import java.util.HashMap; @@ -42,10 +41,9 @@ import java.util.Set; * no output row at all. So the driver resolves an accumulator-backed {@link DistributedRegistry}, the * lookup closure captures it, tasks add into their own copy, and Spark merges those copies home. */ +@Slf4j public class RecordIndexLookupMetrics { - private static final Logger LOG = LoggerFactory.getLogger(RecordIndexLookupMetrics.class); - /** The registry name, as reported. Keying is done by {@link #registryKey}. */ public static final String REGISTRY_NAME = "HoodieRecordIndexLookup"; @@ -135,7 +133,7 @@ public class RecordIndexLookupMetrics { publish(context, config, hoodieMetrics); } catch (Exception e) { // This runs after the commit has landed. Reporting is not worth failing a completed write over. - LOG.warn("Failed to publish record index lookup metrics; the commit is unaffected.", e); + log.warn("Failed to publish record index lookup metrics; the commit is unaffected.", e); } } @@ -143,7 +141,7 @@ public class RecordIndexLookupMetrics { HoodieMetrics hoodieMetrics) { if (!config.isRecordIndexLookupMetricsEnabled()) { // Callers gate before reaching here, so this is a wiring mistake rather than normal flow. - LOG.warn("Record index lookup metrics drain reached with the feature disabled; nothing published."); + log.warn("Record index lookup metrics drain reached with the feature disabled; nothing published."); return; } if (!(context instanceof HoodieSparkEngineContext)) { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java index 6162cd6e2dab..b0000e37c6d8 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java @@ -34,15 +34,14 @@ import org.apache.hudi.keygen.KeyGeneratorInterface; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.table.HoodieTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; +@Slf4j public abstract class BaseBootstrapMetadataHandler implements BootstrapMetadataHandler { - private static final Logger LOG = LoggerFactory.getLogger(ParquetBootstrapMetadataHandler.class); protected HoodieWriteConfig config; protected HoodieTable table; protected HoodieFileStatus srcFileStatus; @@ -65,7 +64,7 @@ public abstract class BaseBootstrapMetadataHandler implements BootstrapMetadataH .collect(Collectors.toList()); HoodieSchema recordKeySchema = HoodieSchemaUtils.generateProjectionSchema(schema, recordKeyColumns); - LOG.info("Schema to be used for reading record keys: {}", recordKeySchema); + log.info("Schema to be used for reading record keys: {}", recordKeySchema); executeBootstrap(bootstrapHandle, sourceFilePath, keyGenerator, partitionPath, recordKeySchema); } catch (Exception e) { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionTTLActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionTTLActionExecutor.java index 9fa4f16d3333..175fda20c155 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionTTLActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkPartitionTTLActionExecutor.java @@ -30,18 +30,16 @@ import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.ttl.strategy.HoodiePartitionTTLStrategyFactory; import org.apache.hudi.table.action.ttl.strategy.PartitionTTLStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; import java.util.Collections; import java.util.List; +@Slf4j public class SparkPartitionTTLActionExecutor<T> extends BaseSparkCommitActionExecutor<T> { - private static final Logger LOG = LoggerFactory.getLogger(ConsistentBucketBulkInsertDataInternalWriterHelper.class); - public SparkPartitionTTLActionExecutor(HoodieEngineContext context, HoodieWriteConfig config, HoodieTable table, String instantTime) { super(context, config, table, instantTime, WriteOperationType.DELETE_PARTITION); @@ -58,10 +56,10 @@ public class SparkPartitionTTLActionExecutor<T> if (expiredPartitions.isEmpty()) { return emptyResult; } - LOG.info("Partition ttl find the following expired partitions to delete: {}", String.join(",", expiredPartitions)); + log.info("Partition ttl find the following expired partitions to delete: {}", String.join(",", expiredPartitions)); return new SparkAutoCommitExecutor(new SparkDeletePartitionCommitActionExecutor<>(context, config, table, instantTime, expiredPartitions)).execute(); } catch (HoodieDeletePartitionPendingTableServiceException deletePartitionPendingTableServiceException) { - LOG.info("Partition is under table service, do nothing, call delete partition next time."); + log.info("Partition is under table service, do nothing, call delete partition next time."); return emptyResult; } catch (IOException e) { throw new HoodieIOException("Error executing hoodie partition ttl: ", e); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/ClusteringIdentityTestExecutionStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/ClusteringIdentityTestExecutionStrategy.java index 81df56907bf8..385d3a62229f 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/ClusteringIdentityTestExecutionStrategy.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/ClusteringIdentityTestExecutionStrategy.java @@ -33,18 +33,16 @@ import org.apache.hudi.io.MergeUtils; import org.apache.hudi.io.SingleFileHandleCreateFactory; import org.apache.hudi.table.HoodieTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.Collections; import java.util.List; import java.util.Map; +@Slf4j public class ClusteringIdentityTestExecutionStrategy<T extends HoodieRecordPayload<T>> extends SingleSparkJobExecutionStrategy<T> { - private static final Logger LOG = LoggerFactory.getLogger(ClusteringIdentityTestExecutionStrategy.class); - public ClusteringIdentityTestExecutionStrategy(HoodieTable table, HoodieEngineContext engineContext, HoodieWriteConfig writeConfig) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java index 86bd48f20ac4..ccdcb66424b5 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineDynamicRepartition.java @@ -22,15 +22,13 @@ package org.apache.hudi.client.common; import org.apache.hudi.common.data.HoodiePairData; import org.apache.hudi.common.util.Option; import org.apache.hudi.data.HoodieJavaPairRDD; -import org.apache.hudi.metadata.HoodieBackedTestDelayedTableMetadata; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaSparkContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -46,10 +44,9 @@ import scala.Tuple3; import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j public class TestHoodieSparkEngineDynamicRepartition { - private static final Logger LOG = LoggerFactory.getLogger(HoodieBackedTestDelayedTableMetadata.class); - private JavaSparkContext jsc; /** @@ -210,7 +207,7 @@ public class TestHoodieSparkEngineDynamicRepartition { } catch (AssertionError e) { logRDDContent("Original RDD", originalRdd); logRDDContent("Repartitioned RDD", repartitionedRdd); - LOG.error("Validation failed: {}", e.getMessage(), e); + log.error("Validation failed: {}", e.getMessage(), e); throw e; // rethrow to fail the test } } @@ -249,7 +246,7 @@ public class TestHoodieSparkEngineDynamicRepartition { private static void logRDDContent(String label, HoodiePairData<Integer, String> pairData) { JavaPairRDD<Integer, String> rdd = HoodieJavaPairRDD.getJavaPairRDD(pairData); - LOG.info("===== {} =====", label); + log.info("===== {} =====", label); rdd .mapPartitionsWithIndex((idx, iter) -> { StringBuilder builder = new StringBuilder(); @@ -263,8 +260,8 @@ public class TestHoodieSparkEngineDynamicRepartition { return Collections.singletonList(builder.toString()).iterator(); }, true) .collect() - .forEach(LOG::info); - LOG.info("============================\n"); + .forEach(log::info); + log.info("============================\n"); } @BeforeEach diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java index a7fb48f9e16d..bfad1c42088f 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java @@ -45,11 +45,10 @@ import org.apache.hudi.index.SparkHoodieIndexFactory; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.table.HoodieSparkTable; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; @@ -65,10 +64,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Base Class providing setup/cleanup and utility methods for testing Hoodie Client facing tests. */ +@Slf4j public class HoodieClientTestBase extends HoodieSparkClientTestHarness { - protected static final Logger LOG = LoggerFactory.getLogger(HoodieClientTestBase.class); - @BeforeEach public void setUp() throws Exception { initResources(); diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java index 9cfe32750acf..565250b59c15 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java @@ -75,6 +75,7 @@ import org.apache.hudi.table.marker.SimpleTransactionDirectMarkerBasedDetectionS import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.timeline.service.handlers.marker.AsyncTimelineServerBasedDetectionStrategy; +import lombok.extern.slf4j.Slf4j; import org.apache.curator.test.TestingServer; import org.apache.spark.SparkException; import org.apache.spark.api.java.JavaRDD; @@ -139,6 +140,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @Tag("functional") +@Slf4j public class TestHoodieClientMultiWriter extends HoodieClientTestBase { // Pin the tolerable heartbeat misses used by the early-conflict-detection test so its @@ -1033,7 +1035,7 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase { writer1Succeeded.set(true); } catch (Exception e) { // Expected - one writer may fail due to concurrent execution - LOG.info("Writer 1 failed with exception: {}", e.getMessage()); + log.info("Writer 1 failed with exception: {}", e.getMessage()); writer1Succeeded.set(false); } }); @@ -1048,7 +1050,7 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase { writer2Succeeded.set(true); } catch (Exception e) { // Expected - one writer may fail due to concurrent execution - LOG.info("Writer 2 failed with exception: {}", e.getMessage()); + log.info("Writer 2 failed with exception: {}", e.getMessage()); writer2Succeeded.set(false); } }); @@ -1647,9 +1649,9 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase { try { ingestBatch(writeFn, client1, newCommitTime1, writeRecords1, runCountDownLatch); } catch (IOException e) { - LOG.error("IOException thrown {}", e.getMessage()); + log.error("IOException thrown {}", e.getMessage()); } catch (InterruptedException e) { - LOG.error("Interrupted Exception thrown {}", e.getMessage()); + log.error("Interrupted Exception thrown {}", e.getMessage()); } catch (Exception e) { client1Succeeded.set(false); } @@ -1660,9 +1662,9 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase { try { ingestBatch(writeFn, client2, newCommitTime2, writeRecords2, runCountDownLatch); } catch (IOException e) { - LOG.error("IOException thrown {}", e.getMessage()); + log.error("IOException thrown {}", e.getMessage()); } catch (InterruptedException e) { - LOG.error("Interrupted Exception thrown {}", e.getMessage()); + log.error("Interrupted Exception thrown {}", e.getMessage()); } catch (Exception e) { client2Succeeded.set(false); } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/clustering/run/strategy/TestClusteringTaskBinaryPayloads.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/clustering/run/strategy/TestClusteringTaskBinaryPayloads.java index 05545ff5e550..b666fe3fea2b 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/clustering/run/strategy/TestClusteringTaskBinaryPayloads.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/clustering/run/strategy/TestClusteringTaskBinaryPayloads.java @@ -44,6 +44,7 @@ import org.apache.hudi.data.HoodieJavaRDD; import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.testutils.HoodieClientTestBase; +import lombok.extern.slf4j.Slf4j; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.rdd.RDD; import org.apache.spark.sql.types.StructType; @@ -93,6 +94,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; * written in full. */ @Tag("functional") +@Slf4j public class TestClusteringTaskBinaryPayloads extends HoodieClientTestBase { private static final int SMALL_PLAN_GROUPS = 2; @@ -151,7 +153,7 @@ public class TestClusteringTaskBinaryPayloads extends HoodieClientTestBase { TaskBinary small = taskBinary(strategy, config, firstGroups(plan, SMALL_PLAN_GROUPS), schema, clusteringInstant); TaskBinary large = taskBinary(strategy, config, firstGroups(plan, LARGE_PLAN_GROUPS), schema, clusteringInstant); String report = report(small, large); - LOG.info("Clustering task binary payloads, {}, row writer {}:\n{}", strategy, rowWriter, report); + log.info("Clustering task binary payloads, {}, row writer {}:\n{}", strategy, rowWriter, report); int extraGroups = large.groups - small.groups; for (String payload : small.counts.keySet()) { diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java index a0e5d6dce773..accaf7e66ce3 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestDataValidationCheckForLogCompactionActions.java @@ -47,6 +47,7 @@ import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieSparkWriteableTestTable; import org.apache.hudi.testutils.MetadataMergeWriteStatus; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.spark.api.java.JavaRDD; @@ -74,6 +75,7 @@ import static org.apache.hudi.testutils.GenericRecordValidationTestUtils.assertG import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Slf4j public class TestDataValidationCheckForLogCompactionActions extends HoodieClientTestBase { private static final String RECORD_KEY_APPEND_VALUE = "-EXP"; @@ -122,12 +124,12 @@ public class TestDataValidationCheckForLogCompactionActions extends HoodieClient // Total ingestion writes. int totalWrites = 15; - LOG.warn("Starting trial with seed {}", seed); + log.warn("Starting trial with seed {}", seed); // Current ingestion commit. int curr = 1; while (curr < totalWrites) { - LOG.warn("Starting write No. {}", curr); + log.warn("Starting write No. {}", curr); // Pick an action. It can be insert/update/delete and write data to main table. boolean status = writeOnMainTable(mainTable, curr); @@ -145,7 +147,7 @@ public class TestDataValidationCheckForLogCompactionActions extends HoodieClient // Verify the records in both the tables. verifyRecords(mainTable, experimentTable); - LOG.warn("For write No.{}, verification passed. Last ingestion commit timestamp is {}", curr, mainTable.commitTimeOnMainTable); + log.warn("For write No.{}, verification passed. Last ingestion commit timestamp is {}", curr, mainTable.commitTimeOnMainTable); } curr++; } @@ -195,7 +197,7 @@ public class TestDataValidationCheckForLogCompactionActions extends HoodieClient result = deleteDataIntoMainTable(mainTable, commitTime); } } catch (IllegalArgumentException e) { - LOG.warn("{} ignoring current command.", e.getMessage()); + log.warn("{} ignoring current command.", e.getMessage()); return false; } }
