This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20765-4d7a60132bad2cdc58d62d80a52bcbb34f2ddb34 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit ab0c92675f91302d80e4667d766e783e1989ae10 Author: voonhous <[email protected]> AuthorDate: Fri Nov 20 02:21:28 2026 +0000 refactor(hadoop-mr): use Lombok @Slf4j for loggers (#20765) Replace the hand-declared static SLF4J loggers in hudi-hadoop-mr (23 classes) with Lombok @Slf4j: add the annotation, drop the field and the org.slf4j imports, and rename LOG to log. Topic preserved: RealtimeCompactedRecordReader logged under its parent AbstractRealtimeRecordReader, so it uses @Slf4j(topic = "org.apache.hudi.hadoop.realtime.AbstractRealtimeRecordReader"). Category change: HoodieColumnProjectionUtils declared its logger under Hive's ColumnProjectionUtils (not a supertype); it now logs under its own name. The field was never used, so no log line changes category. The now-unused ColumnProjectionUtils import is removed. Public LOG fields removed from HiveHoodieTableFileIndex, HoodieColumnProjectionUtils, InputPathHandler, HoodieCombineHiveInputFormat, HoodieHiveUtils and Hive3Shim have no references outside their own class anywhere in the repo. --- .../BootstrapColumnStichingRecordReader.java | 8 ++-- .../hudi/hadoop/HiveHoodieTableFileIndex.java | 6 +-- .../hudi/hadoop/HoodieColumnProjectionUtils.java | 7 +--- .../hadoop/HoodieCopyOnWriteTableInputFormat.java | 8 ++-- .../HoodieFileGroupReaderBasedRecordReader.java | 8 ++-- .../hudi/hadoop/HoodieParquetInputFormat.java | 15 ++++---- .../hudi/hadoop/HoodieROTablePathFilter.java | 27 +++++++------- .../org/apache/hudi/hadoop/InputPathHandler.java | 8 ++-- .../hudi/hadoop/RecordReaderValueIterator.java | 8 ++-- .../apache/hudi/hadoop/SchemaEvolutionContext.java | 22 +++++------ .../hadoop/hive/HoodieCombineHiveInputFormat.java | 43 +++++++++++----------- .../realtime/AbstractRealtimeRecordReader.java | 28 +++++++------- .../HoodieCombineRealtimeRecordReader.java | 9 ++--- .../realtime/HoodieHFileRealtimeInputFormat.java | 10 ++--- .../realtime/HoodieParquetRealtimeInputFormat.java | 10 ++--- .../realtime/HoodieRealtimeRecordReader.java | 13 +++---- .../realtime/RealtimeCompactedRecordReader.java | 16 ++++---- .../hudi/hadoop/utils/HiveAvroSerializer.java | 12 +++--- .../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 12 +++--- .../hudi/hadoop/utils/HoodieInputFormatUtils.java | 18 ++++----- .../utils/HoodieRealtimeInputFormatUtils.java | 10 ++--- .../utils/HoodieRealtimeRecordReaderUtils.java | 8 ++-- .../apache/hudi/hadoop/utils/shims/Hive3Shim.java | 10 ++--- 23 files changed, 137 insertions(+), 179 deletions(-) diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java index a23f94abde92..cdb921642b96 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/BootstrapColumnStichingRecordReader.java @@ -20,12 +20,11 @@ package org.apache.hudi.hadoop; import org.apache.hudi.common.util.ValidationUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.RecordReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.stream.Collectors; @@ -34,10 +33,9 @@ import java.util.stream.IntStream; /** * Stitches 2 record reader returned rows and presents a concatenated view to clients. */ +@Slf4j public class BootstrapColumnStichingRecordReader implements RecordReader<NullWritable, ArrayWritable> { - private static final Logger LOG = LoggerFactory.getLogger(BootstrapColumnStichingRecordReader.class); - private final RecordReader<NullWritable, ArrayWritable> leftColsRecordReader; private final RecordReader<NullWritable, ArrayWritable> rightColsRecordReader; private final int numLeftColumns; @@ -60,7 +58,7 @@ public class BootstrapColumnStichingRecordReader implements RecordReader<NullWri .toArray(new String[0]); values = new ArrayWritable(vals); } - LOG.info("Total ArrayWritable Length :{}", values.get().length); + log.info("Total ArrayWritable Length :{}", values.get().length); } @Override diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java index 26d31d2e74ba..20015e26cc3e 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HiveHoodieTableFileIndex.java @@ -27,8 +27,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.core.read.BaseHoodieTableFileIndex; import org.apache.hudi.storage.StoragePath; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.Map; @@ -37,10 +36,9 @@ import java.util.stream.Collectors; /** * Implementation of {@link BaseHoodieTableFileIndex} for Hive-based query engines */ +@Slf4j public class HiveHoodieTableFileIndex extends BaseHoodieTableFileIndex { - public static final Logger LOG = LoggerFactory.getLogger(HiveHoodieTableFileIndex.class); - public HiveHoodieTableFileIndex(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient, TypedProperties configProperties, diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieColumnProjectionUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieColumnProjectionUtils.java index 719e3aa6b873..3ea77a22bd1b 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieColumnProjectionUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieColumnProjectionUtils.java @@ -25,9 +25,9 @@ import org.apache.hudi.common.schema.HoodieSchemaType; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.io.IOConstants; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -36,8 +36,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -57,9 +55,8 @@ import static org.apache.hadoop.hive.serde.serdeConstants.TIMESTAMP_TYPE_NAME; * Needed to copy as we see NoSuchMethod errors when directly using these APIs with/without Spark. * Some of these methods are not available across hive versions. */ +@Slf4j public class HoodieColumnProjectionUtils { - public static final Logger LOG = LoggerFactory.getLogger(ColumnProjectionUtils.class); - public static final String READ_COLUMN_IDS_CONF_STR = "hive.io.file.readcolumn.ids"; /** * the nested column path is the string from the root to the leaf diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java index 0408500aa71a..43583eeca7c4 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieCopyOnWriteTableInputFormat.java @@ -36,6 +36,7 @@ import org.apache.hudi.hadoop.utils.HoodieHiveUtils; import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -48,8 +49,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapreduce.Job; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; @@ -77,10 +76,9 @@ import static org.apache.hudi.common.config.HoodieMetadataConfig.ENABLE; * * NOTE: This class is invariant of the underlying file-format of the files being read */ +@Slf4j public class HoodieCopyOnWriteTableInputFormat extends HoodieTableInputFormat { - private static final Logger LOG = LoggerFactory.getLogger(HoodieCopyOnWriteTableInputFormat.class); - @Override protected boolean isSplitable(FileSystem fs, Path filename) { return !(filename instanceof PathWithBootstrapFileStatus); @@ -212,7 +210,7 @@ public class HoodieCopyOnWriteTableInputFormat extends HoodieTableInputFormat { private BootstrapBaseFileSplit makeExternalFileSplit(PathWithBootstrapFileStatus file, FileSplit split) { try { - LOG.info("Making external data split for {}", file); + log.info("Making external data split for {}", file); FileStatus externalFileStatus = file.getBootstrapFileStatus(); FileSplit externalFileSplit = makeSplit(externalFileStatus.getPath(), 0, externalFileStatus.getLen(), new String[0], new String[0]); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java index 22d5ba701711..ee2286c14a90 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieFileGroupReaderBasedRecordReader.java @@ -40,6 +40,7 @@ import org.apache.hudi.hadoop.utils.HoodieRealtimeInputFormatUtils; import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -51,8 +52,6 @@ import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Arrays; @@ -86,10 +85,9 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getTableBasePa /** * {@link HoodieFileGroupReader} based implementation of Hive's {@link RecordReader} for {@link ArrayWritable}. */ +@Slf4j public class HoodieFileGroupReaderBasedRecordReader implements RecordReader<NullWritable, ArrayWritable> { - private static final Logger LOG = LoggerFactory.getLogger(HoodieFileGroupReaderBasedRecordReader.class); - public interface HiveReaderCreator { org.apache.hadoop.mapred.RecordReader<NullWritable, ArrayWritable> getRecordReader( final org.apache.hadoop.mapred.InputSplit split, @@ -142,7 +140,7 @@ public class HoodieFileGroupReaderBasedRecordReader implements RecordReader<Null props.setProperty(MERGE_TYPE.key(), REALTIME_PAYLOAD_COMBINE); } } - LOG.debug("Creating HoodieFileGroupReaderRecordReader with tableBasePath={}, latestCommitTime={}, fileSplit={}", tableBasePath, latestCommitTime, fileSplit.getPath()); + log.debug("Creating HoodieFileGroupReaderRecordReader with tableBasePath={}, latestCommitTime={}, fileSplit={}", tableBasePath, latestCommitTime, fileSplit.getPath()); FileSlice fileSlice = getFileSliceFromSplit(fileSplit, getFs(tableBasePath, jobConfCopy), tableBasePath); this.containsBaseFile = fileSlice.getBaseFile().isPresent(); this.recordIterator = HoodieFileGroupReader.<ArrayWritable>builder() diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormat.java index 5c6d7451a9e2..9ee7fac11168 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieParquetInputFormat.java @@ -36,6 +36,7 @@ import org.apache.hudi.hadoop.utils.HoodieRealtimeInputFormatUtils; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -62,8 +63,6 @@ import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.Type; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Constructor; @@ -90,9 +89,9 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.shouldUseFileg */ @UseRecordReaderFromInputFormat @UseFileSplitsFromInputFormat +@Slf4j public class HoodieParquetInputFormat extends HoodieParquetInputFormatBase { - private static final Logger LOG = LoggerFactory.getLogger(HoodieParquetInputFormat.class); // Compiled once: the raw columns.types screen below runs on every legacy-path split. private static final Pattern WHITESPACE = Pattern.compile("\\s"); // A member a synced variant's Hive type carries, as it appears in a struct<...> type string: @@ -181,7 +180,7 @@ public class HoodieParquetInputFormat extends HoodieParquetInputFormatBase { // if (HoodieHiveUtil.INCREMENTAL_SCAN_MODE.equals(mode)) { // FilterPredicate predicate = constructHoodiePredicate(job, tableName, split); - // LOG.info("Setting parquet predicate push down as " + predicate); + // log.info("Setting parquet predicate push down as " + predicate); // ParquetInputFormat.setFilterPredicate(job, predicate); // clearOutExistingPredicate(job); // } @@ -199,7 +198,7 @@ public class HoodieParquetInputFormat extends HoodieParquetInputFormatBase { SchemaEvolutionContext schemaEvolutionContext = new SchemaEvolutionContext(split, job); schemaEvolutionContext.doEvolutionForParquetFormat(); - LOG.debug("EMPLOYING DEFAULT RECORD READER - {}", split); + log.debug("EMPLOYING DEFAULT RECORD READER - {}", split); return getRecordReaderInternal(split, job, reporter, schemaEvolutionContext.internalSchemaOption); } @@ -248,7 +247,7 @@ public class HoodieParquetInputFormat extends HoodieParquetInputFormatBase { List<Pair<String, String>> colNamesWithTypesForExternal = colNameWithTypes.stream() .filter(p -> !HoodieRecord.HOODIE_META_COLUMNS.contains(p.getKey())).collect(Collectors.toList()); - LOG.info("colNameWithTypes ={}, Num Entries ={}", colNameWithTypes, colNameWithTypes.size()); + log.info("colNameWithTypes ={}, Num Entries ={}", colNameWithTypes, colNameWithTypes.size()); Option<FileSplit> singleSplit = resolveSingleFileSplit(eSplit, !hoodieColsProjected.isEmpty(), !externalColsProjected.isEmpty()); @@ -265,7 +264,7 @@ public class HoodieParquetInputFormat extends HoodieParquetInputFormatBase { jobConfCopy.unset(TableScanDesc.FILTER_EXPR_CONF_STR); jobConfCopy.unset(ConvertAstToSearchArg.SARG_PUSHDOWN); - LOG.info("Generating column stitching reader for {} and {}", eSplit.getPath(), rightSplit.getPath()); + log.info("Generating column stitching reader for {} and {}", eSplit.getPath(), rightSplit.getPath()); return new BootstrapColumnStichingRecordReader(getRecordReaderInternal(eSplit, jobConfCopy, reporter), HoodieRecord.HOODIE_META_COLUMNS.size(), getRecordReaderInternal(rightSplit, jobConfCopy, reporter), @@ -378,7 +377,7 @@ public class HoodieParquetInputFormat extends HoodieParquetInputFormatBase { // The screen above strips whitespace and lower-cases; the TypeInfoUtils parse tolerates // neither, so a string it lets through can still fail to parse. Bail out like the pairing // check below rather than failing a read the plain parquet reader would serve. - LOG.debug("Skipping the shredded variant guard for {}: {} did not parse", filePath, IOConstants.COLUMNS_TYPES, e); + log.debug("Skipping the shredded variant guard for {}: {} did not parse", filePath, IOConstants.COLUMNS_TYPES, e); return; } if (ioColumns.size() != ioColumnTypes.size()) { diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java index 0322135ce07a..fb80ac5c21a6 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java @@ -36,12 +36,11 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StorageConfiguration; import org.apache.hudi.storage.StoragePath; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.HashMap; @@ -68,10 +67,10 @@ import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath; * hadoopConf.setClass("mapreduce.input.pathFilter.class", org.apache.hudi.hadoop .HoodieROTablePathFilter.class, * org.apache.hadoop.fs.PathFilter.class) */ +@Slf4j public class HoodieROTablePathFilter implements Configurable, PathFilter, Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(HoodieROTablePathFilter.class); /** * Its quite common, to have all files from a given partition path be passed into accept(), cache the check for hoodie @@ -151,7 +150,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial this.engineContext = new HoodieLocalEngineContext(this.conf); } - LOG.debug("Checking acceptance for path {}", path); + log.debug("Checking acceptance for path {}", path); Path folder = null; try { if (storage == null) { @@ -162,12 +161,12 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial folder = path.getParent(); // get the immediate parent. // Try to use the caches. if (nonHoodiePathCache.contains(folder.toString())) { - LOG.debug("Accepting non-hoodie path from cache: {}", path); + log.debug("Accepting non-hoodie path from cache: {}", path); return true; } if (hoodiePathCache.containsKey(folder.toString())) { - LOG.debug("{} Hoodie path checked against cache, accept => {}", path, hoodiePathCache.get(folder.toString()).contains(path)); + log.debug("{} Hoodie path checked against cache, accept => {}", path, hoodiePathCache.get(folder.toString()).contains(path)); return hoodiePathCache.get(folder.toString()).contains(path); } @@ -175,7 +174,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial String filePath = path.toString(); if (filePath.contains("/" + HoodieTableMetaClient.METAFOLDER_NAME + "/") || filePath.endsWith("/" + HoodieTableMetaClient.METAFOLDER_NAME)) { - LOG.debug("Skipping Hoodie Metadata file {}", filePath); + log.debug("Skipping Hoodie Metadata file {}", filePath); return false; } @@ -193,7 +192,7 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial if (baseDir != null) { // Check whether baseDir in nonHoodiePathCache if (nonHoodiePathCache.contains(baseDir.toString())) { - LOG.debug("Accepting non-hoodie path from cache: {}", path); + log.debug("Accepting non-hoodie path from cache: {}", path); return true; } HoodieTableFileSystemView fsView = null; @@ -233,19 +232,19 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial if (!hoodiePathCache.containsKey(folder.toString())) { hoodiePathCache.put(folder.toString(), new HashSet<>()); } - LOG.info("Based on hoodie metadata from base path: {}, caching {} files under {}", baseDir, latestFiles.size(), folder); + log.info("Based on hoodie metadata from base path: {}, caching {} files under {}", baseDir, latestFiles.size(), folder); for (HoodieBaseFile lfile : latestFiles) { hoodiePathCache.get(folder.toString()).add(new Path(lfile.getPath())); } // accept the path, if its among the latest files. - if (LOG.isDebugEnabled()) { - LOG.debug("{} checked after cache population, accept => {}", path, hoodiePathCache.get(folder.toString()).contains(path)); + if (log.isDebugEnabled()) { + log.debug("{} checked after cache population, accept => {}", path, hoodiePathCache.get(folder.toString()).contains(path)); } return hoodiePathCache.get(folder.toString()).contains(path); } catch (TableNotFoundException e) { // Non-hoodie path, accept it. - LOG.debug("(1) Caching non-hoodie path under {} with basePath {}", folder, baseDir); + log.debug("(1) Caching non-hoodie path under {} with basePath {}", folder, baseDir); nonHoodiePathCache.add(folder.toString()); nonHoodiePathCache.add(baseDir.toString()); return true; @@ -256,13 +255,13 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial } } else { // files is at < 3 level depth in FS tree, can't be hoodie dataset - LOG.debug("(2) Caching non-hoodie path under{}", folder); + log.debug("(2) Caching non-hoodie path under{}", folder); nonHoodiePathCache.add(folder.toString()); return true; } } catch (Exception e) { String msg = "Error checking path :" + path + ", under folder: " + folder; - LOG.error(msg, e); + log.error(msg, e); throw new HoodieException(msg, e); } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java index 09f6f4bbabdb..db7deecebdb8 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/InputPathHandler.java @@ -24,10 +24,9 @@ import org.apache.hudi.exception.InvalidTableException; import org.apache.hudi.exception.TableNotFoundException; import org.apache.hudi.hadoop.utils.HoodieHiveUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -50,10 +49,9 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.getTableMetaCl * This class is written in such a way that it can handle multiple tables and properties unique to * a table but for table level property such check is not required. */ +@Slf4j public class InputPathHandler { - public static final Logger LOG = LoggerFactory.getLogger(InputPathHandler.class); - private final Configuration conf; // tableName to metadata mapping for all Hoodie tables(both incremental & snapshot) private final Map<String, HoodieTableMetaClient> tableMetaClientMap; @@ -114,7 +112,7 @@ public class InputPathHandler { tagAsIncrementalOrSnapshot(inputPath, metaClient, incrementalTables); } catch (TableNotFoundException | InvalidTableException e) { // This is a non Hoodie inputPath - LOG.info("Handling a non-hoodie path {}", inputPath); + log.info("Handling a non-hoodie path {}", inputPath); nonHoodieInputPaths.add(inputPath); } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java index c08c358c0c87..a21798547eb8 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java @@ -21,9 +21,8 @@ package org.apache.hudi.hadoop; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.exception.HoodieException; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.mapred.RecordReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.NoSuchElementException; @@ -34,10 +33,9 @@ import java.util.NoSuchElementException; * @param <K> Key Type * @param <V> Value Type */ +@Slf4j public class RecordReaderValueIterator<K, V> implements ClosableIterator<V> { - private static final Logger LOG = LoggerFactory.getLogger(RecordReaderValueIterator.class); - private final RecordReader<K, V> reader; private V nextVal = null; @@ -62,7 +60,7 @@ public class RecordReaderValueIterator<K, V> implements ClosableIterator<V> { } this.nextVal = val; } catch (IOException e) { - LOG.error("Got error reading next record from record reader"); + log.error("Got error reading next record from record reader"); throw new HoodieException(e); } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SchemaEvolutionContext.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SchemaEvolutionContext.java index d3a4532ba3c0..49413dc1b85b 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SchemaEvolutionContext.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SchemaEvolutionContext.java @@ -45,6 +45,7 @@ import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -64,8 +65,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; @@ -85,10 +84,9 @@ import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath; * If field is renamed in queried schema, its old name will be returned, which is relevant at the provided time. * If type of field is changed, its old type will be returned, and projection will be created that will convert the old type to the queried one. */ +@Slf4j public class SchemaEvolutionContext { - private static final Logger LOG = LoggerFactory.getLogger(SchemaEvolutionContext.class); - private static final String HIVE_TMP_READ_COLUMN_NAMES_CONF_STR = "hive.tmp.io.file.readcolumn.ids"; private static final String HIVE_TMP_COLUMNS = "hive.tmp.columns"; private static final String HIVE_EVOLUTION_ENABLE = "hudi.hive.schema.evolution"; @@ -106,7 +104,7 @@ public class SchemaEvolutionContext { this.split = split; this.job = job; if (!job.getBoolean(HIVE_EVOLUTION_ENABLE, true)) { - LOG.info("Schema evolution is disabled for split: {}", split); + log.info("Schema evolution is disabled for split: {}", split); internalSchemaOption = Option.empty(); this.metaClient = null; return; @@ -158,7 +156,7 @@ public class SchemaEvolutionContext { try { return parser.apply(cachedJson); } catch (Exception e) { - LOG.warn("Failed to parse data from cache with key: {}", cacheKey, e); + log.warn("Failed to parse data from cache with key: {}", cacheKey, e); return Option.empty(); } } @@ -184,7 +182,7 @@ public class SchemaEvolutionContext { return HoodieTableMetaClient.builder().setBasePath(tablePath.get().toString()) .setConf(HadoopFSUtils.getStorageConfWithCopy(job)).build(); } catch (Exception e) { - LOG.error("Not a valid hoodie table, table path: {}", ((FileSplit) split).getPath(), e); + log.error("Not a valid hoodie table, table path: {}", ((FileSplit) split).getPath(), e); return null; } } @@ -197,7 +195,7 @@ public class SchemaEvolutionContext { */ public void doEvolutionForRealtimeInputFormat(AbstractRealtimeRecordReader realtimeRecordReader) throws Exception { if (!(split instanceof RealtimeSplit)) { - LOG.warn("Expected realtime split for mor table. Found split: {}", split); + log.warn("Expected realtime split for mor table. Found split: {}", split); return; } if (internalSchemaOption.isPresent()) { @@ -221,7 +219,7 @@ public class SchemaEvolutionContext { realtimeRecordReader.setHiveSchema(hiveSchema); internalSchemaOption = Option.of(prunedInternalSchema); RealtimeSplit realtimeSplit = (RealtimeSplit) split; - LOG.info("About to read compacted logs {} for base split {}, projecting cols {}", + log.info("About to read compacted logs {} for base split {}, projecting cols {}", realtimeSplit.getDeltaLogPaths(), realtimeSplit.getPath(), requiredColumns); } } @@ -359,7 +357,7 @@ public class SchemaEvolutionContext { case TIME_MILLIS: throw new UnsupportedOperationException(String.format("cannot convert %s type to hive", type)); default: - LOG.error("cannot convert unknown type: {} to Hive", type); + log.error("cannot convert unknown type: {} to Hive", type); throw new UnsupportedOperationException(String.format("cannot convert unknown type: %s to Hive", type)); } } @@ -398,8 +396,8 @@ public class SchemaEvolutionContext { } String filterText = filterExpr.getExprString(); String serializedFilterExpr = SerializationUtilities.serializeExpression(filterExpr); - if (LOG.isDebugEnabled()) { - LOG.debug("Pushdown initiated with filterText = {}, filterExpr = {}, serializedFilterExpr = {}", + if (log.isDebugEnabled()) { + log.debug("Pushdown initiated with filterText = {}, filterExpr = {}, serializedFilterExpr = {}", filterText, filterExpr, serializedFilterExpr); } job.set(TableScanDesc.FILTER_TEXT_CONF_STR, filterText); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java index d9cc9e9a9912..83db1076992d 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java @@ -37,6 +37,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -76,8 +77,6 @@ import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.lib.CombineFileInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; import org.apache.hadoop.mapreduce.JobContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.DataInput; import java.io.DataOutput; @@ -111,11 +110,11 @@ import java.util.concurrent.Future; * <p> * NOTE : This class is implemented to work with Hive 2.x + */ +@Slf4j public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extends Writable> extends HiveInputFormat<K, V> { private static final String CLASS_NAME = HoodieCombineHiveInputFormat.class.getName(); - public static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME); // max number of threads we can use to check non-combinable paths private static final int MAX_CHECK_NONCOMBINABLE_THREAD_NUM = 50; @@ -174,19 +173,19 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend Class<?> inputFormatClass = part.getInputFileFormatClass(); String inputFormatClassName = inputFormatClass.getName(); InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); - LOG.info("Input Format => {}", inputFormatClass.getName()); + log.info("Input Format => {}", inputFormatClass.getName()); // **MOD** Set the hoodie filter in the combine if (inputFormatClass.getName().equals(getParquetInputFormatClassName())) { combine.setHoodieFilter(true); } else if (inputFormatClass.getName().equals(getParquetRealtimeInputFormatClassName())) { - LOG.info("Setting hoodie filter and realtime input format"); + log.info("Setting hoodie filter and realtime input format"); combine.setHoodieFilter(true); combine.setRealTime(true); if (job.get(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "").isEmpty()) { List<String> partitions = new ArrayList<>(part.getPartSpec().keySet()); if (!partitions.isEmpty()) { String partitionStr = String.join("/", partitions); - LOG.info("Setting Partitions in jobConf - Partition Keys for Path : {} is :{}", path, partitionStr); + log.info("Setting Partitions in jobConf - Partition Keys for Path : {} is :{}", path, partitionStr); job.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, partitionStr); } else { job.set(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, ""); @@ -198,7 +197,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend deserializerClassName = part.getDeserializer(job).getClass().getName(); } catch (Exception e) { // ignore - LOG.error("Getting deserializer class name error ", e); + log.error("Getting deserializer class name error ", e); } // don't combine if inputformat is a SymlinkTextInputFormat @@ -224,11 +223,11 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend f = poolMap.get(combinePathInputFormat); if (f == null) { f = new CombineFilter(filterPath); - LOG.info("CombineHiveInputSplit creating pool for {}; using filter path {}", path, filterPath); + log.info("CombineHiveInputSplit creating pool for {}; using filter path {}", path, filterPath); combine.createPool(job, f); poolMap.put(combinePathInputFormat, f); } else { - LOG.info("CombineHiveInputSplit: pool is already created for {}; using filter path {}", path, filterPath); + log.info("CombineHiveInputSplit: pool is already created for {}; using filter path {}", path, filterPath); f.addPath(filterPath); } } else { @@ -286,7 +285,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend result.add(csplit); } - LOG.info("number of splits {}", result.size()); + log.info("number of splits {}", result.size()); return result.toArray(new CombineHiveInputSplit[result.size()]); } @@ -295,7 +294,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend */ public Set<Integer> getNonCombinablePathIndices(JobConf job, Path[] paths, int numThreads) throws ExecutionException, InterruptedException { - LOG.info("Total number of paths: {}, launching {} threads to check non-combinable ones.", paths.length, numThreads); + log.info("Total number of paths: {}, launching {} threads to check non-combinable ones.", paths.length, numThreads); int numPathPerThread = (int) Math.ceil((double) paths.length / numThreads); ExecutorService executor = Executors.newFixedThreadPool(numThreads); @@ -350,7 +349,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend } } } catch (Exception e) { - LOG.error("Error checking non-combinable path", e); + log.error("Error checking non-combinable path", e); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); throw new IOException(e); } @@ -358,7 +357,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend // Store the previous value for the path specification String oldPaths = job.get(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR); - LOG.debug("The received input paths are: [{}] against the property {}", oldPaths, + log.debug("The received input paths are: [{}] against the property {}", oldPaths, org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR); // Process the normal splits @@ -409,7 +408,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend String avroSchema = schemaUtil.getTableSchema().toString(); Option<InternalSchema> internalSchema = schemaUtil.getTableInternalSchemaFromCommitMetadata(); if (internalSchema.isPresent()) { - LOG.info("Set internal and avro schema cache with path: {}", path); + log.info("Set internal and avro schema cache with path: {}", path); job.set(SCHEMA_CACHE_KEY_PREFIX + "." + path, avroSchema); job.set(INTERNAL_SCHEMA_CACHE_KEY_PREFIX + "." + path, SerDeHelper.toJson(internalSchema.get())); } else { @@ -419,11 +418,11 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend } } } catch (Exception e) { - LOG.warn("Failed to set schema cache", e); + log.warn("Failed to set schema cache", e); } } - LOG.info("Number of all splits {}", result.size()); + log.info("Number of all splits {}", result.size()); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); return result.toArray(new InputSplit[result.size()]); } finally { @@ -558,7 +557,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend retLists.add(split); long splitgLength = split.getLength(); if (size + splitgLength >= targetSize) { - LOG.info("Sample alias {} using {}splits", entry.getValue(), (i + 1)); + log.info("Sample alias {} using {}splits", entry.getValue(), (i + 1)); if (size + splitgLength > targetSize) { ((InputSplitShim) split).shrinkSplit(targetSize - size); } @@ -926,15 +925,15 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend @Override protected List<FileStatus> listStatus(JobContext job) throws IOException { - LOG.info("Listing status in HoodieCombineHiveInputFormat.HoodieCombineFileInputFormatShim"); + log.info("Listing status in HoodieCombineHiveInputFormat.HoodieCombineFileInputFormatShim"); List<FileStatus> result; if (hoodieFilter) { HoodieParquetInputFormatBase input; if (isRealTime) { - LOG.info("Using HoodieRealtimeInputFormat"); + log.info("Using HoodieRealtimeInputFormat"); input = createParquetRealtimeInputFormat(); } else { - LOG.info("Using HoodieInputFormat"); + log.info("Using HoodieInputFormat"); input = createParquetInputFormat(); } input.setConf(job.getConfiguration()); @@ -962,7 +961,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend if (job.getLong(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.SPLIT_MAXSIZE, 0L) == 0L) { super.setMaxSplitSize(minSize); } - LOG.info("mapreduce.input.fileinputformat.split.minsize={}, mapreduce.input.fileinputformat.split.maxsize={}", minSize, maxSize); + log.info("mapreduce.input.fileinputformat.split.minsize={}, mapreduce.input.fileinputformat.split.maxsize={}", minSize, maxSize); if (isRealTime) { job.set("hudi.hive.realtime", "true"); @@ -1063,7 +1062,7 @@ public class HoodieCombineHiveInputFormat<K extends WritableComparable, V extend InputFormat<WritableComparable, Writable> inputFormat = getInputFormatFromCache(inputFormatClass, conf); if (inputFormat instanceof AvoidSplitCombination && ((AvoidSplitCombination) inputFormat).shouldSkipCombine(paths[i + start], conf)) { - LOG.debug("The path [{}] is being parked for HiveInputFormat.getSplits", paths[i + start]); + log.debug("The path [{}] is being parked for HiveInputFormat.getSplits", paths[i + start]); nonCombinablePathIndices.add(i + start); } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java index 1f01f6ccdce2..1fa8f394308f 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/AbstractRealtimeRecordReader.java @@ -38,6 +38,7 @@ import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.io.parquet.serde.ArrayWritableObjectInspector; import org.apache.hadoop.hive.serde.serdeConstants; @@ -47,8 +48,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.mapred.JobConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -63,9 +62,8 @@ import static org.apache.hudi.common.util.StringUtils.EMPTY_STRING; /** * Record Reader implementation to merge fresh avro data with base parquet data, to support real time queries. */ +@Slf4j public abstract class AbstractRealtimeRecordReader { - private static final Logger LOG = LoggerFactory.getLogger(AbstractRealtimeRecordReader.class); - @Getter protected final RealtimeSplit split; @Getter @@ -94,9 +92,9 @@ public abstract class AbstractRealtimeRecordReader { public AbstractRealtimeRecordReader(RealtimeSplit split, JobConf job) { this.split = split; this.jobConf = job; - LOG.info("cfg ==> {}", job.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR)); - LOG.info("columnIds ==> {}", job.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); - LOG.info("partitioningColumns ==> {}", job.get(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "")); + log.info("cfg ==> {}", job.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR)); + log.info("columnIds ==> {}", job.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + log.info("partitioningColumns ==> {}", job.get(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, "")); this.supportPayload = Boolean.parseBoolean(job.get("hoodie.support.payload", "true")); try { metaClient = HoodieTableMetaClient.builder() @@ -106,7 +104,7 @@ public abstract class AbstractRealtimeRecordReader { this.payloadProps.setProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY, metaClient.getTableConfig().getOrderingFieldsStr().orElse(null)); } this.usesCustomPayload = usesCustomPayload(metaClient); - LOG.info("usesCustomPayload ==> {}", this.usesCustomPayload); + log.info("usesCustomPayload ==> {}", this.usesCustomPayload); // get timestamp columns supportTimestamp = HoodieColumnProjectionUtils.supportTimestamp(jobConf); @@ -147,7 +145,7 @@ public abstract class AbstractRealtimeRecordReader { if (writerSchemaColNames.contains(lastColName)) { break; } - LOG.debug("remove virtual column: {}", lastColName); + log.debug("remove virtual column: {}", lastColName); columnNameList.remove(columnNameList.size() - 1); columnTypeList.remove(columnTypeList.size() - 1); } @@ -155,7 +153,7 @@ public abstract class AbstractRealtimeRecordReader { this.serializer = new HiveAvroSerializer(new ArrayWritableObjectInspector(rowTypeInfo), columnNameList, columnTypeList); } catch (Exception e) { // fallback to origin logical - LOG.warn("Failed to init HiveAvroSerializer to support payload merge. Fallback to origin logical", e); + log.warn("Failed to init HiveAvroSerializer to support payload merge. Fallback to origin logical", e); this.supportPayload = false; } } @@ -166,7 +164,7 @@ public abstract class AbstractRealtimeRecordReader { * job conf. */ private void init() throws Exception { - LOG.info("Getting writer schema from table avro schema "); + log.info("Getting writer schema from table avro schema "); writerSchema = new TableSchemaResolver(metaClient).getTableSchema(); // Add partitioning fields to writer schema for resulting row to contain null values for these fields @@ -184,13 +182,13 @@ public abstract class AbstractRealtimeRecordReader { // to null out fields not present before readerSchema = HoodieRealtimeRecordReaderUtils.generateProjectionSchema(writerSchema, schemaFieldsMap, projectionFields); - LOG.info(String.format("About to read compacted logs %s for base split %s, projecting cols %s", + log.info(String.format("About to read compacted logs %s for base split %s, projecting cols %s", split.getDeltaLogPaths(), split.getPath(), projectionFields)); } public HoodieSchema constructHiveOrderedSchema(HoodieSchema writerSchema, Map<String, HoodieSchemaField> schemaFieldsMap, String hiveColumnString) { String[] hiveColumns = hiveColumnString.isEmpty() ? new String[0] : hiveColumnString.split(","); - LOG.info("Hive Columns : {}", hiveColumnString); + log.info("Hive Columns : {}", hiveColumnString); List<HoodieSchemaField> hiveSchemaFields = new ArrayList<>(); for (String columnName : hiveColumns) { @@ -201,13 +199,13 @@ public abstract class AbstractRealtimeRecordReader { } else { // Hive has some extra virtual columns like BLOCK__OFFSET__INSIDE__FILE which do not exist in table schema. // They will get skipped as they won't be found in the original schema. - LOG.debug("Skipping Hive Column => {}", columnName); + log.debug("Skipping Hive Column => {}", columnName); } } HoodieSchema hiveSchema = HoodieSchema.createRecord(writerSchema.getName(), writerSchema.getDoc().orElse(null), writerSchema.getNamespace().orElse(null), writerSchema.isError(), hiveSchemaFields); - LOG.debug("HIVE Schema is :{}", hiveSchema); + log.debug("HIVE Schema is :{}", hiveSchema); return hiveSchema; } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieCombineRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieCombineRealtimeRecordReader.java index 90cd422620d2..7eb54d67ae63 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieCombineRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieCombineRealtimeRecordReader.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.hadoop.HoodieFileGroupReaderBasedRecordReader; import org.apache.hudi.hadoop.hive.HoodieCombineRealtimeFileSplit; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.IOContextMap; import org.apache.hadoop.io.ArrayWritable; @@ -30,8 +31,6 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.lib.CombineFileSplit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.LinkedList; @@ -42,9 +41,9 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.shouldUseFileg /** * Allows to read multiple realtime file splits grouped together by CombineInputFormat. */ +@Slf4j public class HoodieCombineRealtimeRecordReader implements RecordReader<NullWritable, ArrayWritable> { - private static final Logger LOG = LoggerFactory.getLogger(HoodieCombineRealtimeRecordReader.class); // RecordReaders for each split private final List<RecordReader> recordReaders = new LinkedList<>(); // Points to the currently iterating record reader @@ -60,12 +59,12 @@ public class HoodieCombineRealtimeRecordReader implements RecordReader<NullWrita .size(), "Num Splits does not match number of unique RecordReaders!"); for (InputSplit rtSplit : ((HoodieCombineRealtimeFileSplit) split).getRealtimeFileSplits()) { if (useFileGroupReader) { - LOG.info("Creating new HoodieFileGroupReaderRecordReader for split"); + log.info("Creating new HoodieFileGroupReaderRecordReader for split"); RecordReader reader = readers.remove(0); ValidationUtils.checkArgument(reader instanceof HoodieFileGroupReaderBasedRecordReader, reader.toString() + "not instance of HoodieFileGroupReaderRecordReader "); recordReaders.add(reader); } else { - LOG.info("Creating new RealtimeRecordReader for split"); + log.info("Creating new RealtimeRecordReader for split"); recordReaders.add( new HoodieRealtimeRecordReader((HoodieRealtimeFileSplit) rtSplit, jobConf, readers.remove(0))); } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java index ef12dee0ad06..012b20c64728 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieHFileRealtimeInputFormat.java @@ -26,6 +26,7 @@ import org.apache.hudi.hadoop.UseRecordReaderFromInputFormat; import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import org.apache.hudi.hadoop.utils.HoodieRealtimeInputFormatUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; @@ -35,8 +36,6 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; @@ -45,10 +44,9 @@ import java.io.IOException; */ @UseRecordReaderFromInputFormat @UseFileSplitsFromInputFormat +@Slf4j public class HoodieHFileRealtimeInputFormat extends HoodieMergeOnReadTableInputFormat { - private static final Logger LOG = LoggerFactory.getLogger(HoodieHFileRealtimeInputFormat.class); - // NOTE: We're only using {@code HoodieHFileInputFormat} to compose {@code RecordReader} private final HoodieHFileInputFormat hFileInputFormat = new HoodieHFileInputFormat(); @@ -62,7 +60,7 @@ public class HoodieHFileRealtimeInputFormat extends HoodieMergeOnReadTableInputF // actual heavy lifting of reading the parquet files happen. if (jobConf.get(HoodieInputFormatUtils.HOODIE_READ_COLUMNS_PROP) == null) { synchronized (jobConf) { - LOG.info("Before adding Hoodie columns, Projections :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + log.info("Before adding Hoodie columns, Projections :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); if (jobConf.get(HoodieInputFormatUtils.HOODIE_READ_COLUMNS_PROP) == null) { // Hive (across all versions) fails for queries like select count(`_hoodie_commit_time`) from table; // In this case, the projection fields gets removed. Looking at HiveInputFormat implementation, in some cases @@ -80,7 +78,7 @@ public class HoodieHFileRealtimeInputFormat extends HoodieMergeOnReadTableInputF } HoodieRealtimeInputFormatUtils.cleanProjectionColumnIds(jobConf); - LOG.info("Creating record reader with readCols :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + log.info("Creating record reader with readCols :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); // sanity check ValidationUtils.checkArgument(split instanceof HoodieRealtimeFileSplit, "HoodieRealtimeRecordReader can only work on HoodieRealtimeFileSplit and not with " + split); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java index 48e78f147644..9fc94cd43c02 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java @@ -28,6 +28,7 @@ import org.apache.hudi.hadoop.UseRecordReaderFromInputFormat; import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import org.apache.hudi.hadoop.utils.HoodieRealtimeInputFormatUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.NullWritable; @@ -35,8 +36,6 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -52,10 +51,9 @@ import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.shouldUseFileg */ @UseRecordReaderFromInputFormat @UseFileSplitsFromInputFormat +@Slf4j public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat { - private static final Logger LOG = LoggerFactory.getLogger(HoodieParquetRealtimeInputFormat.class); - public HoodieParquetRealtimeInputFormat() { super(new HoodieMergeOnReadTableInputFormat()); } @@ -80,7 +78,7 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat { HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(getStorageConf(jobConf)).setBasePath(realtimeSplit.getBasePath()).build(); HoodieTableConfig tableConfig = metaClient.getTableConfig(); addProjectionToJobConf(realtimeSplit, jobConf, tableConfig); - LOG.info("Creating record reader with readCols :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + log.info("Creating record reader with readCols :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); // for log only split, set the parquet reader as empty. if (isLogFile(realtimeSplit.getPath())) { @@ -99,7 +97,7 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat { // actual heavy lifting of reading the parquet files happen. if (HoodieRealtimeInputFormatUtils.canAddProjectionToJobConf(realtimeSplit, jobConf)) { synchronized (jobConf) { - LOG.info("Before adding Hoodie columns, Projections :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); + log.info("Before adding Hoodie columns, Projections :{}, Ids :{}", jobConf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), jobConf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); if (HoodieRealtimeInputFormatUtils.canAddProjectionToJobConf(realtimeSplit, jobConf)) { // Hive (across all versions) fails for queries like select count(`_hoodie_commit_time`) from table; // In this case, the projection fields gets removed. Looking at HiveInputFormat implementation, in some cases diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java index eea8200d7fe0..d898ed0e2ee6 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReader.java @@ -20,12 +20,11 @@ package org.apache.hudi.hadoop.realtime; import org.apache.hudi.exception.HoodieException; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; @@ -33,13 +32,13 @@ import java.io.IOException; * Realtime Record Reader which can do compacted (merge-on-read) record reading or unmerged reading (parquet and log * files read in parallel) based on job configuration. */ +@Slf4j public class HoodieRealtimeRecordReader implements RecordReader<NullWritable, ArrayWritable> { // Property to enable parallel reading of parquet and log files without merging. public static final String REALTIME_SKIP_MERGE_PROP = "hoodie.realtime.merge.skip"; // By default, we do merged-reading public static final String DEFAULT_REALTIME_SKIP_MERGE = "false"; - private static final Logger LOG = LoggerFactory.getLogger(HoodieRealtimeRecordReader.class); private final RecordReader<NullWritable, ArrayWritable> reader; public HoodieRealtimeRecordReader(RealtimeSplit split, JobConf job, @@ -63,19 +62,19 @@ public class HoodieRealtimeRecordReader implements RecordReader<NullWritable, Ar JobConf jobConf, RecordReader<NullWritable, ArrayWritable> realReader) { try { if (canSkipMerging(jobConf)) { - LOG.info("Enabling un-merged reading of realtime records"); + log.info("Enabling un-merged reading of realtime records"); return new RealtimeUnmergedRecordReader(split, jobConf, realReader); } - LOG.info("Enabling merged reading of realtime records for split {}", split); + log.info("Enabling merged reading of realtime records for split {}", split); return new RealtimeCompactedRecordReader(split, jobConf, realReader); } catch (Exception e) { - LOG.error("Got exception when constructing record reader", e); + log.error("Got exception when constructing record reader", e); try { if (null != realReader) { realReader.close(); } } catch (IOException ioe) { - LOG.error("Unable to close real reader", ioe); + log.error("Unable to close real reader", ioe); } throw new HoodieException("Exception when constructing record reader ", e); } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java index 6f315aeb5ab1..dced64dc3b84 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java @@ -43,6 +43,7 @@ import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; import org.apache.hudi.io.util.FileIOUtils; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.io.ArrayWritable; @@ -50,8 +51,6 @@ import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashSet; @@ -59,11 +58,10 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +@Slf4j(topic = "org.apache.hudi.hadoop.realtime.AbstractRealtimeRecordReader") public class RealtimeCompactedRecordReader extends AbstractRealtimeRecordReader implements RecordReader<NullWritable, ArrayWritable> { - private static final Logger LOG = LoggerFactory.getLogger(AbstractRealtimeRecordReader.class); - protected final RecordReader<NullWritable, ArrayWritable> parquetReader; private final Map<String, HoodieRecord> deltaRecordMap; @@ -174,8 +172,8 @@ public class RealtimeCompactedRecordReader extends AbstractRealtimeRecordReader // schema, we use writerSchema to create the arrayWritable from the latest generic record ArrayWritable aWritable = (ArrayWritable) HoodieRealtimeRecordReaderUtils.avroToArrayWritable(recordToReturn, getHiveSchema().toAvroSchema(), isSupportTimestamp()); Writable[] replaceValue = aWritable.get(); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("key %s, base values: %s, log values: %s", key, HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable), + if (log.isDebugEnabled()) { + log.debug(String.format("key %s, base values: %s, log values: %s", key, HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable), HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable))); } Writable[] originalValue = arrayWritable.get(); @@ -186,9 +184,9 @@ public class RealtimeCompactedRecordReader extends AbstractRealtimeRecordReader Math.min(originalValue.length, replaceValue.length)); arrayWritable.set(originalValue); } catch (RuntimeException re) { - LOG.error("Got exception when doing array copy", re); - LOG.error("Base record :{}", HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable)); - LOG.error("Log record :{}", HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable)); + log.error("Got exception when doing array copy", re); + log.error("Base record :{}", HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable)); + log.error("Log record :{}", HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable)); String errMsg = "Base-record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(arrayWritable) + " ,Log-record :" + HoodieRealtimeRecordReaderUtils.arrayWritableToString(aWritable) + " ,Error :" + re.getMessage(); throw new RuntimeException(errMsg, re); diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java index 760666feb83f..b2921d883e25 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.exception.HoodieAvroSchemaException; import org.apache.hudi.exception.HoodieException; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericEnumSymbol; @@ -55,8 +56,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.ArrayWritable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.math.BigDecimal; import java.util.ArrayList; @@ -71,6 +70,7 @@ import static org.apache.hudi.common.schema.HoodieSchemaUtils.isMetadataField; /** * Helper class to serialize hive writable type to avro record. */ +@Slf4j public class HiveAvroSerializer { private final List<String> columnNames; @@ -78,8 +78,6 @@ public class HiveAvroSerializer { private final ArrayWritableObjectInspector objectInspector; private final HoodieSchema recordSchema; - private static final Logger LOG = LoggerFactory.getLogger(HiveAvroSerializer.class); - public HiveAvroSerializer(HoodieSchema schema) { if (schema.getNonNullType().getType() != HoodieSchemaType.RECORD) { throw new IllegalArgumentException("Expected record schema, but got: " + schema); @@ -257,9 +255,9 @@ public class HiveAvroSerializer { setUpRecordFieldFromWritable(columnTypes.get(i), structFieldsDataAsList.get(i), allStructFieldRefs.get(i).getFieldObjectInspector(), record, field); } catch (Exception e) { - LOG.error(String.format("current columnNames: %s", columnNames.stream().collect(Collectors.joining(",")))); - LOG.error(String.format("current type: %s", columnTypes.stream().map(f -> f.getTypeName()).collect(Collectors.joining(",")))); - LOG.error(String.format("current value: %s", HoodieRealtimeRecordReaderUtils.arrayWritableToString((ArrayWritable) o))); + log.error(String.format("current columnNames: %s", columnNames.stream().collect(Collectors.joining(",")))); + log.error(String.format("current type: %s", columnTypes.stream().map(f -> f.getTypeName()).collect(Collectors.joining(",")))); + log.error(String.format("current value: %s", HoodieRealtimeRecordReaderUtils.arrayWritableToString((ArrayWritable) o))); throw e; } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java index 385df53f28b8..5ced7350ab7b 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java @@ -23,13 +23,12 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.hadoop.utils.shims.HiveShim; import org.apache.hudi.hadoop.utils.shims.HiveShims; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hive.common.util.HiveVersionInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -40,10 +39,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +@Slf4j public class HoodieHiveUtils { - public static final Logger LOG = LoggerFactory.getLogger(HoodieHiveUtils.class); - public static final String HOODIE_INCREMENTAL_USE_DATABASE = "hoodie.incremental.use.database"; public static final String HOODIE_CONSUME_MODE_PATTERN = "hoodie.%s.consume.mode"; public static final String HOODIE_START_COMMIT_PATTERN = "hoodie.%s.consume.start.timestamp"; @@ -93,7 +91,7 @@ public class HoodieHiveUtils { public static boolean stopAtCompaction(JobContext job, String tableName) { String compactionPropName = String.format(HOODIE_STOP_AT_COMPACTION_PATTERN, tableName); boolean stopAtCompaction = job.getConfiguration().getBoolean(compactionPropName, true); - LOG.info("Read stop at compaction - {}", stopAtCompaction); + log.info("Read stop at compaction - {}", stopAtCompaction); return stopAtCompaction; } @@ -103,13 +101,13 @@ public class HoodieHiveUtils { if (maxCommits == MAX_COMMIT_ALL) { maxCommits = Integer.MAX_VALUE; } - LOG.info("Read max commits - {}", maxCommits); + log.info("Read max commits - {}", maxCommits); return maxCommits; } public static String readStartCommitTime(JobContext job, String tableName) { String startCommitTimestampName = String.format(HOODIE_START_COMMIT_PATTERN, tableName); - LOG.info("Read start commit time - {}", job.getConfiguration().get(startCommitTimestampName)); + log.info("Read start commit time - {}", job.getConfiguration().get(startCommitTimestampName)); return job.getConfiguration().get(startCommitTimestampName); } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java index 3d9265bbf8bc..6d8fbcf85552 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java @@ -56,6 +56,7 @@ import org.apache.hudi.storage.HoodieStorage; import org.apache.hudi.storage.StoragePath; import org.apache.hudi.storage.StoragePathInfo; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocatedFileStatus; @@ -72,8 +73,6 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; @@ -94,6 +93,7 @@ import static org.apache.hudi.common.config.HoodieMetadataConfig.ENABLE; import static org.apache.hudi.common.table.timeline.TimelineUtils.handleHollowCommitIfNeeded; import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath; +@Slf4j public class HoodieInputFormatUtils { // These positions have to be deterministic across all tables @@ -102,8 +102,6 @@ public class HoodieInputFormatUtils { public static final int HOODIE_PARTITION_PATH_COL_POS = 3; public static final String HOODIE_READ_COLUMNS_PROP = "hoodie.read.columns.set"; - private static final Logger LOG = LoggerFactory.getLogger(HoodieInputFormatUtils.class); - public static FileInputFormat getInputFormat(HoodieFileFormat baseFileFormat, boolean realtime, Configuration conf) { switch (baseFileFormat) { case PARQUET: @@ -263,7 +261,7 @@ public class HoodieInputFormatUtils { .findInstantsBefore(pendingCompactionInstant.get().requestedTime()); int numCommitsFilteredByCompaction = commitsAndCompactionTimeline.getCommitsTimeline().countInstants() - instantsTimeline.getCommitsTimeline().countInstants(); - LOG.info("Earliest pending compaction instant is: {} skipping {} commits", + log.info("Earliest pending compaction instant is: {} skipping {} commits", pendingCompactionInstant.get().requestedTime(), numCommitsFilteredByCompaction); return instantsTimeline; @@ -375,7 +373,7 @@ public class HoodieInputFormatUtils { String lastIncrementalTs = HoodieHiveUtils.readStartCommitTime(job, tableName); // Total number of commits to return in this batch. Set this to -1 to get all the commits. Integer maxCommits = HoodieHiveUtils.readMaxCommits(job, tableName); - LOG.info("Last Incremental timestamp was set as {}", lastIncrementalTs); + log.info("Last Incremental timestamp was set as {}", lastIncrementalTs); return timeline.findInstantsAfter(lastIncrementalTs, maxCommits); } @@ -415,7 +413,7 @@ public class HoodieInputFormatUtils { if (!baseDir.isPresent()) { throw new TableNotFoundException(partitionPath.toString()); } - LOG.info("Reading hoodie metadata from path {}", baseDir.get()); + log.info("Reading hoodie metadata from path {}", baseDir.get()); return HoodieTableMetaClient.builder() .setConf(storage.getConf().newInstance()).setBasePath(baseDir.get().toString()).build(); } @@ -456,11 +454,11 @@ public class HoodieInputFormatUtils { List<HoodieBaseFile> filteredFiles = roView.getLatestBaseFilesInRange(commitsList).collect(Collectors.toList()); List<FileStatus> returns = new ArrayList<>(); for (HoodieBaseFile filteredFile : filteredFiles) { - LOG.debug("Processing incremental hoodie file - {}", filteredFile.getPath()); + log.debug("Processing incremental hoodie file - {}", filteredFile.getPath()); filteredFile = refreshFileStatus(job.getConfiguration(), filteredFile); returns.add(getFileStatus(filteredFile)); } - LOG.info("Total paths to process after hoodie incremental filter {}", filteredFiles.size()); + log.info("Total paths to process after hoodie incremental filter {}", filteredFiles.size()); return returns; } @@ -503,7 +501,7 @@ public class HoodieInputFormatUtils { try { if (baseFile.getFileSize() == 0) { HoodieStorage storage = HoodieStorageUtils.getStorage(dataPath, HadoopFSUtils.getStorageConf(conf)); - LOG.info("Refreshing file status {}", baseFile.getPath()); + log.info("Refreshing file status {}", baseFile.getPath()); return new HoodieBaseFile(storage.getPathInfo(dataPath), baseFile.getBootstrapBaseFile().orElse(null)); } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java index 791aba0d0400..40cdb1922bbc 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeInputFormatUtils.java @@ -25,13 +25,12 @@ import org.apache.hudi.hadoop.realtime.HoodieRealtimeFileSplit; import org.apache.hudi.hadoop.realtime.HoodieVirtualKeyInfo; import org.apache.hudi.hadoop.realtime.RealtimeSplit; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.JobConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; @@ -39,10 +38,9 @@ import java.util.stream.Collectors; import static org.apache.hudi.common.util.TypeUtils.unsafeCast; +@Slf4j public class HoodieRealtimeInputFormatUtils extends HoodieInputFormatUtils { - private static final Logger LOG = LoggerFactory.getLogger(HoodieRealtimeInputFormatUtils.class); - public static boolean doesBelongToIncrementalQuery(FileSplit s) { if (s instanceof HoodieRealtimeFileSplit) { HoodieRealtimeFileSplit bs = unsafeCast(s); @@ -75,7 +73,7 @@ public class HoodieRealtimeInputFormatUtils extends HoodieInputFormatUtils { // If not already in the list - then add it conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, readColNamesPrefix + fieldName); conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, readColIdsPrefix + fieldIndex); - LOG.debug("Adding extra column {}, to enable log merging cols ({}) ids ({})", + log.debug("Adding extra column {}, to enable log merging cols ({}) ids ({})", fieldName, conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR), conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR)); @@ -167,7 +165,7 @@ public class HoodieRealtimeInputFormatUtils extends HoodieInputFormatUtils { .collect(Collectors.joining(",")); if (!cleanedColumnIds.equals(columnIds)) { conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, cleanedColumnIds); - LOG.debug("The projection Ids: {{}} contained blank entries. Cleaned to: {{}}", columnIds, cleanedColumnIds); + log.debug("The projection Ids: {{}} contained blank entries. Cleaned to: {{}}", columnIds, cleanedColumnIds); } } } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java index 70d62cbc0f98..a6c434dc249e 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieRealtimeRecordReaderUtils.java @@ -34,6 +34,7 @@ import org.apache.hudi.exception.HoodieException; import org.apache.hudi.hadoop.fs.HadoopFSUtils; import org.apache.hudi.storage.StorageConfiguration; +import lombok.extern.slf4j.Slf4j; import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -53,8 +54,6 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; @@ -73,9 +72,8 @@ import java.util.stream.Collectors; import static org.apache.hudi.common.util.ConfigUtils.getReaderConfigs; import static org.apache.hudi.hadoop.fs.HadoopFSUtils.convertToStoragePath; +@Slf4j public class HoodieRealtimeRecordReaderUtils { - private static final Logger LOG = LoggerFactory.getLogger(HoodieRealtimeRecordReaderUtils.class); - /** * get the max compaction memory in bytes from JobConf. */ @@ -209,7 +207,7 @@ public class HoodieRealtimeRecordReaderUtils { if (record.getSchema().getField(field.name()) != null) { fieldValue = record.get(field.name()); } else { - LOG.debug("Field: {} not found in Schema: {}", field.name(), schema); + log.debug("Field: {} not found in Schema: {}", field.name(), schema); } recordValues[recordValueIndex++] = avroToArrayWritable(fieldValue, field.schema(), supportTimestamp); } diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java index bc5b7b3e124b..bf2d411f1c92 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java @@ -20,9 +20,8 @@ package org.apache.hudi.hadoop.utils.shims; import org.apache.hudi.exception.HoodieException; +import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.io.Writable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -32,10 +31,9 @@ import java.sql.Timestamp; /** * Shim clazz for Hive3. */ +@Slf4j public class Hive3Shim implements HiveShim { - public static final Logger LOG = LoggerFactory.getLogger(Hive3Shim.class); - public static final String TIMESTAMP_CLASS_NAME = "org.apache.hadoop.hive.common.type.Timestamp"; public static final String TIMESTAMP_WRITEABLE_V2_CLASS_NAME = "org.apache.hadoop.hive.serde2.io.TimestampWritableV2"; public static final String DATE_WRITEABLE_V2_CLASS_NAME = "org.apache.hadoop.hive.serde2.io.DateWritableV2"; @@ -65,7 +63,7 @@ public class Hive3Shim implements HiveShim { TIMESTAMP_WRITEABLE_V2_CONSTRUCTOR = TIMESTAMP_WRITABLE_V2_CLASS.getConstructor(TIMESTAMP_CLASS); } catch (ClassNotFoundException | NoSuchMethodException e) { // This will be printed out when Hive3Shim initialization as a singleton fails - LOG.warn("cannot find hive3 timestampv2 class or method, use hive2 class!", e); + log.warn("cannot find hive3 timestampv2 class or method, use hive2 class!", e); } // date @@ -75,7 +73,7 @@ public class Hive3Shim implements HiveShim { DATE_WRITEABLE_V2_CONSTRUCTOR = DATE_WRITEABLE_CLASS.getConstructor(int.class); } catch (ClassNotFoundException | NoSuchMethodException e) { // This will be printed out when Hive3Shim initialization as a singleton fails - LOG.warn("cannot find hive3 datev2 class or method, use hive2 class!", e); + log.warn("cannot find hive3 datev2 class or method, use hive2 class!", e); } }
