danny0405 commented on code in PR #19198: URL: https://github.com/apache/hudi/pull/19198#discussion_r3529462461
########## hudi-common/src/main/java/org/apache/hudi/common/fs/FileNameParser.java: ########## @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.fs; + +import org.apache.hudi.common.table.cdc.HoodieCDCUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.exception.HoodieValidationException; +import org.apache.hudi.storage.StoragePath; + +import lombok.Getter; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parser for Hudi-generated base and log file names. + * + * <p>The parser accepts either a file name or a full path and always decodes only the last path component. + * A non-matching input returns {@link Option#empty()} instead of throwing.</p> + */ +public final class FileNameParser { + // Inline log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1 + // Inline archive log files are of this pattern - .commits_.archive.1_1-0-1 + // Native log files are of this pattern - b5068208-e1a4-11e6-bf01-fe55135034f3_1-0-1_20170101134598_1.log.parquet + // For native log files, the file extension is log/deletes/cdc and the suffix is the native file format. + + static final Pattern INLINE_LOG_FILE_PATTERN = + Pattern.compile("^\\.([^._]+)_([^.]*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(\\.cdc)?)?$"); + static final Pattern NATIVE_LOG_FILE_PATTERN = + Pattern.compile("^([^._]+)_((\\d+)-(\\d+)-(\\d+))_([^_]+)_(\\d+)\\.(log|deletes|cdc)\\.([^.]+)$"); + static final Pattern BASE_FILE_PATTERN = Pattern.compile("[a-zA-Z0-9-]+_[a-zA-Z0-9-]+_[0-9]+\\.[a-zA-Z0-9]+"); + private static final Pattern PREFIX_BY_FILE_ID_PATTERN = Pattern.compile("^(.+)-(\\d+)"); + + private FileNameParser() { + } + + /** + * Parses a Hudi-generated base file name. + * + * <p>Expected format: {@code <fileId>_<writeToken>_<commitTime>.<extension>}. + * The decoded {@link BaseFileName#getFileExtension()} value includes the leading dot, matching existing + * base-file helper API behavior.</p> + * + * @param fileName file name or full path + * @return decoded base file name when the input matches the Hudi base-file format + */ + public static Option<BaseFileName> parseBaseFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + Matcher matcher = BASE_FILE_PATTERN.matcher(actualFileName); + if (!matcher.matches()) { + return Option.empty(); + } + + int firstUnderscoreIndex = actualFileName.indexOf('_'); + int secondUnderscoreIndex = actualFileName.indexOf('_', firstUnderscoreIndex + 1); + int dotIndex = actualFileName.indexOf('.', secondUnderscoreIndex + 1); + return Option.of(new BaseFileName( + actualFileName.substring(0, firstUnderscoreIndex), + actualFileName.substring(firstUnderscoreIndex + 1, secondUnderscoreIndex), + actualFileName.substring(secondUnderscoreIndex + 1, dotIndex), + actualFileName.substring(dotIndex))); + } + + /** + * Parses either an inline log file name or a native log file name. + * + * <p>Inline log format: {@code .<fileId>_<deltaCommitTime>.<extension>.<version>_<writeToken>[.cdc]}. + * Native log format: {@code <fileId>_<writeToken>_<deltaCommitTime>_<version>.<extension>.<nativeFormat>}. + * The decoded {@link LogFileName#getFileExtension()} value does not include a leading dot. For inline CDC + * logs, {@link LogFileName#getSuffix()} is {@code .cdc}; for native logs, it is the native storage format, + * such as {@code parquet}.</p> + * + * @param fileName file name or full path + * @return decoded log file name when the input matches either supported log-file format + */ + public static Option<LogFileName> parseLogFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + Option<LogFileName> nativeLogFileName = parseNativeLogFileFromActualFileName(actualFileName); + if (nativeLogFileName.isPresent()) { + return nativeLogFileName; + } + + Matcher matcher = INLINE_LOG_FILE_PATTERN.matcher(actualFileName); + return matcher.matches() ? Option.of(LogFileName.fromInlineLogFile(matcher)) : Option.empty(); + } + + /** + * Parses only a native log file name. + * + * <p>Use {@link #parseLogFile(String)} when both inline and native log formats are acceptable.</p> + * + * @param fileName file name or full path + * @return decoded native log file name when the input matches the native log-file format + */ + public static Option<LogFileName> parseNativeLogFile(String fileName) { + return matchNativeLogFile(fileName).map(LogFileName::fromNativeLogFile); + } + + /** + * Returns the file group prefix encoded in a Hudi file id. + * + * @param fileId Hudi file id ending with a numeric suffix separated by {@code -} + * @return file id prefix before the trailing numeric suffix + * @throws HoodieValidationException when the file id does not contain the expected suffix + */ + public static String getFileIdPfxFromFileId(String fileId) { + Matcher matcher = PREFIX_BY_FILE_ID_PATTERN.matcher(fileId); + if (!matcher.find()) { + throw new HoodieValidationException("Failed to get prefix from " + fileId); + } + return matcher.group(1); + } + + static Option<Matcher> matchNativeLogFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + return matchNativeLogFileFromActualFileName(actualFileName); + } + + private static Option<LogFileName> parseNativeLogFileFromActualFileName(String actualFileName) { + return matchNativeLogFileFromActualFileName(actualFileName).map(LogFileName::fromNativeLogFile); + } + + private static Option<Matcher> matchNativeLogFileFromActualFileName(String actualFileName) { + Matcher matcher = NATIVE_LOG_FILE_PATTERN.matcher(actualFileName); + return matcher.matches() ? Option.of(matcher) : Option.empty(); + } + + private static String getActualFileName(String fileName) { + return fileName.contains(StoragePath.SEPARATOR) + ? fileName.substring(fileName.lastIndexOf(StoragePath.SEPARATOR) + 1) + : fileName; + } + + /** + * Decoded parts of a Hudi-generated base file name. + */ + @Getter + public static final class BaseFileName { + private final String fileId; + private final String writeToken; + private final String commitTime; + private final String fileExtension; + + private BaseFileName(String fileId, String writeToken, String commitTime, String fileExtension) { + this.fileId = fileId; + this.writeToken = writeToken; + this.commitTime = commitTime; + this.fileExtension = fileExtension; + } + + } + + /** + * Decoded parts of a Hudi-generated log file name. + */ + @Getter + public static final class LogFileName { + private final String fileId; + private final String deltaCommitTime; + private final int logVersion; + private final String writeToken; + private final Integer taskPartitionId; + private final Integer stageId; + private final Integer taskAttemptId; + private final String fileExtension; + private final String suffix; Review Comment: Done. I kept the public/accessor name as suffix, but added a declaration-site comment explaining that inline logs use it for the optional .cdc suffix while native logs use it for the native storage format such as parquet. ########## hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java: ########## @@ -181,7 +173,9 @@ public String getFileExtension() { } public boolean isCDC() { - return FSUtils.isCDCLogFile(getFileName()); + return isNativeLogFile() Review Comment: Done. HoodieLogFile now caches the CDC classification from FileNameParser.LogFileName#isCDCLogFile during parseFieldsFromPath, so the CDC predicate logic stays centralized in FileNameParser. ########## hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java: ########## @@ -519,54 +452,31 @@ public static boolean isLogFile(StoragePath logPath) { } public static boolean isLogFile(String fileName) { - if (matchNativeLogFile(fileName).isPresent()) { - return true; - } - if (fileName.startsWith(LOG_FILE_START_WITH_CHARACTER)) { - Matcher matcher = LOG_FILE_PATTERN.matcher(fileName); - return matcher.matches() && matcher.group(3).equals(LOG_FILE_EXTENSION); - } - return false; + return FileNameParser.parseLogFile(fileName) + .map(logFileName -> logFileName.isNativeLogFile() + || logFileName.getFileExtension().equals(LogExtensions.DATA_LOG_EXTENSION)) + .orElse(false); } public static boolean isInlineLogFile(String fileName) { - return matchNativeLogFile(fileName).isEmpty(); + return FileNameParser.parseNativeLogFile(fileName).isEmpty(); } public static Option<Matcher> matchNativeLogFile(String fileName) { - if (StringUtils.isNullOrEmpty(fileName)) { - return Option.empty(); - } - String actualFileName = fileName.contains(StoragePath.SEPARATOR) - ? fileName.substring(fileName.lastIndexOf(StoragePath.SEPARATOR) + 1) - : fileName; - Matcher matcher = NATIVE_LOG_FILE_PATTERN.matcher(actualFileName); - return matcher.matches() ? Option.of(matcher) : Option.empty(); + return FileNameParser.matchNativeLogFile(fileName); Review Comment: Done. The remaining raw Matcher consumers were migrated: TableSchemaResolver and NativeLogFooterMetadata now use FileNameParser.LogFileName, and the read/CDC paths use FSUtils.isNativeLogFile. FSUtils.matchNativeLogFile has been removed from the public API. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
