hudi-agent commented on code in PR #19198: URL: https://github.com/apache/hudi/pull/19198#discussion_r3528356248
########## hudi-common/src/main/java/org/apache/hudi/common/fs/FileNameParser.java: ########## @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.fs; + +import org.apache.hudi.common.table.cdc.HoodieCDCUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.exception.HoodieValidationException; +import org.apache.hudi.storage.StoragePath; + +import lombok.Getter; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parser for Hudi-generated base and log file names. + * + * <p>The parser accepts either a file name or a full path and always decodes only the last path component. + * A non-matching input returns {@link Option#empty()} instead of throwing.</p> + */ +public final class FileNameParser { + // Inline log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1 + // Inline archive log files are of this pattern - .commits_.archive.1_1-0-1 + // Native log files are of this pattern - b5068208-e1a4-11e6-bf01-fe55135034f3_1-0-1_20170101134598_1.log.parquet + // For native log files, the file extension is log/deletes/cdc and the suffix is the native file format. + + static final Pattern INLINE_LOG_FILE_PATTERN = + Pattern.compile("^\\.([^._]+)_([^.]*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(\\.cdc)?)?$"); + static final Pattern NATIVE_LOG_FILE_PATTERN = + Pattern.compile("^([^._]+)_((\\d+)-(\\d+)-(\\d+))_([^_]+)_(\\d+)\\.(log|deletes|cdc)\\.([^.]+)$"); + static final Pattern BASE_FILE_PATTERN = Pattern.compile("[a-zA-Z0-9-]+_[a-zA-Z0-9-]+_[0-9]+\\.[a-zA-Z0-9]+"); + private static final Pattern PREFIX_BY_FILE_ID_PATTERN = Pattern.compile("^(.+)-(\\d+)"); + + private FileNameParser() { + } + + /** + * Parses a Hudi-generated base file name. + * + * <p>Expected format: {@code <fileId>_<writeToken>_<commitTime>.<extension>}. + * The decoded {@link BaseFileName#getFileExtension()} value includes the leading dot, matching existing + * base-file helper API behavior.</p> + * + * @param fileName file name or full path + * @return decoded base file name when the input matches the Hudi base-file format + */ + public static Option<BaseFileName> parseBaseFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + Matcher matcher = BASE_FILE_PATTERN.matcher(actualFileName); + if (!matcher.matches()) { + return Option.empty(); + } + + int firstUnderscoreIndex = actualFileName.indexOf('_'); Review Comment: 🤖 nit: `BASE_FILE_PATTERN` validates the format here but then the code re-parses the same string manually with `indexOf`. Could you add capture groups to the regex (e.g. `([a-zA-Z0-9-]+)_([a-zA-Z0-9-]+)_([0-9]+)(\.[a-zA-Z0-9]+)`) and use `matcher.group(N)` directly? That way the validator and extractor stay in sync automatically if the format ever changes. <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> ########## hudi-common/src/main/java/org/apache/hudi/common/fs/FileNameParser.java: ########## @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.fs; + +import org.apache.hudi.common.table.cdc.HoodieCDCUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.exception.HoodieValidationException; +import org.apache.hudi.storage.StoragePath; + +import lombok.Getter; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parser for Hudi-generated base and log file names. + * + * <p>The parser accepts either a file name or a full path and always decodes only the last path component. + * A non-matching input returns {@link Option#empty()} instead of throwing.</p> + */ +public final class FileNameParser { + // Inline log files are of this pattern - .b5068208-e1a4-11e6-bf01-fe55135034f3_20170101134598.log.1_1-0-1 + // Inline archive log files are of this pattern - .commits_.archive.1_1-0-1 + // Native log files are of this pattern - b5068208-e1a4-11e6-bf01-fe55135034f3_1-0-1_20170101134598_1.log.parquet + // For native log files, the file extension is log/deletes/cdc and the suffix is the native file format. + + static final Pattern INLINE_LOG_FILE_PATTERN = + Pattern.compile("^\\.([^._]+)_([^.]*)\\.(log|archive)\\.(\\d+)(_((\\d+)-(\\d+)-(\\d+))(\\.cdc)?)?$"); + static final Pattern NATIVE_LOG_FILE_PATTERN = + Pattern.compile("^([^._]+)_((\\d+)-(\\d+)-(\\d+))_([^_]+)_(\\d+)\\.(log|deletes|cdc)\\.([^.]+)$"); + static final Pattern BASE_FILE_PATTERN = Pattern.compile("[a-zA-Z0-9-]+_[a-zA-Z0-9-]+_[0-9]+\\.[a-zA-Z0-9]+"); + private static final Pattern PREFIX_BY_FILE_ID_PATTERN = Pattern.compile("^(.+)-(\\d+)"); + + private FileNameParser() { + } + + /** + * Parses a Hudi-generated base file name. + * + * <p>Expected format: {@code <fileId>_<writeToken>_<commitTime>.<extension>}. + * The decoded {@link BaseFileName#getFileExtension()} value includes the leading dot, matching existing + * base-file helper API behavior.</p> + * + * @param fileName file name or full path + * @return decoded base file name when the input matches the Hudi base-file format + */ + public static Option<BaseFileName> parseBaseFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + Matcher matcher = BASE_FILE_PATTERN.matcher(actualFileName); + if (!matcher.matches()) { + return Option.empty(); + } + + int firstUnderscoreIndex = actualFileName.indexOf('_'); + int secondUnderscoreIndex = actualFileName.indexOf('_', firstUnderscoreIndex + 1); + int dotIndex = actualFileName.indexOf('.', secondUnderscoreIndex + 1); + return Option.of(new BaseFileName( + actualFileName.substring(0, firstUnderscoreIndex), + actualFileName.substring(firstUnderscoreIndex + 1, secondUnderscoreIndex), + actualFileName.substring(secondUnderscoreIndex + 1, dotIndex), + actualFileName.substring(dotIndex))); + } + + /** + * Parses either an inline log file name or a native log file name. + * + * <p>Inline log format: {@code .<fileId>_<deltaCommitTime>.<extension>.<version>_<writeToken>[.cdc]}. + * Native log format: {@code <fileId>_<writeToken>_<deltaCommitTime>_<version>.<extension>.<nativeFormat>}. + * The decoded {@link LogFileName#getFileExtension()} value does not include a leading dot. For inline CDC + * logs, {@link LogFileName#getSuffix()} is {@code .cdc}; for native logs, it is the native storage format, + * such as {@code parquet}.</p> + * + * @param fileName file name or full path + * @return decoded log file name when the input matches either supported log-file format + */ + public static Option<LogFileName> parseLogFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + Option<LogFileName> nativeLogFileName = parseNativeLogFileFromActualFileName(actualFileName); + if (nativeLogFileName.isPresent()) { + return nativeLogFileName; + } + + Matcher matcher = INLINE_LOG_FILE_PATTERN.matcher(actualFileName); + return matcher.matches() ? Option.of(LogFileName.fromInlineLogFile(matcher)) : Option.empty(); + } + + /** + * Parses only a native log file name. + * + * <p>Use {@link #parseLogFile(String)} when both inline and native log formats are acceptable.</p> + * + * @param fileName file name or full path + * @return decoded native log file name when the input matches the native log-file format + */ + public static Option<LogFileName> parseNativeLogFile(String fileName) { + return matchNativeLogFile(fileName).map(LogFileName::fromNativeLogFile); + } + + /** + * Returns the file group prefix encoded in a Hudi file id. + * + * @param fileId Hudi file id ending with a numeric suffix separated by {@code -} + * @return file id prefix before the trailing numeric suffix + * @throws HoodieValidationException when the file id does not contain the expected suffix + */ + public static String getFileIdPfxFromFileId(String fileId) { + Matcher matcher = PREFIX_BY_FILE_ID_PATTERN.matcher(fileId); + if (!matcher.find()) { + throw new HoodieValidationException("Failed to get prefix from " + fileId); + } + return matcher.group(1); + } + + static Option<Matcher> matchNativeLogFile(String fileName) { + if (StringUtils.isNullOrEmpty(fileName)) { + return Option.empty(); + } + + String actualFileName = getActualFileName(fileName); + return matchNativeLogFileFromActualFileName(actualFileName); + } + + private static Option<LogFileName> parseNativeLogFileFromActualFileName(String actualFileName) { + return matchNativeLogFileFromActualFileName(actualFileName).map(LogFileName::fromNativeLogFile); + } + + private static Option<Matcher> matchNativeLogFileFromActualFileName(String actualFileName) { + Matcher matcher = NATIVE_LOG_FILE_PATTERN.matcher(actualFileName); + return matcher.matches() ? Option.of(matcher) : Option.empty(); + } + + private static String getActualFileName(String fileName) { + return fileName.contains(StoragePath.SEPARATOR) + ? fileName.substring(fileName.lastIndexOf(StoragePath.SEPARATOR) + 1) + : fileName; + } + + /** + * Decoded parts of a Hudi-generated base file name. + */ + @Getter + public static final class BaseFileName { + private final String fileId; + private final String writeToken; + private final String commitTime; + private final String fileExtension; + + private BaseFileName(String fileId, String writeToken, String commitTime, String fileExtension) { + this.fileId = fileId; + this.writeToken = writeToken; + this.commitTime = commitTime; + this.fileExtension = fileExtension; + } + + } + + /** + * Decoded parts of a Hudi-generated log file name. + */ + @Getter + public static final class LogFileName { + private final String fileId; + private final String deltaCommitTime; + private final int logVersion; + private final String writeToken; + private final Integer taskPartitionId; + private final Integer stageId; + private final Integer taskAttemptId; + private final String fileExtension; + private final String suffix; Review Comment: 🤖 nit: the `suffix` field means different things depending on the log file type — it's `.cdc` (with a leading dot) for inline CDC log files, but `parquet` (no dot) for native log files, and `isCDCLogFile()` has to branch on `nativeLogFile` to account for this. Have you considered a rename like `cdcOrNativeFormat` with an inline comment, or splitting into two optional fields, to make this asymmetry explicit at the declaration site? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
