LompleZ commented on code in PR #64224:
URL: https://github.com/apache/doris/pull/64224#discussion_r3387402255
##########
fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java:
##########
@@ -1385,55 +1370,113 @@ private static TBrokerFD parseUUIDToFD(UUID uuid) {
* 2. For other authentication modes, the lastAccessTime is used to
determine whether it has expired
*/
private BrokerFileSystem updateCachedFileSystem(FileSystemIdentity
fileSystemIdentity, Map<String, String> properties) {
- BrokerFileSystem brokerFileSystem;
- if (cachedFileSystem.containsKey(fileSystemIdentity)) {
- brokerFileSystem = cachedFileSystem.get(fileSystemIdentity);
+ return cachedFileSystem.compute(fileSystemIdentity, (key,
brokerFileSystemInMap) -> {
+ if (brokerFileSystemInMap == null) {
+ BrokerFileSystem newBrokerFileSystem = new
BrokerFileSystem(fileSystemIdentity);
+ newBrokerFileSystem.incrementActiveOperations();
+ return newBrokerFileSystem;
+ }
if (UserGroupInformation.isSecurityEnabled()) {
try {
UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
} catch (Exception e) {
logger.error("errors while refresh TGT: ", e);
}
- } else if (brokerFileSystem.isExpiredByLastAccessTime()) {
- brokerFileSystem.getLock().lock();
- BrokerFileSystem bfs =
cachedFileSystem.get(fileSystemIdentity);
- if (!bfs.isExpiredByLastAccessTime()) {
- return bfs;
- }
- try {
- logger.info("file system " + brokerFileSystem + " is
expired, update it.");
- brokerFileSystem.closeFileSystem();
- } catch (Throwable t) {
- logger.error("errors while close file system: ", t);
- } finally {
- brokerFileSystem.getLock().unlock();
- }
- brokerFileSystem = new BrokerFileSystem(fileSystemIdentity);
- cachedFileSystem.put(fileSystemIdentity, brokerFileSystem);
+ } else if (brokerFileSystemInMap.isExpiredByLastAccessTime()) {
+ logger.info("file system " + brokerFileSystemInMap + " is
expired, move to recycle bin and update it.");
+ fileSystemRecycleBin.add(brokerFileSystemInMap);
+ BrokerFileSystem newBrokerFileSystem = new
BrokerFileSystem(fileSystemIdentity);
+ newBrokerFileSystem.incrementActiveOperations();
+ return newBrokerFileSystem;
}
- } else {
- brokerFileSystem = new BrokerFileSystem(fileSystemIdentity);
- cachedFileSystem.put(fileSystemIdentity, brokerFileSystem);
- }
- return brokerFileSystem;
+ brokerFileSystemInMap.incrementActiveOperations();
+ brokerFileSystemInMap.updateLastUpdateAccessTime();
+ return brokerFileSystemInMap;
+ });
}
public long fileSize(String path, Map<String, String> properties) {
WildcardURI pathUri = new WildcardURI(path);
- BrokerFileSystem fileSystem = getFileSystem(path, properties);
+ BrokerFileSystem fileSystem = null;
Path filePath = new Path(pathUri.getPath());
try {
+ fileSystem = getFileSystem(path, properties);
FileStatus fileStatus =
fileSystem.getDFSFileSystem().getFileStatus(filePath);
if (fileStatus.isDirectory()) {
throw new
BrokerException(TBrokerOperationStatusCode.INVALID_INPUT_FILE_PATH,
"not a file: {}", path);
}
return fileStatus.getLen();
- } catch (IOException e) {
+ } catch (Exception e) {
logger.error("errors while getting file size: " + path);
Review Comment:
done.
I have fixed this issue. Added an extra BrokerException type check here.
File systems will no longer be mistakenly evicted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
