prashantwason commented on a change in pull request #3873: URL: https://github.com/apache/hudi/pull/3873#discussion_r744072765
########## File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java ########## @@ -419,52 +394,53 @@ private boolean bootstrapFromFilesystem(HoodieEngineContext engineContext, Hoodi * @param dataMetaClient * @return Map of partition names to a list of FileStatus for all the files in the partition */ - private Map<String, List<FileStatus>> getPartitionsToFilesMapping(HoodieTableMetaClient dataMetaClient) { + private List<DirectoryInfo> listAllPartitions(HoodieTableMetaClient datasetMetaClient) { List<Path> pathsToList = new LinkedList<>(); pathsToList.add(new Path(dataWriteConfig.getBasePath())); - Map<String, List<FileStatus>> partitionToFileStatus = new HashMap<>(); + List<DirectoryInfo> foundPartitionsList = new LinkedList<>(); final int fileListingParallelism = metadataWriteConfig.getFileListingParallelism(); SerializableConfiguration conf = new SerializableConfiguration(dataMetaClient.getHadoopConf()); final String dirFilterRegex = dataWriteConfig.getMetadataConfig().getDirectoryFilterRegex(); + final String datasetBasePath = dataMetaClient.getBasePath(); while (!pathsToList.isEmpty()) { - int listingParallelism = Math.min(fileListingParallelism, pathsToList.size()); + // In each round we will list a section of directories + int numDirsToList = Math.min(fileListingParallelism, pathsToList.size()); // List all directories in parallel - List<Pair<Path, FileStatus[]>> dirToFileListing = engineContext.map(pathsToList, path -> { + List<DirectoryInfo> foundDirsList = engineContext.map(pathsToList.subList(0, numDirsToList), path -> { Review comment: Renamed. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org