This is an automated email from the ASF dual-hosted git repository. dionusos pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push: new 8abe0d8 OOZIE-3652 Oozie launcher should retry directory listing when NoSuchFileException occurs (aajisaka via dionusos) 8abe0d8 is described below commit 8abe0d874e9b9a2ff41bf27432178ce3712e8597 Author: Denes Bodo <dionu...@apache.org> AuthorDate: Wed Feb 9 16:12:04 2022 +0100 OOZIE-3652 Oozie launcher should retry directory listing when NoSuchFileException occurs (aajisaka via dionusos) --- release-log.txt | 1 + .../oozie/action/hadoop/LocalFsOperations.java | 30 ++++++++++++++--- .../oozie/action/hadoop/TestLocalFsOperations.java | 38 ++++++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/release-log.txt b/release-log.txt index 3bbfc69..930fd31 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.3.0 release (trunk - unreleased) +OOZIE-3652 Oozie launcher should retry directory listing when NoSuchFileException occurs (aajisaka via dionusos) OOZIE-3655 upgrade jdom to jdom2 2.0.6.1 (pj.fanning via dionusos) OOZIE-3653 Upgrade commons-io to 2.11.0 (groot via dionusos) OOZIE-3657 Upgrade Jetty to 9.4.44.v20210927 (pj.fanning via asalamon74) diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java index 14f43ef..4277e25 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LocalFsOperations.java @@ -24,11 +24,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitOption; import java.nio.file.FileVisitResult; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.EnumSet; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; public class LocalFsOperations { @@ -56,18 +58,36 @@ public class LocalFsOperations { System.out.println("======================"); final Path root = folder.toPath(); - Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), WALK_DEPTH, new SimpleFileVisitor<Path>() { + // If a file is deleted between listing and printing, NoSuchFileException is thrown. + // This situation can happen when an agent attaches to the JVM. + // Retry at most 5 times to reduce the possibility of the error. + final int maxRetries = 5; + for (int i = 0; i < maxRetries; i++) { + try { + System.out.print(listContents(root)); + return; + } catch (NoSuchFileException e) { + System.err.println("A file or directory may be deleted when printing the current directory. Retrying."); + } + } + throw new IOException("Cannot list the current dir after " + maxRetries + " tries: " + folder.getAbsolutePath()); + } + + @VisibleForTesting + String listContents(Path path) throws IOException { + final StringBuilder builder = new StringBuilder(); + Files.walkFileTree(path, EnumSet.of(FileVisitOption.FOLLOW_LINKS), WALK_DEPTH, new SimpleFileVisitor<Path>() { @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (attrs.isRegularFile()) { - System.out.println(" File: " + root.relativize(file)); + builder.append(" File: ").append(path.relativize(file)).append("\n"); } else if (attrs.isDirectory()) { - System.out.println(" Dir: " + root.relativize(file) + "/"); + builder.append(" Dir: ").append(path.relativize(file)).append("/\n"); } - return FileVisitResult.CONTINUE; } }); + return builder.toString(); } /** diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLocalFsOperations.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLocalFsOperations.java new file mode 100644 index 0000000..e968825 --- /dev/null +++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLocalFsOperations.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.oozie.action.hadoop; + +import org.junit.Test; + +import java.io.File; +import java.nio.file.NoSuchFileException; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class TestLocalFsOperations { + @Test + public void testPrintContentsOfDirWithRetry() throws Exception { + LocalFsOperations spy = spy(new LocalFsOperations()); + // Throw an exception on 1st call + when(spy.listContents(new File(".").toPath())) + .thenThrow(NoSuchFileException.class) + .thenReturn("success"); + spy.printContentsOfDir(new File(".")); + } +}