hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
Repository: hadoop Updated Branches: refs/heads/trunk 5f11288e4 -> bcc8e76ba HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcc8e76b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcc8e76b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcc8e76b Branch: refs/heads/trunk Commit: bcc8e76badc1341a6cf995c8e44fa5e422158de8 Parents: 5f11288 Author: Kihwal Lee Authored: Tue May 22 11:19:15 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:20:51 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcc8e76b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index a4fa8fd..595aeed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcc8e76b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete(); } - - /** - * Verify DiskChecker ignores at least 2 transient file creation e
hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 9788c8c01 -> 6b8a5af6e HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. (cherry picked from commit bcc8e76badc1341a6cf995c8e44fa5e422158de8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b8a5af6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b8a5af6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b8a5af6 Branch: refs/heads/branch-3.1 Commit: 6b8a5af6efe61ec619241187fd8c9869de0969ce Parents: 9788c8c Author: Kihwal Lee Authored: Tue May 22 11:22:22 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:22:22 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8a5af6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index a4fa8fd..595aeed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8a5af6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete();
hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
Repository: hadoop Updated Branches: refs/heads/branch-3.0 a5a9c8cf0 -> 96313e03c HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. (cherry picked from commit bcc8e76badc1341a6cf995c8e44fa5e422158de8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96313e03 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96313e03 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96313e03 Branch: refs/heads/branch-3.0 Commit: 96313e03c120907bbf35891579c52f127600acde Parents: a5a9c8c Author: Kihwal Lee Authored: Tue May 22 11:23:55 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:23:55 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/96313e03/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index a4fa8fd..595aeed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/96313e03/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete();
hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
Repository: hadoop Updated Branches: refs/heads/branch-2.9 a4b12f890 -> a10caac94 HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. (cherry picked from commit bcc8e76badc1341a6cf995c8e44fa5e422158de8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a10caac9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a10caac9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a10caac9 Branch: refs/heads/branch-2.9 Commit: a10caac945828d141559e5f8b15317f1b286516b Parents: a4b12f8 Author: Kihwal Lee Authored: Tue May 22 11:24:59 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:24:59 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a10caac9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index 8563232..c47de21 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a10caac9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete();
hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
Repository: hadoop Updated Branches: refs/heads/branch-2 43beb -> 5dcd57cbe HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. (cherry picked from commit bcc8e76badc1341a6cf995c8e44fa5e422158de8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5dcd57cb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5dcd57cb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5dcd57cb Branch: refs/heads/branch-2 Commit: 5dcd57cbeada2915678edbea5c5f0cc20486ddd9 Parents: 43b Author: Kihwal Lee Authored: Tue May 22 11:29:54 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:29:54 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dcd57cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index 8563232..c47de21 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dcd57cb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete(); } -
hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 8e1107d63 -> 1c407d327 HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. (cherry picked from commit bcc8e76badc1341a6cf995c8e44fa5e422158de8) Conflicts: hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c407d32 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c407d32 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c407d32 Branch: refs/heads/branch-2.8 Commit: 1c407d327a655576d2d0d12fd408db5ba0e85de8 Parents: 8e1107d Author: Kihwal Lee Authored: Tue May 22 11:36:25 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:36:25 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 101 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 103 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c407d32/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index 8563232..c47de21 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c407d32/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index 43bd183..ffeee4d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -203,105 +203,4 @@ public class TestDiskChecker { } localDir.delete(); } - - /** - * Verify DiskChecker ignores at least 2 transient file creation errors. - */ - @Test(timeout = 3) -
[39/50] [abbrv] hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcc8e76b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcc8e76b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcc8e76b Branch: refs/heads/HDDS-48 Commit: bcc8e76badc1341a6cf995c8e44fa5e422158de8 Parents: 5f11288 Author: Kihwal Lee Authored: Tue May 22 11:19:15 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:20:51 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcc8e76b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index a4fa8fd..595aeed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcc8e76b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete(); } - - /** - * Verify DiskChecker ignores at least 2 transient file creation errors. - */ - @Test(timeout = 3) - public void testDiskIoIgnoresTransie
[07/50] [abbrv] hadoop git commit: HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal.
HADOOP-15450. Avoid fsync storm triggered by DiskChecker and handle disk full situation. Contributed by Arpit Agarwal. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcc8e76b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcc8e76b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcc8e76b Branch: refs/heads/HDDS-4 Commit: bcc8e76badc1341a6cf995c8e44fa5e422158de8 Parents: 5f11288 Author: Kihwal Lee Authored: Tue May 22 11:19:15 2018 -0500 Committer: Kihwal Lee Committed: Tue May 22 11:20:51 2018 -0500 -- .../org/apache/hadoop/util/DiskChecker.java | 46 - .../org/apache/hadoop/util/TestDiskChecker.java | 102 --- .../hadoop/util/TestDiskCheckerWithDiskIo.java | 173 +++ 3 files changed, 217 insertions(+), 104 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcc8e76b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java index a4fa8fd..595aeed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java @@ -74,12 +74,30 @@ public class DiskChecker { * @throws DiskErrorException */ public static void checkDir(File dir) throws DiskErrorException { +checkDirInternal(dir); + } + + /** + * Create the directory if it doesn't exist and check that dir is + * readable, writable and executable. Perform some disk IO to + * ensure that the disk is usable for writes. + * + * @param dir + * @throws DiskErrorException + */ + public static void checkDirWithDiskIo(File dir) + throws DiskErrorException { +checkDirInternal(dir); +doDiskIo(dir); + } + + private static void checkDirInternal(File dir) + throws DiskErrorException { if (!mkdirsWithExistsCheck(dir)) { throw new DiskErrorException("Cannot create directory: " + dir.toString()); } checkAccessByFileMethods(dir); -doDiskIo(dir); } /** @@ -94,10 +112,34 @@ public class DiskChecker { */ public static void checkDir(LocalFileSystem localFS, Path dir, FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); + } + + + /** + * Create the local directory if necessary, also ensure permissions + * allow it to be read from and written into. Perform some diskIO + * to ensure that the disk is usable for writes. + * + * @param localFS local filesystem + * @param dir directory + * @param expected permission + * @throws DiskErrorException + * @throws IOException + */ + public static void checkDirWithDiskIo(LocalFileSystem localFS, Path dir, +FsPermission expected) + throws DiskErrorException, IOException { +checkDirInternal(localFS, dir, expected); +doDiskIo(localFS.pathToFile(dir)); + } + + private static void checkDirInternal(LocalFileSystem localFS, Path dir, + FsPermission expected) throws DiskErrorException, IOException { mkdirsWithExistsAndPermissionCheck(localFS, dir, expected); checkAccessByFileMethods(localFS.pathToFile(dir)); -doDiskIo(localFS.pathToFile(dir)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcc8e76b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java -- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index bd8e1dd..6b6c6c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util; import java.io.*; import java.nio.file.Files; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.util.DiskChecker.FileIoProvider; import org.junit.After; @@ -214,105 +213,4 @@ public class TestDiskChecker { } localDir.delete(); } - - /** - * Verify DiskChecker ignores at least 2 transient file creation errors. - */ - @Test(timeout = 3) - public void testDiskIoIgnoresTransien