[03/50] [abbrv] hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.
HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c67b0650 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c67b0650 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c67b0650 Branch: refs/heads/HDFS-12943 Commit: c67b0650ea10896c6289703595faef0d262c00b3 Parents: 1697a02 Author: Xiao Chen Authored: Thu Aug 16 23:13:10 2018 -0700 Committer: Xiao Chen Committed: Thu Aug 16 23:14:21 2018 -0700 -- .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +- .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++ 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 70b3679..28c1e27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1217,7 +1217,11 @@ public class DistributedFileSystem extends FileSystem thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); - storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + if (needLocation) { +storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + } else { +storageStatistics.incrementOpCounter(OpType.LIST_STATUS); + } if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index f09255e..46323dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -706,6 +706,7 @@ public class TestDistributedFileSystem { // Iterative ls test long mkdirOp = getOpStatistics(OpType.MKDIRS); long listStatusOp = getOpStatistics(OpType.LIST_STATUS); + long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS); for (int i = 0; i < 10; i++) { Path p = new Path(dir, Integer.toString(i)); fs.mkdirs(p); @@ -729,6 +730,12 @@ public class TestDistributedFileSystem { checkStatistics(fs, readOps, ++writeOps, largeReadOps); checkOpStatistics(OpType.MKDIRS, mkdirOp); checkOpStatistics(OpType.LIST_STATUS, listStatusOp); + +fs.listLocatedStatus(dir); +locatedListStatusOP++; +readOps++; +checkStatistics(fs, readOps, writeOps, largeReadOps); +checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP); } opCount = getOpStatistics(OpType.GET_STATUS); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.
Repository: hadoop Updated Branches: refs/heads/branch-3.1 3532bd5c8 -> 06f0d5e25 HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi. (cherry picked from commit c67b0650ea10896c6289703595faef0d262c00b3) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06f0d5e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06f0d5e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06f0d5e2 Branch: refs/heads/branch-3.1 Commit: 06f0d5e257a3208f905f59019623a589825e4c8b Parents: 3532bd5 Author: Xiao Chen Authored: Thu Aug 16 23:13:10 2018 -0700 Committer: Xiao Chen Committed: Thu Aug 16 23:15:48 2018 -0700 -- .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +- .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++ 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0d5e2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 3519c60..de05f82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1217,7 +1217,11 @@ public class DistributedFileSystem extends FileSystem thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); - storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + if (needLocation) { +storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + } else { +storageStatistics.incrementOpCounter(OpType.LIST_STATUS); + } if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0d5e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 072ee9f..03e6c8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -706,6 +706,7 @@ public class TestDistributedFileSystem { // Iterative ls test long mkdirOp = getOpStatistics(OpType.MKDIRS); long listStatusOp = getOpStatistics(OpType.LIST_STATUS); + long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS); for (int i = 0; i < 10; i++) { Path p = new Path(dir, Integer.toString(i)); fs.mkdirs(p); @@ -729,6 +730,12 @@ public class TestDistributedFileSystem { checkStatistics(fs, readOps, ++writeOps, largeReadOps); checkOpStatistics(OpType.MKDIRS, mkdirOp); checkOpStatistics(OpType.LIST_STATUS, listStatusOp); + +fs.listLocatedStatus(dir); +locatedListStatusOP++; +readOps++; +checkStatistics(fs, readOps, writeOps, largeReadOps); +checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP); } opCount = getOpStatistics(OpType.GET_STATUS); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.
Repository: hadoop Updated Branches: refs/heads/branch-3.0 90bf2d3b5 -> 185c8f2ab HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi. (cherry picked from commit c67b0650ea10896c6289703595faef0d262c00b3) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/185c8f2a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/185c8f2a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/185c8f2a Branch: refs/heads/branch-3.0 Commit: 185c8f2abc364e4941ca4d4522fb61b5b3f5f903 Parents: 90bf2d3 Author: Xiao Chen Authored: Thu Aug 16 23:13:10 2018 -0700 Committer: Xiao Chen Committed: Thu Aug 16 23:15:58 2018 -0700 -- .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +- .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++ 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/185c8f2a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 84d840f..9208e66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1143,7 +1143,11 @@ public class DistributedFileSystem extends FileSystem thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); - storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + if (needLocation) { +storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + } else { +storageStatistics.incrementOpCounter(OpType.LIST_STATUS); + } if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/185c8f2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 072ee9f..03e6c8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -706,6 +706,7 @@ public class TestDistributedFileSystem { // Iterative ls test long mkdirOp = getOpStatistics(OpType.MKDIRS); long listStatusOp = getOpStatistics(OpType.LIST_STATUS); + long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS); for (int i = 0; i < 10; i++) { Path p = new Path(dir, Integer.toString(i)); fs.mkdirs(p); @@ -729,6 +730,12 @@ public class TestDistributedFileSystem { checkStatistics(fs, readOps, ++writeOps, largeReadOps); checkOpStatistics(OpType.MKDIRS, mkdirOp); checkOpStatistics(OpType.LIST_STATUS, listStatusOp); + +fs.listLocatedStatus(dir); +locatedListStatusOP++; +readOps++; +checkStatistics(fs, readOps, writeOps, largeReadOps); +checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP); } opCount = getOpStatistics(OpType.GET_STATUS); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi.
Repository: hadoop Updated Branches: refs/heads/trunk 1697a0230 -> c67b0650e HDFS-13747. Statistic for list_located_status is incremented incorrectly by listStatusIterator. Contributed by Antal Mihalyi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c67b0650 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c67b0650 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c67b0650 Branch: refs/heads/trunk Commit: c67b0650ea10896c6289703595faef0d262c00b3 Parents: 1697a02 Author: Xiao Chen Authored: Thu Aug 16 23:13:10 2018 -0700 Committer: Xiao Chen Committed: Thu Aug 16 23:14:21 2018 -0700 -- .../java/org/apache/hadoop/hdfs/DistributedFileSystem.java| 6 +- .../org/apache/hadoop/hdfs/TestDistributedFileSystem.java | 7 +++ 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 70b3679..28c1e27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1217,7 +1217,11 @@ public class DistributedFileSystem extends FileSystem thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); - storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + if (needLocation) { +storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); + } else { +storageStatistics.incrementOpCounter(OpType.LIST_STATUS); + } if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c67b0650/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index f09255e..46323dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -706,6 +706,7 @@ public class TestDistributedFileSystem { // Iterative ls test long mkdirOp = getOpStatistics(OpType.MKDIRS); long listStatusOp = getOpStatistics(OpType.LIST_STATUS); + long locatedListStatusOP = getOpStatistics(OpType.LIST_LOCATED_STATUS); for (int i = 0; i < 10; i++) { Path p = new Path(dir, Integer.toString(i)); fs.mkdirs(p); @@ -729,6 +730,12 @@ public class TestDistributedFileSystem { checkStatistics(fs, readOps, ++writeOps, largeReadOps); checkOpStatistics(OpType.MKDIRS, mkdirOp); checkOpStatistics(OpType.LIST_STATUS, listStatusOp); + +fs.listLocatedStatus(dir); +locatedListStatusOP++; +readOps++; +checkStatistics(fs, readOps, writeOps, largeReadOps); +checkOpStatistics(OpType.LIST_LOCATED_STATUS, locatedListStatusOP); } opCount = getOpStatistics(OpType.GET_STATUS); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org