HDFS-11267. Avoid redefinition of storageDirs in NNStorage and cleanup its accessors in Storage. (Manoj Govindassamy via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4f66655 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4f66655 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4f66655 Branch: refs/heads/YARN-3926 Commit: a4f66655ec22ca8c960f971f2b0cdafbd3430ad7 Parents: e9f1396 Author: Lei Xu <l...@apache.org> Authored: Thu Dec 29 16:57:40 2016 +0800 Committer: Lei Xu <l...@apache.org> Committed: Thu Dec 29 16:57:40 2016 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/server/common/Storage.java | 8 ++++++-- .../hdfs/server/datanode/BlockPoolSliceStorage.java | 6 +++--- .../hadoop/hdfs/server/datanode/DataStorage.java | 8 ++++---- .../apache/hadoop/hdfs/server/namenode/NNStorage.java | 14 ++++++-------- .../hadoop/hdfs/server/common/StorageAdapter.java | 2 +- .../server/datanode/TestBlockPoolSliceStorage.java | 2 +- 6 files changed, 21 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index c172289..f23a48a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -125,7 +125,7 @@ public abstract class Storage extends StorageInfo { } protected List<StorageDirectory> storageDirs = - new CopyOnWriteArrayList<StorageDirectory>(); + new CopyOnWriteArrayList<>(); private class DirIterator implements Iterator<StorageDirectory> { final StorageDirType dirType; @@ -938,7 +938,11 @@ public abstract class Storage extends StorageInfo { public int getNumStorageDirs() { return storageDirs.size(); } - + + public List<StorageDirectory> getStorageDirs() { + return storageDirs; + } + public StorageDirectory getStorageDir(int idx) { return storageDirs.get(idx); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index dd82a74..3203de2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -295,7 +295,7 @@ public class BlockPoolSliceStorage extends Storage { void remove(File absPathToRemove) { Preconditions.checkArgument(absPathToRemove.isAbsolute()); LOG.info("Removing block level storage: " + absPathToRemove); - for (Iterator<StorageDirectory> it = this.storageDirs.iterator(); + for (Iterator<StorageDirectory> it = getStorageDirs().iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); if (sd.getRoot().getAbsoluteFile().equals(absPathToRemove)) { @@ -788,7 +788,7 @@ public class BlockPoolSliceStorage extends Storage { */ public void clearTrash() { final List<File> trashRoots = new ArrayList<>(); - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { File trashRoot = getTrashRootDir(sd); if (trashRoot.exists() && sd.getPreviousDir().exists()) { LOG.error("Trash and PreviousDir shouldn't both exist for storage " @@ -826,7 +826,7 @@ public class BlockPoolSliceStorage extends Storage { /** trash is enabled if at least one storage directory contains trash root */ @VisibleForTesting public boolean trashEnabled() { - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { if (getTrashRootDir(sd).exists()) { return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 12d9322..f410a85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -181,11 +181,11 @@ public class DataStorage extends Storage { } public void setRollingUpgradeMarker(String bpid) throws IOException { - getBPStorage(bpid).setRollingUpgradeMarkers(storageDirs); + getBPStorage(bpid).setRollingUpgradeMarkers(getStorageDirs()); } public void clearRollingUpgradeMarker(String bpid) throws IOException { - getBPStorage(bpid).clearRollingUpgradeMarkers(storageDirs); + getBPStorage(bpid).clearRollingUpgradeMarkers(getStorageDirs()); } /** @@ -493,7 +493,7 @@ public class DataStorage extends Storage { } StringBuilder errorMsgBuilder = new StringBuilder(); - for (Iterator<StorageDirectory> it = this.storageDirs.iterator(); + for (Iterator<StorageDirectory> it = getStorageDirs().iterator(); it.hasNext(); ) { StorageDirectory sd = it.next(); StorageLocation sdLocation = sd.getStorageLocation(); @@ -988,7 +988,7 @@ public class DataStorage extends Storage { // To handle finalizing a snapshot taken at datanode level while // upgrading to federation, if datanode level snapshot previous exists, // then finalize it. Else finalize the corresponding BP. - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { File prevDir = sd.getPreviousDir(); if (prevDir.exists()) { // data node level storage finalize http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index d6dd8ee..c79ba4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -172,8 +172,6 @@ public class NNStorage extends Storage implements Closeable, throws IOException { super(NodeType.NAME_NODE); - storageDirs = new CopyOnWriteArrayList<>(); - // this may modify the editsDirs, so copy before passing in setStorageDirectories(imageDirs, Lists.newArrayList(editsDirs), @@ -212,7 +210,7 @@ public class NNStorage extends Storage implements Closeable, @Override // Closeable public void close() throws IOException { unlockAll(); - storageDirs.clear(); + getStorageDirs().clear(); } /** @@ -295,7 +293,7 @@ public class NNStorage extends Storage implements Closeable, Collection<URI> fsEditsDirs, Collection<URI> sharedEditsDirs) throws IOException { - this.storageDirs.clear(); + getStorageDirs().clear(); this.removedStorageDirs.clear(); // Add all name dirs with appropriate NameNodeDirType @@ -873,7 +871,7 @@ public class NNStorage extends Storage implements Closeable, + sd.getRoot().getPath(), e); } - if (this.storageDirs.remove(sd)) { + if (getStorageDirs().remove(sd)) { this.removedStorageDirs.add(sd); } @@ -928,7 +926,7 @@ public class NNStorage extends Storage implements Closeable, // getCanonicalPath may need to call stat() or readlink() and it's likely // those calls would fail due to the same underlying IO problem. String absPath = f.getAbsolutePath(); - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { String dirPath = sd.getRoot().getAbsolutePath(); if (!dirPath.endsWith(File.separator)) { dirPath += File.separator; @@ -1140,14 +1138,14 @@ public class NNStorage extends Storage implements Closeable, @Override public void writeAll() throws IOException { this.layoutVersion = getServiceLayoutVersion(); - for (StorageDirectory sd : storageDirs) { + for (StorageDirectory sd : getStorageDirs()) { try { writeProperties(sd); } catch (Exception e) { LOG.warn("Error during write properties to the VERSION file to " + sd.toString(), e); reportErrorsOnDirectory(sd); - if (storageDirs.isEmpty()) { + if (getStorageDirs().isEmpty()) { throw new IOException("All the storage failed while writing " + "properties to VERSION file"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java index 3dd81b2..c2df9bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/StorageAdapter.java @@ -33,7 +33,7 @@ public abstract class StorageAdapter { Storage s, int idx) { StorageDirectory dir = Mockito.spy(s.getStorageDir(idx)); - s.storageDirs.set(idx, dir); + s.getStorageDirs().set(idx, dir); return dir; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4f66655/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java index a8f8b6a..d9bbe88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java @@ -50,7 +50,7 @@ public class TestBlockPoolSliceStorage { String clusterId) { super(namespaceID, bpID, cTime, clusterId); addStorageDir(new StorageDirectory(new File("/tmp/dontcare/" + bpID))); - assertThat(storageDirs.size(), is(1)); + assertThat(getStorageDirs().size(), is(1)); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
