[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14156346#comment-14156346 ] Hudson commented on HDFS-7159: -- FAILURE: Integrated in Hadoop-Yarn-trunk #698 (See [https://builds.apache.org/job/Hadoop-Yarn-trunk/698/]) HDFS-7159. Use block storage policy to set lazy persist preference. (Arpit Agarwal) (arp: rev bb84f1fccb18c6c7373851e05d2451d55e908242) * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java * hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java * hadoop-common-project/hadoop-common/src/test/resources/testConf.xml * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java *
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14156424#comment-14156424 ] Hudson commented on HDFS-7159: -- FAILURE: Integrated in Hadoop-Hdfs-trunk #1889 (See [https://builds.apache.org/job/Hadoop-Hdfs-trunk/1889/]) HDFS-7159. Use block storage policy to set lazy persist preference. (Arpit Agarwal) (arp: rev bb84f1fccb18c6c7373851e05d2451d55e908242) * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java * hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java * hadoop-common-project/hadoop-common/src/test/resources/testConf.xml * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java *
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14156540#comment-14156540 ] Hudson commented on HDFS-7159: -- FAILURE: Integrated in Hadoop-Mapreduce-trunk #1914 (See [https://builds.apache.org/job/Hadoop-Mapreduce-trunk/1914/]) HDFS-7159. Use block storage policy to set lazy persist preference. (Arpit Agarwal) (arp: rev bb84f1fccb18c6c7373851e05d2451d55e908242) * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java * hadoop-common-project/hadoop-common/src/test/resources/testConf.xml * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java * hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImplAllocator.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java * hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt *
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14155010#comment-14155010 ] Hudson commented on HDFS-7159: -- FAILURE: Integrated in Hadoop-trunk-Commit #6163 (See [https://builds.apache.org/job/Hadoop-trunk-Commit/6163/]) HDFS-7159. Use block storage policy to set lazy persist preference. (Arpit Agarwal) (arp: rev bb84f1fccb18c6c7373851e05d2451d55e908242) * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java * hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Stat.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java * hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java * hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsTransientVolumeImpl.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java * hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java * hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java * hadoop-common-project/hadoop-common/src/test/resources/testConf.xml * hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java *
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14151678#comment-14151678 ] Tsz Wo Nicholas Sze commented on HDFS-7159: --- The patch nicely cleans up a lot of code. Good work! Some comments: - Storage policies are configurable. The current code expects that the LAZY_PERSIST poicy exists. Otherwise, there are NPE. Should we support the case that the LAZY_PERSIST policy is not configurated? - Should we add a conf to enable/disable the lazy presist feature? - In FSNamesystem.setStoragePolicyInt, the new checks for isCopyOnCreateFile should be moved FSDirectory.unprotectedSetStoragePolicy so that the path won't be resolved twice. Use block storage policy to set lazy persist preference --- Key: HDFS-7159 URL: https://issues.apache.org/jira/browse/HDFS-7159 Project: Hadoop HDFS Issue Type: Sub-task Components: namenode Reporter: Arpit Agarwal Assignee: Arpit Agarwal Attachments: HDFS-7159.01.patch Now that HDFS-6584 feature is ready and supports block storage policies on both files and directories, we can make use of Storage Policies to store the LAZY_PERSIST preference. This only affects how the preference is persisted in the FsImage/Edit logs. There is no change to the client API or to NN-DN interaction. An additional advantage is that can support setting the lazy persist preference on a directory and it will take effect for all new files. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[
https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14151931#comment-14151931
]
Arpit Agarwal commented on HDFS-7159:
-
Thanks for taking a look Nicholas.
You are right that we should handle it with a more user-friendly exception
instead of throwing NPE. I'll post an updated patch.
bq. Storage policies are configurable. The current code expects that the
LAZY_PERSIST poicy exists. Otherwise, there are NPE. Should we support the case
that the LAZY_PERSIST policy is not configurated?
What do you think of making it a system policy that cannot be overridden since
a feature depends on it? It may be unexpected for administrators that
reconfiguring storage policies could affect the feature. So when we add ability
to reconfigure storage policies, in {{BlockStoragePolicySuite}} we throw an
exception if there is an attempt to redefine the LAZY_PERSIST storage policy.
Use block storage policy to set lazy persist preference
---
Key: HDFS-7159
URL: https://issues.apache.org/jira/browse/HDFS-7159
Project: Hadoop HDFS
Issue Type: Sub-task
Components: namenode
Reporter: Arpit Agarwal
Assignee: Arpit Agarwal
Attachments: HDFS-7159.01.patch
Now that HDFS-6584 feature is ready and supports block storage policies on
both files and directories, we can make use of Storage Policies to store the
LAZY_PERSIST preference.
This only affects how the preference is persisted in the FsImage/Edit logs.
There is no change to the client API or to NN-DN interaction.
An additional advantage is that can support setting the lazy persist
preference on a directory and it will take effect for all new files.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[
https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14152104#comment-14152104
]
Arpit Agarwal commented on HDFS-7159:
-
Nicholas, updated patch to address your comments.
bq. Storage policies are configurable. The current code expects that the
LAZY_PERSIST poicy exists. Otherwise, there are NPE. Should we support the case
that the LAZY_PERSIST policy is not configurated?
Fixed.
bq. In FSNamesystem.setStoragePolicyInt, the new checks for isCopyOnCreateFile
should be moved FSDirectory.unprotectedSetStoragePolicy so that the path won't
be resolved twice.
Fixed.
bq. Should we add a conf to enable/disable the lazy presist feature?
For now the feature will not be used unless the administrator explicitly
configures RAM disk in DataNode configuration. For 2.6 we are looking at
integrating with the {{dfs.datanode.max.locked.memory}} setting. So if that
setting is not configured the feature would not be available. Sound okay?
Let me know what you think of making this policy non-configurable.
Thanks.
Use block storage policy to set lazy persist preference
---
Key: HDFS-7159
URL: https://issues.apache.org/jira/browse/HDFS-7159
Project: Hadoop HDFS
Issue Type: Sub-task
Components: namenode
Reporter: Arpit Agarwal
Assignee: Arpit Agarwal
Attachments: HDFS-7159.01.patch, HDFS-7159.02.patch
Now that HDFS-6584 feature is ready and supports block storage policies on
both files and directories, we can make use of Storage Policies to store the
LAZY_PERSIST preference.
This only affects how the preference is persisted in the FsImage/Edit logs.
There is no change to the client API or to NN-DN interaction.
An additional advantage is that can support setting the lazy persist
preference on a directory and it will take effect for all new files.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14152557#comment-14152557 ] Tsz Wo Nicholas Sze commented on HDFS-7159: --- bq. For now the feature will not be used unless the administrator explicitly configures RAM disk in DataNode configuration. For 2.6 we are looking at integrating with the dfs.datanode.max.locked.memory setting. So if that setting is not configured the feature would not be available. Sound okay? Sound good. What will happen if a user creates a file using the LAZY_PERSIST CreateFlag? +1 the new patch looks good. Use block storage policy to set lazy persist preference --- Key: HDFS-7159 URL: https://issues.apache.org/jira/browse/HDFS-7159 Project: Hadoop HDFS Issue Type: Sub-task Components: namenode Reporter: Arpit Agarwal Assignee: Arpit Agarwal Attachments: HDFS-7159.01.patch, HDFS-7159.02.patch, HDFS-7159.03.patch Now that HDFS-6584 feature is ready and supports block storage policies on both files and directories, we can make use of Storage Policies to store the LAZY_PERSIST preference. This only affects how the preference is persisted in the FsImage/Edit logs. There is no change to the client API or to NN-DN interaction. An additional advantage is that can support setting the lazy persist preference on a directory and it will take effect for all new files. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[
https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14152567#comment-14152567
]
Jing Zhao commented on HDFS-7159:
-
Thanks for working on this, Arpit! Some minor/nitty comments:
# In DFSOutputStream, instead of creating a default policy suite locally, it
may be better to use the RPC call {{getStoragePolicies}} so that the policy's
read-only semantic can be hidden on the server side.
# Nit: the instanciation of lpPolicy can be moved into the if block.
{code}
+BlockStoragePolicy lpPolicy =
+blockManager.getStoragePolicy(LAZY_PERSIST);
+
+BlockStoragePolicy effectivePolicy =
+blockManager.getStoragePolicy(inode.getStoragePolicyID());
+
+if (isLazyPersist) {
+ // Set LAZY_PERSIST storage policy if the flag was passed to
+ // CreateFile.
{code}
# This can addressed in a separate jira: looks like the changes in
{{BlockStoragePolicy#chooseStorageTypes}} depends on the assumption that the
transient storage types are listed before others. Can we have a more general
implementation here?
Use block storage policy to set lazy persist preference
---
Key: HDFS-7159
URL: https://issues.apache.org/jira/browse/HDFS-7159
Project: Hadoop HDFS
Issue Type: Sub-task
Components: namenode
Reporter: Arpit Agarwal
Assignee: Arpit Agarwal
Attachments: HDFS-7159.01.patch, HDFS-7159.02.patch,
HDFS-7159.03.patch
Now that HDFS-6584 feature is ready and supports block storage policies on
both files and directories, we can make use of Storage Policies to store the
LAZY_PERSIST preference.
This only affects how the preference is persisted in the FsImage/Edit logs.
There is no change to the client API or to NN-DN interaction.
An additional advantage is that can support setting the lazy persist
preference on a directory and it will take effect for all new files.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14152571#comment-14152571 ] Arpit Agarwal commented on HDFS-7159: - bq. Sound good. What will happen if a user creates a file using the LAZY_PERSIST CreateFlag? The flag will be ignored and the replicas will be created on fallback DISK. Same as when we are out of RAM Disk. Thanks for the reviews Nicholas! I think [~jingzhao] said he is also reviewing so will hold off committing for now. Use block storage policy to set lazy persist preference --- Key: HDFS-7159 URL: https://issues.apache.org/jira/browse/HDFS-7159 Project: Hadoop HDFS Issue Type: Sub-task Components: namenode Reporter: Arpit Agarwal Assignee: Arpit Agarwal Attachments: HDFS-7159.01.patch, HDFS-7159.02.patch, HDFS-7159.03.patch Now that HDFS-6584 feature is ready and supports block storage policies on both files and directories, we can make use of Storage Policies to store the LAZY_PERSIST preference. This only affects how the preference is persisted in the FsImage/Edit logs. There is no change to the client API or to NN-DN interaction. An additional advantage is that can support setting the lazy persist preference on a directory and it will take effect for all new files. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Commented] (HDFS-7159) Use block storage policy to set lazy persist preference
[ https://issues.apache.org/jira/browse/HDFS-7159?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14152810#comment-14152810 ] Jing Zhao commented on HDFS-7159: - bq. Would it be okay if we fixed it separately? Sure. +1 for the latest patch. Use block storage policy to set lazy persist preference --- Key: HDFS-7159 URL: https://issues.apache.org/jira/browse/HDFS-7159 Project: Hadoop HDFS Issue Type: Sub-task Components: namenode Reporter: Arpit Agarwal Assignee: Arpit Agarwal Attachments: HDFS-7159.01.patch, HDFS-7159.02.patch, HDFS-7159.03.patch, HDFS-7159.04.patch Now that HDFS-6584 feature is ready and supports block storage policies on both files and directories, we can make use of Storage Policies to store the LAZY_PERSIST preference. This only affects how the preference is persisted in the FsImage/Edit logs. There is no change to the client API or to NN-DN interaction. An additional advantage is that can support setting the lazy persist preference on a directory and it will take effect for all new files. -- This message was sent by Atlassian JIRA (v6.3.4#6332)
