HDFS-11598. Improve -setrep for Erasure Coded files. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbd68478 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbd68478 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbd68478 Branch: refs/heads/YARN-2915 Commit: bbd68478d5743b3b2911bf3febed7daa89479e45 Parents: bc7aff7 Author: Wei-Chiu Chuang <weic...@apache.org> Authored: Mon Apr 3 07:57:28 2017 -0700 Committer: Wei-Chiu Chuang <weic...@apache.org> Committed: Mon Apr 3 07:57:28 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/fs/shell/SetReplication.java | 17 ++++++-- .../hadoop/hdfs/TestSetrepIncreasing.java | 44 ++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbd68478/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java index 2231c58..16e6e92 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java @@ -85,11 +85,20 @@ class SetReplication extends FsCommand { } if (item.stat.isFile()) { - if (!item.fs.setReplication(item.path, newRep)) { - throw new IOException("Could not set replication for: " + item); + // Do the checking if the file is erasure coded since + // replication factor for an EC file is meaningless. + if (!item.stat.isErasureCoded()) { + if (!item.fs.setReplication(item.path, newRep)) { + throw new IOException("Could not set replication for: " + item); + } + out.println("Replication " + newRep + " set: " + item); + if (waitOpt) { + waitList.add(item); + } + } else { + out.println("Did not set replication for: " + item + + ", because it's an erasure coded file."); } - out.println("Replication " + newRep + " set: " + item); - if (waitOpt) waitList.add(item); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbd68478/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java index fee30b5..50d7b27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; @@ -28,6 +30,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.junit.Test; @@ -102,4 +105,45 @@ public class TestSetrepIncreasing { cluster.shutdown(); } } + + @Test + public void testSetRepOnECFile() throws Exception { + ClientProtocol client; + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .build(); + cluster.waitActive(); + client = NameNodeProxies.createProxy(conf, + cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); + client.setErasureCodingPolicy("/", + StripedFileTestUtil.getDefaultECPolicy().getName()); + + FileSystem dfs = cluster.getFileSystem(); + try { + Path d = new Path("/tmp"); + dfs.mkdirs(d); + Path f = new Path(d, "foo"); + dfs.createNewFile(f); + FileStatus file = dfs.getFileStatus(f); + assertTrue(file.isErasureCoded()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + String[] args = {"-setrep", "2", "" + f}; + FsShell shell = new FsShell(); + shell.setConf(conf); + assertEquals(0, shell.run(args)); + assertTrue( + out.toString().contains("Did not set replication for: /tmp/foo")); + + // verify the replication factor of the EC file + file = dfs.getFileStatus(f); + assertEquals(1, file.getReplication()); + } finally { + dfs.close(); + cluster.shutdown(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
