HDFS-12505. Extend TestFileStatusWithECPolicy with a random EC policy. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8211a3d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8211a3d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8211a3d4 Branch: refs/heads/HDFS-7240 Commit: 8211a3d4693fea46cff11c5883c16a9b4df7b4de Parents: f82d38d Author: Xiao Chen <x...@apache.org> Authored: Tue Mar 13 10:48:35 2018 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Tue Mar 13 10:48:45 2018 -0700 ---------------------------------------------------------------------- .../hdfs/TestFileStatusWithDefaultECPolicy.java | 107 +++++++++++++++++++ .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 102 ------------------ .../hdfs/TestFileStatusWithRandomECPolicy.java | 49 +++++++++ 3 files changed, 156 insertions(+), 102 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8211a3d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithDefaultECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithDefaultECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithDefaultECPolicy.java new file mode 100644 index 0000000..a57777a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithDefaultECPolicy.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +/** + * This test ensures the statuses of EC files with the default policy. + */ +public class TestFileStatusWithDefaultECPolicy { + private MiniDFSCluster cluster; + private DistributedFileSystem fs; + private DFSClient client; + + @Rule + public Timeout globalTimeout = new Timeout(300000); + + @Before + public void before() throws IOException { + HdfsConfiguration conf = new HdfsConfiguration(); + cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + client = fs.getClient(); + fs.enableErasureCodingPolicy(getEcPolicy().getName()); + } + + @After + public void after() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + public ErasureCodingPolicy getEcPolicy() { + return StripedFileTestUtil.getDefaultECPolicy(); + } + + @Test + public void testFileStatusWithECPolicy() throws Exception { + // test directory doesn't have an EC policy + final Path dir = new Path("/foo"); + assertTrue(fs.mkdir(dir, FsPermission.getDirDefault())); + ContractTestUtils.assertNotErasureCoded(fs, dir); + assertNull(client.getFileInfo(dir.toString()).getErasureCodingPolicy()); + // test file doesn't have an EC policy + final Path file = new Path(dir, "foo"); + fs.create(file).close(); + assertNull(client.getFileInfo(file.toString()).getErasureCodingPolicy()); + ContractTestUtils.assertNotErasureCoded(fs, file); + fs.delete(file, true); + + final ErasureCodingPolicy ecPolicy1 = getEcPolicy(); + // set EC policy on dir + fs.setErasureCodingPolicy(dir, ecPolicy1.getName()); + ContractTestUtils.assertErasureCoded(fs, dir); + final ErasureCodingPolicy ecPolicy2 = + client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); + assertNotNull(ecPolicy2); + assertTrue(ecPolicy1.equals(ecPolicy2)); + + // test file with EC policy + fs.create(file).close(); + final ErasureCodingPolicy ecPolicy3 = + fs.getClient().getFileInfo(file.toUri().getPath()) + .getErasureCodingPolicy(); + assertNotNull(ecPolicy3); + assertTrue(ecPolicy1.equals(ecPolicy3)); + ContractTestUtils.assertErasureCoded(fs, file); + FileStatus status = fs.getFileStatus(file); + assertTrue(file + " should have erasure coding set in " + + "FileStatus#toString(): " + status, + status.toString().contains("isErasureCoded=true")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8211a3d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java deleted file mode 100644 index 077cf3a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -public class TestFileStatusWithECPolicy { - private MiniDFSCluster cluster; - private DistributedFileSystem fs; - private DFSClient client; - - @Rule - public Timeout globalTimeout = new Timeout(300000); - - @Before - public void before() throws IOException { - HdfsConfiguration conf = new HdfsConfiguration(); - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - fs = cluster.getFileSystem(); - client = fs.getClient(); - fs.enableErasureCodingPolicy( - StripedFileTestUtil.getDefaultECPolicy().getName()); - } - - @After - public void after() { - if (cluster != null) { - cluster.shutdown(); - cluster = null; - } - } - - @Test - public void testFileStatusWithECPolicy() throws Exception { - // test directory doesn't have an EC policy - final Path dir = new Path("/foo"); - assertTrue(fs.mkdir(dir, FsPermission.getDirDefault())); - ContractTestUtils.assertNotErasureCoded(fs, dir); - assertNull(client.getFileInfo(dir.toString()).getErasureCodingPolicy()); - // test file doesn't have an EC policy - final Path file = new Path(dir, "foo"); - fs.create(file).close(); - assertNull(client.getFileInfo(file.toString()).getErasureCodingPolicy()); - ContractTestUtils.assertNotErasureCoded(fs, file); - fs.delete(file, true); - - final ErasureCodingPolicy ecPolicy1 = - StripedFileTestUtil.getDefaultECPolicy(); - // set EC policy on dir - fs.setErasureCodingPolicy(dir, ecPolicy1.getName()); - ContractTestUtils.assertErasureCoded(fs, dir); - final ErasureCodingPolicy ecPolicy2 = - client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); - assertNotNull(ecPolicy2); - assertTrue(ecPolicy1.equals(ecPolicy2)); - - // test file with EC policy - fs.create(file).close(); - final ErasureCodingPolicy ecPolicy3 = - fs.getClient().getFileInfo(file.toUri().getPath()) - .getErasureCodingPolicy(); - assertNotNull(ecPolicy3); - assertTrue(ecPolicy1.equals(ecPolicy3)); - ContractTestUtils.assertErasureCoded(fs, file); - FileStatus status = fs.getFileStatus(file); - assertTrue(file + " should have erasure coding set in " + - "FileStatus#toString(): " + status, - status.toString().contains("isErasureCoded=true")); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8211a3d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithRandomECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithRandomECPolicy.java new file mode 100644 index 0000000..18902a7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithRandomECPolicy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This test extends TestFileStatusWithDefaultECPolicy to use a random + * (non-default) EC policy. + */ +public class TestFileStatusWithRandomECPolicy extends + TestFileStatusWithDefaultECPolicy { + private static final Logger LOG = LoggerFactory.getLogger( + TestFileStatusWithRandomECPolicy.class); + + private ErasureCodingPolicy ecPolicy; + + public TestFileStatusWithRandomECPolicy() { + // If you want to debug this test with a specific ec policy, please use + // SystemErasureCodingPolicies class. + // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID); + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info("run {} with {}.", + TestFileStatusWithRandomECPolicy.class + .getSuperclass().getSimpleName(), ecPolicy.getName()); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
