HDFS-12637. Extend TestDistributedFileSystemWithECFile with a random EC policy. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bd70094 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bd70094 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bd70094 Branch: refs/heads/YARN-1011 Commit: 7bd700941d0a423d5232331fa19eab5fdab6a6fb Parents: 035c6ee Author: Xiao Chen <x...@apache.org> Authored: Mon Oct 16 09:54:37 2017 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Mon Oct 16 09:55:22 2017 -0700 ---------------------------------------------------------------------- .../TestDistributedFileSystemWithECFile.java | 37 +++++++++------ ...dFileSystemWithECFileWithRandomECPolicy.java | 49 ++++++++++++++++++++ 2 files changed, 73 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd70094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java index d4e01b7..14a2ec4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java @@ -39,34 +39,45 @@ import static org.junit.Assert.assertTrue; * FileSystem.listFiles for erasure coded files. */ public class TestDistributedFileSystemWithECFile { - private final ErasureCodingPolicy ecPolicy = - StripedFileTestUtil.getDefaultECPolicy(); - private final int cellSize = ecPolicy.getCellSize(); - private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); - private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); - private final int numDNs = dataBlocks + parityBlocks; - private final int stripesPerBlock = 4; - private final int blockSize = stripesPerBlock * cellSize; - private final int blockGroupSize = blockSize * dataBlocks; + private ErasureCodingPolicy ecPolicy; + private int cellSize; + private short dataBlocks; + private short parityBlocks; + private int numDNs; + private int stripesPerBlock; + private int blockSize; + private int blockGroupSize; private MiniDFSCluster cluster; private FileContext fileContext; private DistributedFileSystem fs; private Configuration conf = new HdfsConfiguration(); + public ErasureCodingPolicy getEcPolicy() { + return StripedFileTestUtil.getDefaultECPolicy(); + } + @Before public void setup() throws IOException { + ecPolicy = getEcPolicy(); + cellSize = ecPolicy.getCellSize(); + dataBlocks = (short) ecPolicy.getNumDataUnits(); + parityBlocks = (short) ecPolicy.getNumParityUnits(); + numDNs = dataBlocks + parityBlocks; + stripesPerBlock = 4; + blockSize = stripesPerBlock * cellSize; + blockGroupSize = blockSize * dataBlocks; + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); fileContext = FileContext.getFileContext(cluster.getURI(0), conf); fs = cluster.getFileSystem(); - fs.enableErasureCodingPolicy( - StripedFileTestUtil.getDefaultECPolicy().getName()); + fs.enableErasureCodingPolicy(ecPolicy.getName()); fs.mkdirs(new Path("/ec")); cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", - StripedFileTestUtil.getDefaultECPolicy().getName()); + ecPolicy.getName()); } @After @@ -121,7 +132,7 @@ public class TestDistributedFileSystemWithECFile { @Test(timeout=60000) public void testListECFilesSmallerThanOneStripe() throws Exception { - int dataBlocksNum = 3; + int dataBlocksNum = dataBlocks; createFile("/ec/smallstripe", cellSize * dataBlocksNum); RemoteIterator<LocatedFileStatus> iter = cluster.getFileSystem().listFiles(new Path("/ec"), true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd70094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java new file mode 100644 index 0000000..afa7569 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This test extends TestDistributedFileSystemWithECFile to use a random + * (non-default) EC policy. + */ +public class TestDistributedFileSystemWithECFileWithRandomECPolicy extends + TestDistributedFileSystemWithECFile { + private static final Logger LOG = LoggerFactory.getLogger( + TestDistributedFileSystemWithECFileWithRandomECPolicy.class); + + private ErasureCodingPolicy ecPolicy; + + public TestDistributedFileSystemWithECFileWithRandomECPolicy() { + // If you want to debug this test with a specific ec policy, please use + // SystemErasureCodingPolicies class. + // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID); + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info("run {} with {}.", + TestDistributedFileSystemWithECFileWithRandomECPolicy.class + .getSuperclass().getSimpleName(), ecPolicy.getName()); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
