HDFS-11823. Extend TestDFSStripedIutputStream/TestDFSStripedOutputStream with a random EC policy. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c8dd6d3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c8dd6d3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c8dd6d3 Branch: refs/heads/YARN-1011 Commit: 1c8dd6d3d10773f281538e1dea0ffdca9db34bfe Parents: dcf4559 Author: Jing Zhao <ji...@apache.org> Authored: Wed May 24 11:14:19 2017 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Wed May 24 11:14:19 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 12 ++++++ .../TestDFSRSDefault10x4StripedInputStream.java | 34 --------------- ...TestDFSRSDefault10x4StripedOutputStream.java | 35 --------------- ...fault10x4StripedOutputStreamWithFailure.java | 35 --------------- ...DFSStripedInputStreamWithRandomECPolicy.java | 45 ++++++++++++++++++++ ...tputStreamWithFailureWithRandomECPolicy.java | 45 ++++++++++++++++++++ ...FSStripedOutputStreamWithRandomECPolicy.java | 45 ++++++++++++++++++++ .../hdfs/TestDFSXORStripedInputStream.java | 32 -------------- .../hdfs/TestDFSXORStripedOutputStream.java | 34 --------------- ...estDFSXORStripedOutputStreamWithFailure.java | 35 --------------- 10 files changed, 147 insertions(+), 205 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index 1bab5db..057e94a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; @@ -568,4 +569,15 @@ public class StripedFileTestUtil { public static ErasureCodingPolicy getDefaultECPolicy() { return SystemErasureCodingPolicies.getPolicies().get(0); } + + /** + * Get non-default Erasure Coding Policy randomly. + * @return ErasureCodingPolicy + */ + public static ErasureCodingPolicy getRandomNonDefaultECPolicy() { + Random rand = new Random(); + List<ErasureCodingPolicy> policies = SystemErasureCodingPolicies + .getPolicies(); + return policies.get(1 + rand.nextInt(policies.size() - 1)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java deleted file mode 100644 index 1d09a6c..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; - -/** - * This tests read operation of DFS striped file with RS-10-4-64k - * erasure code policy. - */ -public class TestDFSRSDefault10x4StripedInputStream extends - TestDFSStripedInputStream { - - public ErasureCodingPolicy getEcPolicy() { - return SystemErasureCodingPolicies.getByID( - SystemErasureCodingPolicies.RS_10_4_POLICY_ID); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java deleted file mode 100644 index 080033d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; - -/** - * This tests write operation of DFS striped file with RS-10-4-64k - * erasure code policy. - */ -public class TestDFSRSDefault10x4StripedOutputStream - extends TestDFSStripedOutputStream { - - @Override - public ErasureCodingPolicy getEcPolicy() { - return SystemErasureCodingPolicies.getByID( - SystemErasureCodingPolicies.RS_10_4_POLICY_ID); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java deleted file mode 100644 index 5de9dd3..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; - -/** - * This tests write operation of DFS striped file with RS-10-4-64k - * erasure code policy under Datanode failure conditions. - */ -public class TestDFSRSDefault10x4StripedOutputStreamWithFailure - extends TestDFSStripedOutputStreamWithFailure { - - @Override - public ErasureCodingPolicy getEcPolicy() { - return SystemErasureCodingPolicies.getByID( - SystemErasureCodingPolicies.RS_10_4_POLICY_ID); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStreamWithRandomECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStreamWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStreamWithRandomECPolicy.java new file mode 100644 index 0000000..568b018 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStreamWithRandomECPolicy.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; + +/** + * This tests read operation of DFS striped file with a random erasure code + * policy except for the default policy. + */ +public class TestDFSStripedInputStreamWithRandomECPolicy extends + TestDFSStripedInputStream { + + private static final Log LOG = LogFactory.getLog( + TestDFSStripedInputStreamWithRandomECPolicy.class.getName()); + + private ErasureCodingPolicy ecPolicy; + + public TestDFSStripedInputStreamWithRandomECPolicy() { + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info(ecPolicy); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java new file mode 100644 index 0000000..7fc5233 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureWithRandomECPolicy.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; + +/** + * This tests write operation of DFS striped file with a random erasure code + * policy except for the default policy under Datanode failure conditions. + */ +public class TestDFSStripedOutputStreamWithFailureWithRandomECPolicy extends + TestDFSStripedOutputStreamWithFailure { + + private static final Log LOG = LogFactory.getLog( + TestDFSStripedOutputStreamWithRandomECPolicy.class.getName()); + + private ErasureCodingPolicy ecPolicy; + + public TestDFSStripedOutputStreamWithFailureWithRandomECPolicy() { + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info(ecPolicy); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithRandomECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithRandomECPolicy.java new file mode 100644 index 0000000..9a783cd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithRandomECPolicy.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; + +/** + * This tests write operation of DFS striped file with a random erasure code + * policy except for the default policy. + */ +public class TestDFSStripedOutputStreamWithRandomECPolicy extends + TestDFSStripedOutputStream { + + private static final Log LOG = LogFactory.getLog( + TestDFSStripedOutputStreamWithRandomECPolicy.class.getName()); + + private ErasureCodingPolicy ecPolicy; + + public TestDFSStripedOutputStreamWithRandomECPolicy() { + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info(ecPolicy); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java deleted file mode 100644 index d19093d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; - -/** - * This tests read operation of DFS striped file with XOR-2-1-64k erasure code - * policy. - */ -public class TestDFSXORStripedInputStream extends TestDFSStripedInputStream{ - - public ErasureCodingPolicy getEcPolicy() { - return SystemErasureCodingPolicies.getByID( - SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java deleted file mode 100644 index ba620b8..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; - -/** - * This tests write operation of DFS striped file with XOR-2-1-64k erasure code - * policy. - */ -public class TestDFSXORStripedOutputStream extends TestDFSStripedOutputStream{ - - @Override - public ErasureCodingPolicy getEcPolicy() { - return SystemErasureCodingPolicies.getByID( - SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8dd6d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java deleted file mode 100644 index d9ab3d5..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; - -/** - * This tests write operation of DFS striped file with XOR-2-1-64k erasure code - * policy when there is data node failure. - */ -public class TestDFSXORStripedOutputStreamWithFailure - extends TestDFSStripedOutputStreamWithFailure{ - - @Override - public ErasureCodingPolicy getEcPolicy() { - return SystemErasureCodingPolicies.getByID( - SystemErasureCodingPolicies.XOR_2_1_POLICY_ID); - } -} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org