Repository: hadoop Updated Branches: refs/heads/branch-3.0 1b6de3fac -> 1c96e0d69
HDFS-12682. ECAdmin -listPolicies will always show SystemErasureCodingPolicies state as DISABLED. (cherry picked from commit 4f4cea56b25d5acc14dd6d61dea70fd4a0c7a9d9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c96e0d6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c96e0d6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c96e0d6 Branch: refs/heads/branch-3.0 Commit: 1c96e0d69397bb687e26036784a7b656cdeadc39 Parents: 1b6de3f Author: Xiao Chen <x...@apache.org> Authored: Thu Nov 2 21:26:45 2017 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Thu Nov 2 21:27:43 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +- .../hadoop/hdfs/DistributedFileSystem.java | 3 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 4 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hdfs/protocol/ErasureCodingPolicy.java | 62 ++------- .../hdfs/protocol/ErasureCodingPolicyInfo.java | 106 ++++++++++++++++ .../hdfs/protocol/ErasureCodingPolicyState.java | 4 +- .../ClientNamenodeProtocolTranslatorPB.java | 13 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 57 +++++++-- .../protocol/TestErasureCodingPolicyInfo.java | 72 +++++++++++ ...tNamenodeProtocolServerSideTranslatorPB.java | 8 +- .../federation/router/RouterRpcServer.java | 4 +- .../namenode/ErasureCodingPolicyManager.java | 126 ++++++++++++------- .../server/namenode/FSDirErasureCodingOp.java | 7 +- .../server/namenode/FSImageFormatProtobuf.java | 10 +- .../hdfs/server/namenode/FSNamesystem.java | 7 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 4 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 16 +++ .../hadoop/hdfs/TestErasureCodingPolicies.java | 22 ++-- .../server/namenode/TestFSEditLogLoader.java | 12 +- .../hdfs/server/namenode/TestFSImage.java | 17 +-- .../java/org/apache/hadoop/fs/TestDFSIO.java | 16 +-- 23 files changed, 421 insertions(+), 162 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 8d51a9c..467d6be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -119,6 +119,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; @@ -2793,7 +2794,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } } - public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { + public ErasureCodingPolicyInfo[] getErasureCodingPolicies() + throws IOException { checkOpen(); try (TraceScope ignored = tracer.newScope("getErasureCodingPolicies")) { return namenode.getErasureCodingPolicies(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index a8a5cfa..a90c95d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -75,6 +75,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction; @@ -2622,7 +2623,7 @@ public class DistributedFileSystem extends FileSystem { * @return all erasure coding policies supported by this file system. * @throws IOException */ - public Collection<ErasureCodingPolicy> getAllErasureCodingPolicies() + public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies() throws IOException { return Arrays.asList(dfs.getErasureCodingPolicies()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 8dc5810..9116167 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; @@ -537,7 +538,8 @@ public class HdfsAdmin { * * @throws IOException */ - public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { + public ErasureCodingPolicyInfo[] getErasureCodingPolicies() + throws IOException { return dfs.getClient().getErasureCodingPolicies(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 25f63bd..f61ec75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1619,7 +1619,7 @@ public interface ClientProtocol { * @throws IOException */ @Idempotent - ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException; + ErasureCodingPolicyInfo[] getErasureCodingPolicies() throws IOException; /** * Get the erasure coding codecs loaded in Namenode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java index 6cbe0e6..39489b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java @@ -21,7 +21,6 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; @@ -29,22 +28,23 @@ import java.io.Serializable; /** * A policy about how to write/read/code an erasure coding file. + * <p> + * Note this class should be lightweight and immutable, because it's cached + * by {@link SystemErasureCodingPolicies}, to be returned as a part of + * {@link HdfsFileStatus}. */ -@InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceAudience.Private public final class ErasureCodingPolicy implements Serializable { private static final long serialVersionUID = 0x0079fe4e; - private String name; + private final String name; private final ECSchema schema; private final int cellSize; - private byte id; - private ErasureCodingPolicyState state; - + private final byte id; public ErasureCodingPolicy(String name, ECSchema schema, - int cellSize, byte id, ErasureCodingPolicyState state) { + int cellSize, byte id) { Preconditions.checkNotNull(name); Preconditions.checkNotNull(schema); Preconditions.checkArgument(cellSize > 0, "cellSize must be positive"); @@ -54,22 +54,14 @@ public final class ErasureCodingPolicy implements Serializable { this.schema = schema; this.cellSize = cellSize; this.id = id; - this.state = state; - } - - public ErasureCodingPolicy(String name, ECSchema schema, int cellSize, - byte id) { - this(name, schema, cellSize, id, ErasureCodingPolicyState.DISABLED); } public ErasureCodingPolicy(ECSchema schema, int cellSize, byte id) { - this(composePolicyName(schema, cellSize), schema, cellSize, id, - ErasureCodingPolicyState.DISABLED); + this(composePolicyName(schema, cellSize), schema, cellSize, id); } public ErasureCodingPolicy(ECSchema schema, int cellSize) { - this(composePolicyName(schema, cellSize), schema, cellSize, (byte) -1, - ErasureCodingPolicyState.DISABLED); + this(composePolicyName(schema, cellSize), schema, cellSize, (byte) -1); } public static String composePolicyName(ECSchema schema, int cellSize) { @@ -86,10 +78,6 @@ public final class ErasureCodingPolicy implements Serializable { return name; } - public void setName(String name) { - this.name = name; - } - public ECSchema getSchema() { return schema; } @@ -114,39 +102,14 @@ public final class ErasureCodingPolicy implements Serializable { return id; } - public void setId(byte id) { - this.id = id; - } - - public boolean isReplicationPolicy() { return (id == ErasureCodeConstants.REPLICATION_POLICY_ID); } - public ErasureCodingPolicyState getState() { - return state; - } - - public void setState(ErasureCodingPolicyState state) { - this.state = state; - } - public boolean isSystemPolicy() { return (this.id < ErasureCodeConstants.USER_DEFINED_POLICY_START_ID); } - public boolean isEnabled() { - return (this.state == ErasureCodingPolicyState.ENABLED); - } - - public boolean isDisabled() { - return (this.state == ErasureCodingPolicyState.DISABLED); - } - - public boolean isRemoved() { - return (this.state == ErasureCodingPolicyState.REMOVED); - } - @Override public boolean equals(Object o) { if (o == null) { @@ -164,7 +127,6 @@ public final class ErasureCodingPolicy implements Serializable { .append(schema, rhs.schema) .append(cellSize, rhs.cellSize) .append(id, rhs.id) - .append(state, rhs.state) .isEquals(); } @@ -175,7 +137,6 @@ public final class ErasureCodingPolicy implements Serializable { .append(schema) .append(cellSize) .append(id) - .append(state) .toHashCode(); } @@ -184,8 +145,7 @@ public final class ErasureCodingPolicy implements Serializable { return "ErasureCodingPolicy=[" + "Name=" + name + ", " + "Schema=[" + schema.toString() + "], " + "CellSize=" + cellSize + ", " - + "Id=" + id + ", " - + "State=" + state.toString() + + "Id=" + id + "]"; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java new file mode 100644 index 0000000..a5b95cb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience; + +import java.io.Serializable; + +/** + * HDFS internal presentation of a {@link ErasureCodingPolicy}. Also contains + * additional information such as {@link ErasureCodingPolicyState}. + */ +@InterfaceAudience.Private +public class ErasureCodingPolicyInfo implements Serializable { + + private static final long serialVersionUID = 0x31; + + private final ErasureCodingPolicy policy; + private ErasureCodingPolicyState state; + + public ErasureCodingPolicyInfo(final ErasureCodingPolicy thePolicy, + final ErasureCodingPolicyState theState) { + Preconditions.checkNotNull(thePolicy); + Preconditions.checkNotNull(theState); + policy = thePolicy; + state = theState; + } + + public ErasureCodingPolicyInfo(final ErasureCodingPolicy thePolicy) { + this(thePolicy, ErasureCodingPolicyState.DISABLED); + } + + public ErasureCodingPolicy getPolicy() { + return policy; + } + + public ErasureCodingPolicyState getState() { + return state; + } + + public void setState(final ErasureCodingPolicyState newState) { + Preconditions.checkNotNull(newState, "New state should not be null."); + state = newState; + } + + public boolean isEnabled() { + return (this.state == ErasureCodingPolicyState.ENABLED); + } + + public boolean isDisabled() { + return (this.state == ErasureCodingPolicyState.DISABLED); + } + + public boolean isRemoved() { + return (this.state == ErasureCodingPolicyState.REMOVED); + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + if (o == this) { + return true; + } + if (o.getClass() != getClass()) { + return false; + } + ErasureCodingPolicyInfo rhs = (ErasureCodingPolicyInfo) o; + return new EqualsBuilder() + .append(policy, rhs.policy) + .append(state, rhs.state) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(303855623, 582626729) + .append(policy) + .append(state) + .toHashCode(); + } + + @Override + public String toString() { + return policy.toString() + ", State=" + state.toString(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyState.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyState.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyState.java index a8586dc..b045926 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyState.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; import java.io.DataInput; import java.io.DataOutput; @@ -27,8 +26,7 @@ import java.io.IOException; /** * Value denotes the possible states of an ErasureCodingPolicy. */ -@InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceAudience.Private public enum ErasureCodingPolicyState { /** Policy is disabled. It's policy default state. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 0b443c6..aef7c1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction; @@ -1782,17 +1783,17 @@ public class ClientNamenodeProtocolTranslatorPB implements } @Override - public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { + public ErasureCodingPolicyInfo[] getErasureCodingPolicies() + throws IOException { try { GetErasureCodingPoliciesResponseProto response = rpcProxy .getErasureCodingPolicies(null, VOID_GET_EC_POLICIES_REQUEST); - ErasureCodingPolicy[] ecPolicies = - new ErasureCodingPolicy[response.getEcPoliciesCount()]; + ErasureCodingPolicyInfo[] ecPolicies = + new ErasureCodingPolicyInfo[response.getEcPoliciesCount()]; int i = 0; - for (ErasureCodingPolicyProto ecPolicyProto : - response.getEcPoliciesList()) { + for (ErasureCodingPolicyProto proto : response.getEcPoliciesList()) { ecPolicies[i++] = - PBHelperClient.convertErasureCodingPolicy(ecPolicyProto); + PBHelperClient.convertErasureCodingPolicyInfo(proto); } return ecPolicies; } catch (ServiceException e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index db97031..c0df325 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -55,6 +55,7 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.inotify.Event; import org.apache.hadoop.hdfs.inotify.EventBatch; import org.apache.hadoop.hdfs.inotify.EventBatchList; @@ -79,6 +80,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FsPermissionExtension; @@ -2936,6 +2938,9 @@ public class PBHelperClient { return HdfsProtos.ErasureCodingPolicyState.valueOf(state.getValue()); } + /** + * Convert the protobuf to a {@link ErasureCodingPolicy}. + */ public static ErasureCodingPolicy convertErasureCodingPolicy( ErasureCodingPolicyProto proto) { final byte id = (byte) (proto.getId() & 0xFF); @@ -2949,28 +2954,62 @@ public class PBHelperClient { "Missing schema field in ErasureCodingPolicy proto"); Preconditions.checkArgument(proto.hasCellSize(), "Missing cellsize field in ErasureCodingPolicy proto"); - Preconditions.checkArgument(proto.hasState(), - "Missing state field in ErasureCodingPolicy proto"); return new ErasureCodingPolicy(proto.getName(), convertECSchema(proto.getSchema()), - proto.getCellSize(), id, convertECState(proto.getState())); + proto.getCellSize(), id); } return policy; } - public static ErasureCodingPolicyProto convertErasureCodingPolicy( + /** + * Convert the protobuf to a {@link ErasureCodingPolicyInfo}. This should only + * be needed when the caller is interested in the state of the policy. + */ + public static ErasureCodingPolicyInfo convertErasureCodingPolicyInfo( + ErasureCodingPolicyProto proto) { + ErasureCodingPolicy policy = convertErasureCodingPolicy(proto); + ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy); + Preconditions.checkArgument(proto.hasState(), + "Missing state field in ErasureCodingPolicy proto"); + info.setState(convertECState(proto.getState())); + return info; + } + + private static ErasureCodingPolicyProto.Builder createECPolicyProtoBuilder( ErasureCodingPolicy policy) { - ErasureCodingPolicyProto.Builder builder = ErasureCodingPolicyProto - .newBuilder() - .setId(policy.getId()); + final ErasureCodingPolicyProto.Builder builder = + ErasureCodingPolicyProto.newBuilder().setId(policy.getId()); // If it's not a built-in policy, need to set the optional fields. if (SystemErasureCodingPolicies.getByID(policy.getId()) == null) { builder.setName(policy.getName()) .setSchema(convertECSchema(policy.getSchema())) - .setCellSize(policy.getCellSize()) - .setState(convertECState(policy.getState())); + .setCellSize(policy.getCellSize()); } + return builder; + } + + /** + * Convert a {@link ErasureCodingPolicy} to protobuf. + * This means no state of the policy will be set on the protobuf. + */ + public static ErasureCodingPolicyProto convertErasureCodingPolicy( + ErasureCodingPolicy policy) { + return createECPolicyProtoBuilder(policy).build(); + } + + /** + * Convert a {@link ErasureCodingPolicyInfo} to protobuf. + * The protobuf will have the policy, and state. State is relevant when: + * 1. Persisting a policy to fsimage + * 2. Returning the policy to the RPC call + * {@link DistributedFileSystem#getAllErasureCodingPolicies()} + */ + public static ErasureCodingPolicyProto convertErasureCodingPolicy( + ErasureCodingPolicyInfo info) { + final ErasureCodingPolicyProto.Builder builder = + createECPolicyProtoBuilder(info.getPolicy()); + builder.setState(convertECState(info.getState())); return builder.build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicyInfo.java new file mode 100644 index 0000000..32518e0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicyInfo.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.junit.Test; + +import static org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies.RS_6_3_POLICY_ID; +import static org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState.DISABLED; +import static org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState.ENABLED; +import static org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState.REMOVED; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test {@link ErasureCodingPolicyInfo}. + */ +public class TestErasureCodingPolicyInfo { + + @Test + public void testPolicyAndStateCantBeNull() { + try { + new ErasureCodingPolicyInfo(null); + fail("Null policy should fail"); + } catch (NullPointerException expected) { + } + + try { + new ErasureCodingPolicyInfo(SystemErasureCodingPolicies + .getByID(RS_6_3_POLICY_ID), null); + fail("Null policy should fail"); + } catch (NullPointerException expected) { + } + } + + @Test + public void testStates() { + ErasureCodingPolicyInfo info = + new ErasureCodingPolicyInfo(SystemErasureCodingPolicies + .getByID(RS_6_3_POLICY_ID)); + + info.setState(ENABLED); + assertFalse(info.isDisabled()); + assertTrue(info.isEnabled()); + assertFalse(info.isRemoved()); + + info.setState(REMOVED); + assertFalse(info.isDisabled()); + assertFalse(info.isEnabled()); + assertTrue(info.isRemoved()); + + info.setState(DISABLED); + assertTrue(info.isDisabled()); + assertFalse(info.isEnabled()); + assertFalse(info.isRemoved()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 0527b3b..dee3f25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus; @@ -1682,11 +1683,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements public GetErasureCodingPoliciesResponseProto getErasureCodingPolicies(RpcController controller, GetErasureCodingPoliciesRequestProto request) throws ServiceException { try { - ErasureCodingPolicy[] ecPolicies = server.getErasureCodingPolicies(); + ErasureCodingPolicyInfo[] ecpInfos = server.getErasureCodingPolicies(); GetErasureCodingPoliciesResponseProto.Builder resBuilder = GetErasureCodingPoliciesResponseProto .newBuilder(); - for (ErasureCodingPolicy ecPolicy : ecPolicies) { - resBuilder.addEcPolicies(PBHelperClient.convertErasureCodingPolicy(ecPolicy)); + for (ErasureCodingPolicyInfo info : ecpInfos) { + resBuilder.addEcPolicies( + PBHelperClient.convertErasureCodingPolicy(info)); } return resBuilder.build(); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 53448ec..ed91931 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction; @@ -1785,7 +1786,8 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol { } @Override - public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { + public ErasureCodingPolicyInfo[] getErasureCodingPolicies() + throws IOException { checkOperation(OperationCategory.READ, false); return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 62c7f60..3a310da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -23,6 +23,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -65,18 +66,18 @@ public final class ErasureCodingPolicyManager { * All policies sorted by name for fast querying, include built-in policy, * user defined policy, removed policy. */ - private Map<String, ErasureCodingPolicy> policiesByName; + private Map<String, ErasureCodingPolicyInfo> policiesByName; /** * All policies sorted by ID for fast querying, including built-in policy, * user defined policy, removed policy. */ - private Map<Byte, ErasureCodingPolicy> policiesByID; + private Map<Byte, ErasureCodingPolicyInfo> policiesByID; /** * For better performance when query all Policies. */ - private ErasureCodingPolicy[] allPolicies; + private ErasureCodingPolicyInfo[] allPolicies; /** * All enabled policies sorted by name for fast querying, including built-in @@ -120,15 +121,17 @@ public final class ErasureCodingPolicyManager { */ for (ErasureCodingPolicy policy : SystemErasureCodingPolicies.getPolicies()) { - policiesByName.put(policy.getName(), policy); - policiesByID.put(policy.getId(), policy); + final ErasureCodingPolicyInfo info = new ErasureCodingPolicyInfo(policy); + policiesByName.put(policy.getName(), info); + policiesByID.put(policy.getId(), info); } - if (!defaultPolicyName.trim().isEmpty()) { - ErasureCodingPolicy ecPolicy = policiesByName.get(defaultPolicyName); - if (ecPolicy == null) { + if (!defaultPolicyName.isEmpty()) { + final ErasureCodingPolicyInfo info = + policiesByName.get(defaultPolicyName); + if (info == null) { String names = policiesByName.values() - .stream().map(ErasureCodingPolicy::getName) + .stream().map((pi) -> pi.getPolicy().getName()) .collect(Collectors.joining(", ")); String msg = String.format("EC policy '%s' specified at %s is not a " + "valid policy. Please choose from list of available " @@ -138,11 +141,13 @@ public final class ErasureCodingPolicyManager { names); throw new HadoopIllegalArgumentException(msg); } - enabledPoliciesByName.put(ecPolicy.getName(), ecPolicy); + info.setState(ErasureCodingPolicyState.ENABLED); + enabledPoliciesByName.put(info.getPolicy().getName(), info.getPolicy()); } enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); - allPolicies = policiesByName.values().toArray(new ErasureCodingPolicy[0]); + allPolicies = + policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]); maxCellSize = conf.getInt( DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY, @@ -190,23 +195,50 @@ public final class ErasureCodingPolicyManager { * Get all system defined policies and user defined policies. * @return all policies */ - public ErasureCodingPolicy[] getPolicies() { + public ErasureCodingPolicyInfo[] getPolicies() { return allPolicies; } /** - * Get a policy by policy ID, including system policy and user defined policy. + * Get a {@link ErasureCodingPolicy} by policy ID, including system policy + * and user defined policy. * @return ecPolicy, or null if not found */ public ErasureCodingPolicy getByID(byte id) { + final ErasureCodingPolicyInfo ecpi = getPolicyInfoByID(id); + if (ecpi == null) { + return null; + } + return ecpi.getPolicy(); + } + + /** + * Get a {@link ErasureCodingPolicyInfo} by policy ID, including system policy + * and user defined policy. + */ + private ErasureCodingPolicyInfo getPolicyInfoByID(final byte id) { return this.policiesByID.get(id); } /** - * Get a policy by policy ID, including system policy and user defined policy. + * Get a {@link ErasureCodingPolicy} by policy name, including system + * policy and user defined policy. * @return ecPolicy, or null if not found */ public ErasureCodingPolicy getByName(String name) { + final ErasureCodingPolicyInfo ecpi = getPolicyInfoByName(name); + if (ecpi == null) { + return null; + } + return ecpi.getPolicy(); + } + + /** + * Get a {@link ErasureCodingPolicyInfo} by policy name, including system + * policy and user defined policy. + * @return ecPolicy, or null if not found + */ + private ErasureCodingPolicyInfo getPolicyInfoByName(final String name) { return this.policiesByName.get(name); } @@ -224,9 +256,6 @@ public final class ErasureCodingPolicyManager { */ public synchronized ErasureCodingPolicy addPolicy( ErasureCodingPolicy policy) { - // Set policy state into DISABLED when adding into Hadoop. - policy.setState(ErasureCodingPolicyState.DISABLED); - if (!CodecUtil.hasCodec(policy.getCodecName())) { throw new HadoopIllegalArgumentException("Codec name " + policy.getCodecName() + " is not supported"); @@ -240,7 +269,8 @@ public final class ErasureCodingPolicyManager { String assignedNewName = ErasureCodingPolicy.composePolicyName( policy.getSchema(), policy.getCellSize()); - for (ErasureCodingPolicy p : getPolicies()) { + for (ErasureCodingPolicyInfo info : getPolicies()) { + final ErasureCodingPolicy p = info.getPolicy(); if (p.getName().equals(assignedNewName)) { LOG.info("The policy name " + assignedNewName + " already exists"); return p; @@ -261,11 +291,13 @@ public final class ErasureCodingPolicyManager { ErasureCodeConstants.MAX_POLICY_ID); } - policy.setName(assignedNewName); - policy.setId(getNextAvailablePolicyID()); - this.policiesByName.put(policy.getName(), policy); - this.policiesByID.put(policy.getId(), policy); - allPolicies = policiesByName.values().toArray(new ErasureCodingPolicy[0]); + policy = new ErasureCodingPolicy(assignedNewName, policy.getSchema(), + policy.getCellSize(), getNextAvailablePolicyID()); + final ErasureCodingPolicyInfo pi = new ErasureCodingPolicyInfo(policy); + this.policiesByName.put(policy.getName(), pi); + this.policiesByID.put(policy.getId(), pi); + allPolicies = + policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]); return policy; } @@ -283,12 +315,13 @@ public final class ErasureCodingPolicyManager { * Remove an User erasure coding policy by policyName. */ public synchronized void removePolicy(String name) { - ErasureCodingPolicy ecPolicy = policiesByName.get(name); - if (ecPolicy == null) { + final ErasureCodingPolicyInfo info = policiesByName.get(name); + if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + name + " does not exist"); } + final ErasureCodingPolicy ecPolicy = info.getPolicy(); if (ecPolicy.isSystemPolicy()) { throw new HadoopIllegalArgumentException("System erasure coding policy " + name + " cannot be removed"); @@ -299,7 +332,7 @@ public final class ErasureCodingPolicyManager { enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); } - ecPolicy.setState(ErasureCodingPolicyState.REMOVED); + info.setState(ErasureCodingPolicyState.REMOVED); LOG.info("Remove erasure coding policy " + name); /* @@ -310,10 +343,10 @@ public final class ErasureCodingPolicyManager { @VisibleForTesting public List<ErasureCodingPolicy> getRemovedPolicies() { - ArrayList<ErasureCodingPolicy> removedPolicies = - new ArrayList<ErasureCodingPolicy>(); - for (ErasureCodingPolicy ecPolicy : policiesByName.values()) { - if (ecPolicy.isRemoved()) { + ArrayList<ErasureCodingPolicy> removedPolicies = new ArrayList<>(); + for (ErasureCodingPolicyInfo info : policiesByName.values()) { + final ErasureCodingPolicy ecPolicy = info.getPolicy(); + if (info.isRemoved()) { removedPolicies.add(ecPolicy); } } @@ -324,8 +357,8 @@ public final class ErasureCodingPolicyManager { * Disable an erasure coding policy by policyName. */ public synchronized void disablePolicy(String name) { - ErasureCodingPolicy ecPolicy = policiesByName.get(name); - if (ecPolicy == null) { + ErasureCodingPolicyInfo info = policiesByName.get(name); + if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + name + " does not exist"); } @@ -335,7 +368,7 @@ public final class ErasureCodingPolicyManager { enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); } - ecPolicy.setState(ErasureCodingPolicyState.DISABLED); + info.setState(ErasureCodingPolicyState.DISABLED); LOG.info("Disable the erasure coding policy " + name); } @@ -343,14 +376,15 @@ public final class ErasureCodingPolicyManager { * Enable an erasure coding policy by policyName. */ public synchronized void enablePolicy(String name) { - ErasureCodingPolicy ecPolicy = policiesByName.get(name); - if (ecPolicy == null) { + final ErasureCodingPolicyInfo info = policiesByName.get(name); + if (info == null) { throw new HadoopIllegalArgumentException("The policy name " + name + " does not exist"); } + final ErasureCodingPolicy ecPolicy = info.getPolicy(); enabledPoliciesByName.put(name, ecPolicy); - ecPolicy.setState(ErasureCodingPolicyState.ENABLED); + info.setState(ErasureCodingPolicyState.ENABLED); enabledPolicies = enabledPoliciesByName.values().toArray(new ErasureCodingPolicy[0]); LOG.info("Enable the erasure coding policy " + name); @@ -359,17 +393,19 @@ public final class ErasureCodingPolicyManager { /** * Load an erasure coding policy into erasure coding manager. */ - private void loadPolicy(ErasureCodingPolicy policy) { + private void loadPolicy(ErasureCodingPolicyInfo info) { + Preconditions.checkNotNull(info); + final ErasureCodingPolicy policy = info.getPolicy(); if (!CodecUtil.hasCodec(policy.getCodecName()) || policy.getCellSize() > maxCellSize) { // If policy is not supported in current system, set the policy state to // DISABLED; - policy.setState(ErasureCodingPolicyState.DISABLED); + info.setState(ErasureCodingPolicyState.DISABLED); } - this.policiesByName.put(policy.getName(), policy); - this.policiesByID.put(policy.getId(), policy); - if (policy.isEnabled()) { + this.policiesByName.put(policy.getName(), info); + this.policiesByID.put(policy.getId(), info); + if (info.isEnabled()) { enablePolicy(policy.getName()); } } @@ -380,11 +416,13 @@ public final class ErasureCodingPolicyManager { * @param ecPolicies contains ErasureCodingPolicy list * */ - public synchronized void loadPolicies(List<ErasureCodingPolicy> ecPolicies) { + public synchronized void loadPolicies( + List<ErasureCodingPolicyInfo> ecPolicies) { Preconditions.checkNotNull(ecPolicies); - for (ErasureCodingPolicy p : ecPolicies) { + for (ErasureCodingPolicyInfo p : ecPolicies) { loadPolicy(p); } - allPolicies = policiesByName.values().toArray(new ErasureCodingPolicy[0]); + allPolicies = + policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 391e392..7476f91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableUtils; @@ -366,10 +367,10 @@ final class FSDirErasureCodingOp { * Get available erasure coding polices. * * @param fsn namespace - * @return {@link ErasureCodingPolicy} array + * @return {@link ErasureCodingPolicyInfo} array */ - static ErasureCodingPolicy[] getErasureCodingPolicies(final FSNamesystem fsn) - throws IOException { + static ErasureCodingPolicyInfo[] getErasureCodingPolicies( + final FSNamesystem fsn) throws IOException { assert fsn.hasReadLock(); return fsn.getErasureCodingPolicyManager().getPolicies(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index ad8cdfc..cd5a5526 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -41,7 +41,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.slf4j.Logger; @@ -380,10 +380,10 @@ public final class FSImageFormatProtobuf { private void loadErasureCodingSection(InputStream in) throws IOException { ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in); - List<ErasureCodingPolicy> ecPolicies = Lists + List<ErasureCodingPolicyInfo> ecPolicies = Lists .newArrayListWithCapacity(s.getPoliciesCount()); for (int i = 0; i < s.getPoliciesCount(); ++i) { - ecPolicies.add(PBHelperClient.convertErasureCodingPolicy( + ecPolicies.add(PBHelperClient.convertErasureCodingPolicyInfo( s.getPolicies(i))); } fsn.getErasureCodingPolicyManager().loadPolicies(ecPolicies); @@ -586,11 +586,11 @@ public final class FSImageFormatProtobuf { private void saveErasureCodingSection( FileSummary.Builder summary) throws IOException { final FSNamesystem fsn = context.getSourceNamesystem(); - ErasureCodingPolicy[] ecPolicies = + ErasureCodingPolicyInfo[] ecPolicies = fsn.getErasureCodingPolicyManager().getPolicies(); ArrayList<ErasureCodingPolicyProto> ecPolicyProtoes = new ArrayList<ErasureCodingPolicyProto>(); - for (ErasureCodingPolicy p : ecPolicies) { + for (ErasureCodingPolicyInfo p : ecPolicies) { ecPolicyProtoes.add(PBHelperClient.convertErasureCodingPolicy(p)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 63d6ea4..0f61621 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -89,6 +89,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; @@ -7380,16 +7381,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } /** - * Get available erasure coding polices + * Get all erasure coding polices. */ - ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { + ErasureCodingPolicyInfo[] getErasureCodingPolicies() throws IOException { final String operationName = "getErasureCodingPolicies"; boolean success = false; checkOperation(OperationCategory.READ); readLock(); try { checkOperation(OperationCategory.READ); - final ErasureCodingPolicy[] ret = + final ErasureCodingPolicyInfo[] ret = FSDirErasureCodingOp.getErasureCodingPolicies(this); success = true; return ret; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index c07b521..283547e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -101,6 +101,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSLimitException; import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus; @@ -2299,7 +2300,8 @@ public class NameNodeRpcServer implements NamenodeProtocols { } @Override // ClientProtocol - public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { + public ErasureCodingPolicyInfo[] getErasureCodingPolicies() + throws IOException { checkNNStartup(); return namesystem.getErasureCodingPolicies(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 652f78a..a28f227 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.util.ECPolicyLoader; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.tools.TableListing; @@ -111,16 +112,16 @@ public class ECAdmin extends Configured implements Tool { final DistributedFileSystem dfs = AdminHelper.getDFS(conf); try { - Collection<ErasureCodingPolicy> policies = + final Collection<ErasureCodingPolicyInfo> policies = dfs.getAllErasureCodingPolicies(); if (policies.isEmpty()) { System.out.println("There is no erasure coding policies in the " + "cluster."); } else { System.out.println("Erasure Coding Policies:"); - for (ErasureCodingPolicy policy : policies) { + for (ErasureCodingPolicyInfo policy : policies) { if (policy != null) { - System.out.println(policy.toString()); + System.out.println(policy); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 17db352..1048b0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -118,6 +118,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -147,6 +149,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -300,6 +303,19 @@ public class DFSTestUtil { } } + public static ErasureCodingPolicyState getECPolicyState( + final ErasureCodingPolicy policy) { + final ErasureCodingPolicyInfo[] policyInfos = + ErasureCodingPolicyManager.getInstance().getPolicies(); + for (ErasureCodingPolicyInfo pi : policyInfos) { + if (pi.getPolicy().equals(policy)) { + return pi.getState(); + } + } + throw new IllegalArgumentException("ErasureCodingPolicy <" + policy + + "> doesn't exist in the policies:" + Arrays.toString(policyInfos)); + } + /** class MyFile contains enough information to recreate the contents of * a single file. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 06ac073..e16cac7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -49,6 +50,7 @@ import org.junit.rules.Timeout; import java.io.FileNotFoundException; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; import java.util.List; @@ -375,10 +377,16 @@ public class TestErasureCodingPolicies { @Test public void testGetAllErasureCodingPolicies() throws Exception { - Collection<ErasureCodingPolicy> allECPolicies = fs + Collection<ErasureCodingPolicyInfo> allECPolicies = fs .getAllErasureCodingPolicies(); - assertTrue("All system policies should be enabled", - allECPolicies.containsAll(SystemErasureCodingPolicies.getPolicies())); + final List<ErasureCodingPolicy> sysPolicies = + new ArrayList<>(SystemErasureCodingPolicies.getPolicies()); + for (ErasureCodingPolicyInfo ecpi : allECPolicies) { + if (ecpi.isEnabled()) { + sysPolicies.remove(ecpi.getPolicy()); + } + } + assertTrue("All system policies should be enabled", sysPolicies.isEmpty()); // Query after add a new policy ECSchema toAddSchema = new ECSchema("rs", 5, 2); @@ -609,11 +617,11 @@ public class TestErasureCodingPolicies { fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); String ecPolicyName = null; - Collection<ErasureCodingPolicy> allPolicies = + final Collection<ErasureCodingPolicyInfo> allPoliciesInfo = fs.getAllErasureCodingPolicies(); - for (ErasureCodingPolicy policy : allPolicies) { - if (!ecPolicy.equals(policy)) { - ecPolicyName = policy.getName(); + for (ErasureCodingPolicyInfo info : allPoliciesInfo) { + if (!ecPolicy.equals(info.getPolicy())) { + ecPolicyName = info.getPolicy().getName(); break; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index 14d2e26..c074ae1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -748,7 +748,8 @@ public class TestFSEditLogLoader { // check if new policy is reapplied through edit log ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId()); - assertEquals(ErasureCodingPolicyState.DISABLED, ecPolicy.getState()); + assertEquals(ErasureCodingPolicyState.DISABLED, + DFSTestUtil.getECPolicyState(ecPolicy)); // 2. enable policy fs.enableErasureCodingPolicy(newPolicy.getName()); @@ -756,7 +757,8 @@ public class TestFSEditLogLoader { cluster.waitActive(); ecPolicy = ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId()); - assertEquals(ErasureCodingPolicyState.ENABLED, ecPolicy.getState()); + assertEquals(ErasureCodingPolicyState.ENABLED, + DFSTestUtil.getECPolicyState(ecPolicy)); // create a new file, use the policy final Path dirPath = new Path("/striped"); @@ -773,7 +775,8 @@ public class TestFSEditLogLoader { cluster.waitActive(); ecPolicy = ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId()); - assertEquals(ErasureCodingPolicyState.DISABLED, ecPolicy.getState()); + assertEquals(ErasureCodingPolicyState.DISABLED, + DFSTestUtil.getECPolicyState(ecPolicy)); // read file DFSTestUtil.readFileAsBytes(fs, filePath); @@ -783,7 +786,8 @@ public class TestFSEditLogLoader { cluster.waitActive(); ecPolicy = ErasureCodingPolicyManager.getInstance().getByID(newPolicy.getId()); - assertEquals(ErasureCodingPolicyState.REMOVED, ecPolicy.getState()); + assertEquals(ErasureCodingPolicyState.REMOVED, + DFSTestUtil.getECPolicyState(ecPolicy)); // read file DFSTestUtil.readFileAsBytes(fs, filePath); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index d316c36..38a6dab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -870,7 +870,8 @@ public class TestFSImage { newPolicy, ecPolicy); assertEquals( "Newly added erasure coding policy should be of disabled state", - ErasureCodingPolicyState.DISABLED, ecPolicy.getState()); + ErasureCodingPolicyState.DISABLED, + DFSTestUtil.getECPolicyState(ecPolicy)); // Test enable/disable/remove user customized erasure coding policy testChangeErasureCodingPolicyState(cluster, blockSize, newPolicy); @@ -880,14 +881,12 @@ public class TestFSImage { } } - private void testChangeErasureCodingPolicyState(MiniDFSCluster cluster, int blockSize, ErasureCodingPolicy targetPolicy) throws IOException { DistributedFileSystem fs = cluster.getFileSystem(); // 1. Enable an erasure coding policy fs.enableErasureCodingPolicy(targetPolicy.getName()); - targetPolicy.setState(ErasureCodingPolicyState.ENABLED); // Create file, using the new policy final Path dirPath = new Path("/striped"); final Path filePath = new Path(dirPath, "file"); @@ -910,13 +909,13 @@ public class TestFSImage { assertEquals("The erasure coding policy is not found", targetPolicy, ecPolicy); assertEquals("The erasure coding policy should be of enabled state", - ErasureCodingPolicyState.ENABLED, ecPolicy.getState()); + ErasureCodingPolicyState.ENABLED, + DFSTestUtil.getECPolicyState(ecPolicy)); // Read file regardless of the erasure coding policy state DFSTestUtil.readFileAsBytes(fs, filePath); // 2. Disable an erasure coding policy fs.disableErasureCodingPolicy(ecPolicy.getName()); - targetPolicy.setState(ErasureCodingPolicyState.DISABLED); // Save namespace and restart NameNode fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fs.saveNamespace(); @@ -929,7 +928,8 @@ public class TestFSImage { assertEquals("The erasure coding policy is not found", targetPolicy, ecPolicy); assertEquals("The erasure coding policy should be of disabled state", - ErasureCodingPolicyState.DISABLED, ecPolicy.getState()); + ErasureCodingPolicyState.DISABLED, + DFSTestUtil.getECPolicyState(ecPolicy)); // Read file regardless of the erasure coding policy state DFSTestUtil.readFileAsBytes(fs, filePath); @@ -944,7 +944,7 @@ public class TestFSImage { return; } - targetPolicy.setState(ErasureCodingPolicyState.REMOVED); + fs.removeErasureCodingPolicy(ecPolicy.getName()); // Save namespace and restart NameNode fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); fs.saveNamespace(); @@ -957,7 +957,8 @@ public class TestFSImage { assertEquals("The erasure coding policy saved into and loaded from " + "fsImage is bad", targetPolicy, ecPolicy); assertEquals("The erasure coding policy should be of removed state", - ErasureCodingPolicyState.REMOVED, ecPolicy.getState()); + ErasureCodingPolicyState.REMOVED, + DFSTestUtil.getECPolicyState(ecPolicy)); // Read file regardless of the erasure coding policy state DFSTestUtil.readFileAsBytes(fs, filePath); fs.delete(dirPath, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c96e0d6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index 1d495c4..61485be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; @@ -925,11 +926,11 @@ public class TestDFSIO implements Tool { private boolean checkErasureCodePolicy(String erasureCodePolicyName, FileSystem fs, TestType testType) throws IOException { - Collection<ErasureCodingPolicy> list = + Collection<ErasureCodingPolicyInfo> list = ((DistributedFileSystem) fs).getAllErasureCodingPolicies(); boolean isValid = false; - for (ErasureCodingPolicy ec : list) { - if (erasureCodePolicyName.equals(ec.getName())) { + for (ErasureCodingPolicyInfo ec : list) { + if (erasureCodePolicyName.equals(ec.getPolicy().getName())) { isValid = true; break; } @@ -939,8 +940,8 @@ public class TestDFSIO implements Tool { System.out.println("Invalid erasure code policy: " + erasureCodePolicyName); System.out.println("Current supported erasure code policy list: "); - for (ErasureCodingPolicy ec : list) { - System.out.println(ec.getName()); + for (ErasureCodingPolicyInfo ec : list) { + System.out.println(ec.getPolicy().getName()); } return false; } @@ -999,9 +1000,10 @@ public class TestDFSIO implements Tool { getConf().get(ERASURE_CODE_POLICY_NAME_KEY, null); fs.mkdirs(path); - Collection<ErasureCodingPolicy> list = + Collection<ErasureCodingPolicyInfo> list = ((DistributedFileSystem) fs).getAllErasureCodingPolicies(); - for (ErasureCodingPolicy ec : list) { + for (ErasureCodingPolicyInfo info : list) { + final ErasureCodingPolicy ec = info.getPolicy(); if (erasureCodePolicyName.equals(ec.getName())) { ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec.getName()); LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName + --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
