http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java deleted file mode 100644 index d411a6b..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.client; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ReadContainerResponseProto; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto; -import org.apache.hadoop.scm.XceiverClientManager; -import org.apache.hadoop.scm.XceiverClientSpi; -import org.apache.hadoop.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.scm.protocolPB - .StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.scm.storage.ContainerProtocolCalls; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; -import java.util.UUID; - -import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.ALLOCATED; -import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.OPEN; - -/** - * This class provides the client-facing APIs of container operations. - */ -public class ContainerOperationClient implements ScmClient { - - private static final Logger LOG = - LoggerFactory.getLogger(ContainerOperationClient.class); - private static long containerSizeB = -1; - private final StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient; - private final XceiverClientManager xceiverClientManager; - - public ContainerOperationClient( - StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient, - XceiverClientManager xceiverClientManager) { - this.storageContainerLocationClient = storageContainerLocationClient; - this.xceiverClientManager = xceiverClientManager; - } - - /** - * Return the capacity of containers. The current assumption is that all - * containers have the same capacity. Therefore one static is sufficient for - * any container. - * @return The capacity of one container in number of bytes. - */ - public static long getContainerSizeB() { - return containerSizeB; - } - - /** - * Set the capacity of container. Should be exactly once on system start. - * @param size Capacity of one container in number of bytes. - */ - public static void setContainerSizeB(long size) { - containerSizeB = size; - } - - /** - * @inheritDoc - */ - @Override - public Pipeline createContainer(String containerId, String owner) - throws IOException { - XceiverClientSpi client = null; - try { - Pipeline pipeline = - storageContainerLocationClient.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), containerId, owner); - client = xceiverClientManager.acquireClient(pipeline); - - // Allocated State means that SCM has allocated this pipeline in its - // namespace. The client needs to create the pipeline on the machines - // which was choosen by the SCM. - Preconditions.checkState(pipeline.getLifeCycleState() == ALLOCATED || - pipeline.getLifeCycleState() == OPEN, "Unexpected pipeline state"); - if (pipeline.getLifeCycleState() == ALLOCATED) { - createPipeline(client, pipeline); - } - // TODO : Container Client State needs to be updated. - // TODO : Return ContainerInfo instead of Pipeline - createContainer(containerId, client, pipeline); - return pipeline; - } finally { - if (client != null) { - xceiverClientManager.releaseClient(client); - } - } - } - - /** - * Create a container over pipeline specified by the SCM. - * - * @param containerId - Container ID - * @param client - Client to communicate with Datanodes - * @param pipeline - A pipeline that is already created. - * @throws IOException - */ - public void createContainer(String containerId, XceiverClientSpi client, - Pipeline pipeline) throws IOException { - String traceID = UUID.randomUUID().toString(); - storageContainerLocationClient.notifyObjectStageChange( - ObjectStageChangeRequestProto.Type.container, - containerId, - ObjectStageChangeRequestProto.Op.create, - ObjectStageChangeRequestProto.Stage.begin); - ContainerProtocolCalls.createContainer(client, traceID); - storageContainerLocationClient.notifyObjectStageChange( - ObjectStageChangeRequestProto.Type.container, - containerId, - ObjectStageChangeRequestProto.Op.create, - ObjectStageChangeRequestProto.Stage.complete); - - // Let us log this info after we let SCM know that we have completed the - // creation state. - if (LOG.isDebugEnabled()) { - LOG.debug("Created container " + containerId - + " leader:" + pipeline.getLeader() - + " machines:" + pipeline.getMachines()); - } - } - - /** - * Creates a pipeline over the machines choosen by the SCM. - * - * @param client - Client - * @param pipeline - pipeline to be createdon Datanodes. - * @throws IOException - */ - private void createPipeline(XceiverClientSpi client, Pipeline pipeline) - throws IOException { - - Preconditions.checkNotNull(pipeline.getPipelineName(), "Pipeline " + - "name cannot be null when client create flag is set."); - - // Pipeline creation is a three step process. - // - // 1. Notify SCM that this client is doing a create pipeline on - // datanodes. - // - // 2. Talk to Datanodes to create the pipeline. - // - // 3. update SCM that pipeline creation was successful. - storageContainerLocationClient.notifyObjectStageChange( - ObjectStageChangeRequestProto.Type.pipeline, - pipeline.getPipelineName(), - ObjectStageChangeRequestProto.Op.create, - ObjectStageChangeRequestProto.Stage.begin); - - client.createPipeline(pipeline.getPipelineName(), - pipeline.getMachines()); - - storageContainerLocationClient.notifyObjectStageChange( - ObjectStageChangeRequestProto.Type.pipeline, - pipeline.getPipelineName(), - ObjectStageChangeRequestProto.Op.create, - ObjectStageChangeRequestProto.Stage.complete); - - // TODO : Should we change the state on the client side ?? - // That makes sense, but it is not needed for the client to work. - LOG.debug("Pipeline creation successful. Pipeline: {}", - pipeline.toString()); - } - - /** - * @inheritDoc - */ - @Override - public Pipeline createContainer(OzoneProtos.ReplicationType type, - OzoneProtos.ReplicationFactor factor, - String containerId, String owner) throws IOException { - XceiverClientSpi client = null; - try { - // allocate container on SCM. - Pipeline pipeline = - storageContainerLocationClient.allocateContainer(type, factor, - containerId, owner); - client = xceiverClientManager.acquireClient(pipeline); - - // Allocated State means that SCM has allocated this pipeline in its - // namespace. The client needs to create the pipeline on the machines - // which was choosen by the SCM. - if (pipeline.getLifeCycleState() == ALLOCATED) { - createPipeline(client, pipeline); - } - - // TODO : Return ContainerInfo instead of Pipeline - // connect to pipeline leader and allocate container on leader datanode. - client = xceiverClientManager.acquireClient(pipeline); - createContainer(containerId, client, pipeline); - return pipeline; - } finally { - if (client != null) { - xceiverClientManager.releaseClient(client); - } - } - } - - /** - * Returns a set of Nodes that meet a query criteria. - * - * @param nodeStatuses - A set of criteria that we want the node to have. - * @param queryScope - Query scope - Cluster or pool. - * @param poolName - if it is pool, a pool name is required. - * @return A set of nodes that meet the requested criteria. - * @throws IOException - */ - @Override - public OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> - nodeStatuses, OzoneProtos.QueryScope queryScope, String poolName) - throws IOException { - return storageContainerLocationClient.queryNode(nodeStatuses, queryScope, - poolName); - } - - /** - * Creates a specified replication pipeline. - */ - @Override - public Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type, - OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool) - throws IOException { - return storageContainerLocationClient.createReplicationPipeline(type, - factor, nodePool); - } - - /** - * Delete the container, this will release any resource it uses. - * @param pipeline - Pipeline that represents the container. - * @param force - True to forcibly delete the container. - * @throws IOException - */ - @Override - public void deleteContainer(Pipeline pipeline, boolean force) - throws IOException { - XceiverClientSpi client = null; - try { - client = xceiverClientManager.acquireClient(pipeline); - String traceID = UUID.randomUUID().toString(); - ContainerProtocolCalls.deleteContainer(client, force, traceID); - storageContainerLocationClient - .deleteContainer(pipeline.getContainerName()); - if (LOG.isDebugEnabled()) { - LOG.debug("Deleted container {}, leader: {}, machines: {} ", - pipeline.getContainerName(), - pipeline.getLeader(), - pipeline.getMachines()); - } - } finally { - if (client != null) { - xceiverClientManager.releaseClient(client); - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public List<ContainerInfo> listContainer(String startName, - String prefixName, int count) - throws IOException { - return storageContainerLocationClient.listContainer( - startName, prefixName, count); - } - - /** - * Get meta data from an existing container. - * - * @param pipeline - pipeline that represents the container. - * @return ContainerInfo - a message of protobuf which has basic info - * of a container. - * @throws IOException - */ - @Override - public ContainerData readContainer(Pipeline pipeline) throws IOException { - XceiverClientSpi client = null; - try { - client = xceiverClientManager.acquireClient(pipeline); - String traceID = UUID.randomUUID().toString(); - ReadContainerResponseProto response = - ContainerProtocolCalls.readContainer(client, - pipeline.getContainerName(), traceID); - if (LOG.isDebugEnabled()) { - LOG.debug("Read container {}, leader: {}, machines: {} ", - pipeline.getContainerName(), - pipeline.getLeader(), - pipeline.getMachines()); - } - return response.getContainerData(); - } finally { - if (client != null) { - xceiverClientManager.releaseClient(client); - } - } - } - - /** - * Given an id, return the pipeline associated with the container. - * @param containerId - String Container ID - * @return Pipeline of the existing container, corresponding to the given id. - * @throws IOException - */ - @Override - public Pipeline getContainer(String containerId) throws - IOException { - return storageContainerLocationClient.getContainer(containerId); - } - - /** - * Close a container. - * - * @param pipeline the container to be closed. - * @throws IOException - */ - @Override - public void closeContainer(Pipeline pipeline) throws IOException { - XceiverClientSpi client = null; - try { - LOG.debug("Close container {}", pipeline); - /* - TODO: two orders here, revisit this later: - 1. close on SCM first, then on data node - 2. close on data node first, then on SCM - - with 1: if client failed after closing on SCM, then there is a - container SCM thinks as closed, but is actually open. Then SCM will no - longer allocate block to it, which is fine. But SCM may later try to - replicate this "closed" container, which I'm not sure is safe. - - with 2: if client failed after close on datanode, then there is a - container SCM thinks as open, but is actually closed. Then SCM will still - try to allocate block to it. Which will fail when actually doing the - write. No more data can be written, but at least the correctness and - consistency of existing data will maintain. - - For now, take the #2 way. - */ - // Actually close the container on Datanode - client = xceiverClientManager.acquireClient(pipeline); - String traceID = UUID.randomUUID().toString(); - - String containerId = pipeline.getContainerName(); - - storageContainerLocationClient.notifyObjectStageChange( - ObjectStageChangeRequestProto.Type.container, - containerId, - ObjectStageChangeRequestProto.Op.close, - ObjectStageChangeRequestProto.Stage.begin); - - ContainerProtocolCalls.closeContainer(client, traceID); - // Notify SCM to close the container - storageContainerLocationClient.notifyObjectStageChange( - ObjectStageChangeRequestProto.Type.container, - containerId, - ObjectStageChangeRequestProto.Op.close, - ObjectStageChangeRequestProto.Stage.complete); - } finally { - if (client != null) { - xceiverClientManager.releaseClient(client); - } - } - } - - /** - * Get the the current usage information. - * @param pipeline - Pipeline - * @return the size of the given container. - * @throws IOException - */ - @Override - public long getContainerSize(Pipeline pipeline) throws IOException { - // TODO : Pipeline can be null, handle it correctly. - long size = getContainerSizeB(); - if (size == -1) { - throw new IOException("Container size unknown!"); - } - return size; - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java deleted file mode 100644 index ef2f1e0..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.client; - -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData; -import org.apache.hadoop.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; - -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; - -/** - * The interface to call into underlying container layer. - * - * Written as interface to allow easy testing: implement a mock container layer - * for standalone testing of CBlock API without actually calling into remote - * containers. Actual container layer can simply re-implement this. - * - * NOTE this is temporarily needed class. When SCM containers are full-fledged, - * this interface will likely be removed. - */ -@InterfaceStability.Unstable -public interface ScmClient { - /** - * Creates a Container on SCM and returns the pipeline. - * @param containerId - String container ID - * @return Pipeline - * @throws IOException - */ - Pipeline createContainer(String containerId, String owner) throws IOException; - - /** - * Gets a container by Name -- Throws if the container does not exist. - * @param containerId - String Container ID - * @return Pipeline - * @throws IOException - */ - Pipeline getContainer(String containerId) throws IOException; - - /** - * Close a container by name. - * - * @param pipeline the container to be closed. - * @throws IOException - */ - void closeContainer(Pipeline pipeline) throws IOException; - - /** - * Deletes an existing container. - * @param pipeline - Pipeline that represents the container. - * @param force - true to forcibly delete the container. - * @throws IOException - */ - void deleteContainer(Pipeline pipeline, boolean force) throws IOException; - - /** - * Lists a range of containers and get their info. - * - * @param startName start name, if null, start searching at the head. - * @param prefixName prefix name, if null, then filter is disabled. - * @param count count, if count < 0, the max size is unlimited.( - * Usually the count will be replace with a very big - * value instead of being unlimited in case the db is very big) - * - * @return a list of pipeline. - * @throws IOException - */ - List<ContainerInfo> listContainer(String startName, String prefixName, - int count) throws IOException; - - /** - * Read meta data from an existing container. - * @param pipeline - Pipeline that represents the container. - * @return ContainerInfo - * @throws IOException - */ - ContainerData readContainer(Pipeline pipeline) throws IOException; - - - /** - * Gets the container size -- Computed by SCM from Container Reports. - * @param pipeline - Pipeline - * @return number of bytes used by this container. - * @throws IOException - */ - long getContainerSize(Pipeline pipeline) throws IOException; - - /** - * Creates a Container on SCM and returns the pipeline. - * @param type - Replication Type. - * @param replicationFactor - Replication Factor - * @param containerId - Container ID - * @return Pipeline - * @throws IOException - in case of error. - */ - Pipeline createContainer(OzoneProtos.ReplicationType type, - OzoneProtos.ReplicationFactor replicationFactor, String containerId, - String owner) throws IOException; - - /** - * Returns a set of Nodes that meet a query criteria. - * @param nodeStatuses - A set of criteria that we want the node to have. - * @param queryScope - Query scope - Cluster or pool. - * @param poolName - if it is pool, a pool name is required. - * @return A set of nodes that meet the requested criteria. - * @throws IOException - */ - OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses, - OzoneProtos.QueryScope queryScope, String poolName) throws IOException; - - /** - * Creates a specified replication pipeline. - * @param type - Type - * @param factor - Replication factor - * @param nodePool - Set of machines. - * @throws IOException - */ - Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type, - OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool) - throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java deleted file mode 100644 index e85e542..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.client; - -/** - * This package contains classes for the client of the storage container - * protocol. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java deleted file mode 100644 index f51336f..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - -/** - * Allocated block wraps the result returned from SCM#allocateBlock which - * contains a Pipeline and the key. - */ -public final class AllocatedBlock { - private Pipeline pipeline; - private String key; - // Indicates whether the client should create container before writing block. - private boolean shouldCreateContainer; - - /** - * Builder for AllocatedBlock. - */ - public static class Builder { - private Pipeline pipeline; - private String key; - private boolean shouldCreateContainer; - - public Builder setPipeline(Pipeline p) { - this.pipeline = p; - return this; - } - - public Builder setKey(String k) { - this.key = k; - return this; - } - - public Builder setShouldCreateContainer(boolean shouldCreate) { - this.shouldCreateContainer = shouldCreate; - return this; - } - - public AllocatedBlock build() { - return new AllocatedBlock(pipeline, key, shouldCreateContainer); - } - } - - private AllocatedBlock(Pipeline pipeline, String key, - boolean shouldCreateContainer) { - this.pipeline = pipeline; - this.key = key; - this.shouldCreateContainer = shouldCreateContainer; - } - - public Pipeline getPipeline() { - return pipeline; - } - - public String getKey() { - return key; - } - - public boolean getCreateContainer() { - return shouldCreateContainer; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java deleted file mode 100644 index 11d4438..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - -import com.google.common.base.Preconditions; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; -import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID; -import org.apache.hadoop.util.Time; - -import java.util.Comparator; - -/** - * Class wraps ozone container info. - */ -public class ContainerInfo - implements Comparator<ContainerInfo>, Comparable<ContainerInfo> { - private OzoneProtos.LifeCycleState state; - private Pipeline pipeline; - // Bytes allocated by SCM for clients. - private long allocatedBytes; - // Actual container usage, updated through heartbeat. - private long usedBytes; - private long numberOfKeys; - private long lastUsed; - // The wall-clock ms since the epoch at which the current state enters. - private long stateEnterTime; - private String owner; - private String containerName; - private long containerID; - ContainerInfo( - long containerID, - final String containerName, - OzoneProtos.LifeCycleState state, - Pipeline pipeline, - long allocatedBytes, - long usedBytes, - long numberOfKeys, - long stateEnterTime, - String owner) { - this.containerID = containerID; - this.containerName = containerName; - this.pipeline = pipeline; - this.allocatedBytes = allocatedBytes; - this.usedBytes = usedBytes; - this.numberOfKeys = numberOfKeys; - this.lastUsed = Time.monotonicNow(); - this.state = state; - this.stateEnterTime = stateEnterTime; - this.owner = owner; - } - - /** - * Needed for serialization findbugs. - */ - public ContainerInfo() { - } - - public static ContainerInfo fromProtobuf(OzoneProtos.SCMContainerInfo info) { - ContainerInfo.Builder builder = new ContainerInfo.Builder(); - builder.setPipeline(Pipeline.getFromProtoBuf(info.getPipeline())); - builder.setAllocatedBytes(info.getAllocatedBytes()); - builder.setUsedBytes(info.getUsedBytes()); - builder.setNumberOfKeys(info.getNumberOfKeys()); - builder.setState(info.getState()); - builder.setStateEnterTime(info.getStateEnterTime()); - builder.setOwner(info.getOwner()); - builder.setContainerName(info.getContainerName()); - builder.setContainerID(info.getContainerID()); - return builder.build(); - } - - public long getContainerID() { - return containerID; - } - - public String getContainerName() { - return containerName; - } - - public OzoneProtos.LifeCycleState getState() { - return state; - } - - public void setState(OzoneProtos.LifeCycleState state) { - this.state = state; - } - - public long getStateEnterTime() { - return stateEnterTime; - } - - public Pipeline getPipeline() { - return pipeline; - } - - public long getAllocatedBytes() { - return allocatedBytes; - } - - /** - * Set Allocated bytes. - * - * @param size - newly allocated bytes -- negative size is case of deletes - * can be used. - */ - public void updateAllocatedBytes(long size) { - this.allocatedBytes += size; - } - - public long getUsedBytes() { - return usedBytes; - } - - public long getNumberOfKeys() { - return numberOfKeys; - } - - public ContainerID containerID() { - return new ContainerID(getContainerID()); - } - - /** - * Gets the last used time from SCM's perspective. - * - * @return time in milliseconds. - */ - public long getLastUsed() { - return lastUsed; - } - - public void updateLastUsedTime() { - lastUsed = Time.monotonicNow(); - } - - public void allocate(long size) { - // should we also have total container size in ContainerInfo - // and check before allocating? - allocatedBytes += size; - } - - public OzoneProtos.SCMContainerInfo getProtobuf() { - OzoneProtos.SCMContainerInfo.Builder builder = - OzoneProtos.SCMContainerInfo.newBuilder(); - builder.setPipeline(getPipeline().getProtobufMessage()); - builder.setAllocatedBytes(getAllocatedBytes()); - builder.setUsedBytes(getUsedBytes()); - builder.setNumberOfKeys(getNumberOfKeys()); - builder.setState(state); - builder.setStateEnterTime(stateEnterTime); - builder.setContainerID(getContainerID()); - - if (getOwner() != null) { - builder.setOwner(getOwner()); - } - builder.setContainerName(getContainerName()); - return builder.build(); - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - @Override - public String toString() { - return "ContainerInfo{" - + "state=" + state - + ", pipeline=" + pipeline - + ", stateEnterTime=" + stateEnterTime - + ", owner=" + owner - + ", containerName='" + containerName - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - ContainerInfo that = (ContainerInfo) o; - - return new EqualsBuilder() - .append(pipeline.getContainerName(), that.pipeline.getContainerName()) - - // TODO : Fix this later. If we add these factors some tests fail. - // So Commenting this to continue and will enforce this with - // Changes in pipeline where we remove Container Name to - // SCMContainerinfo from Pipeline. - // .append(pipeline.getFactor(), that.pipeline.getFactor()) - // .append(pipeline.getType(), that.pipeline.getType()) - .append(owner, that.owner) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(11, 811) - .append(pipeline.getContainerName()) - .append(pipeline.getFactor()) - .append(pipeline.getType()) - .append(owner) - .toHashCode(); - } - - /** - * Compares its two arguments for order. Returns a negative integer, zero, or - * a positive integer as the first argument is less than, equal to, or greater - * than the second.<p> - * - * @param o1 the first object to be compared. - * @param o2 the second object to be compared. - * @return a negative integer, zero, or a positive integer as the first - * argument is less than, equal to, or greater than the second. - * @throws NullPointerException if an argument is null and this comparator - * does not permit null arguments - * @throws ClassCastException if the arguments' types prevent them from - * being compared by this comparator. - */ - @Override - public int compare(ContainerInfo o1, ContainerInfo o2) { - return Long.compare(o1.getLastUsed(), o2.getLastUsed()); - } - - /** - * Compares this object with the specified object for order. Returns a - * negative integer, zero, or a positive integer as this object is less than, - * equal to, or greater than the specified object. - * - * @param o the object to be compared. - * @return a negative integer, zero, or a positive integer as this object is - * less than, equal to, or greater than the specified object. - * @throws NullPointerException if the specified object is null - * @throws ClassCastException if the specified object's type prevents it - * from being compared to this object. - */ - @Override - public int compareTo(ContainerInfo o) { - return this.compare(this, o); - } - - /** - * Builder class for ContainerInfo. - */ - public static class Builder { - private OzoneProtos.LifeCycleState state; - private Pipeline pipeline; - private long allocated; - private long used; - private long keys; - private long stateEnterTime; - private String owner; - private String containerName; - private long containerID; - - public Builder setContainerID(long id) { - Preconditions.checkState(id >= 0); - this.containerID = id; - return this; - } - - public Builder setState(OzoneProtos.LifeCycleState lifeCycleState) { - this.state = lifeCycleState; - return this; - } - - public Builder setPipeline(Pipeline containerPipeline) { - this.pipeline = containerPipeline; - return this; - } - - public Builder setAllocatedBytes(long bytesAllocated) { - this.allocated = bytesAllocated; - return this; - } - - public Builder setUsedBytes(long bytesUsed) { - this.used = bytesUsed; - return this; - } - - public Builder setNumberOfKeys(long keyCount) { - this.keys = keyCount; - return this; - } - - public Builder setStateEnterTime(long time) { - this.stateEnterTime = time; - return this; - } - - public Builder setOwner(String containerOwner) { - this.owner = containerOwner; - return this; - } - - public Builder setContainerName(String container) { - this.containerName = container; - return this; - } - - public ContainerInfo build() { - return new - ContainerInfo(containerID, containerName, state, pipeline, - allocated, used, keys, stateEnterTime, owner); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java deleted file mode 100644 index 5dbb853..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - - -import static org.apache.hadoop.ozone.protocol.proto - .ScmBlockLocationProtocolProtos.DeleteScmBlockResult; - -/** - * Class wraps storage container manager block deletion results. - */ -public class DeleteBlockResult { - private String key; - private DeleteScmBlockResult.Result result; - - public DeleteBlockResult(final String key, - final DeleteScmBlockResult.Result result) { - this.key = key; - this.result = result; - } - - /** - * Get key deleted. - * @return key name. - */ - public String getKey() { - return key; - } - - /** - * Get key deletion result. - * @return key deletion result. - */ - public DeleteScmBlockResult.Result getResult() { - return result; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java deleted file mode 100644 index 9d47646..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonFilter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.ser.FilterProvider; -import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; -import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; -import org.apache.hadoop.hdfs.protocol.DatanodeID; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * A pipeline represents the group of machines over which a container lives. - */ -public class Pipeline { - static final String PIPELINE_INFO = "PIPELINE_INFO_FILTER"; - private static final ObjectWriter WRITER; - - static { - ObjectMapper mapper = new ObjectMapper(); - String[] ignorableFieldNames = {"data"}; - FilterProvider filters = new SimpleFilterProvider() - .addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter - .serializeAllExcept(ignorableFieldNames)); - mapper.setVisibility(PropertyAccessor.FIELD, - JsonAutoDetect.Visibility.ANY); - mapper.addMixIn(Object.class, MixIn.class); - - WRITER = mapper.writer(filters); - } - - private String containerName; - private PipelineChannel pipelineChannel; - /** - * Allows you to maintain private data on pipelines. This is not serialized - * via protobuf, just allows us to maintain some private data. - */ - @JsonIgnore - private byte[] data; - /** - * Constructs a new pipeline data structure. - * - * @param containerName - Container - * @param pipelineChannel - transport information for this container - */ - public Pipeline(String containerName, PipelineChannel pipelineChannel) { - this.containerName = containerName; - this.pipelineChannel = pipelineChannel; - data = null; - } - - /** - * Gets pipeline object from protobuf. - * - * @param pipeline - ProtoBuf definition for the pipeline. - * @return Pipeline Object - */ - public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) { - Preconditions.checkNotNull(pipeline); - PipelineChannel pipelineChannel = - PipelineChannel.getFromProtoBuf(pipeline.getPipelineChannel()); - return new Pipeline(pipeline.getContainerName(), pipelineChannel); - } - - public OzoneProtos.ReplicationFactor getFactor() { - return pipelineChannel.getFactor(); - } - - /** - * Returns the first machine in the set of datanodes. - * - * @return First Machine. - */ - @JsonIgnore - public DatanodeID getLeader() { - return pipelineChannel.getDatanodes().get(pipelineChannel.getLeaderID()); - } - - /** - * Returns the leader host. - * - * @return First Machine. - */ - public String getLeaderHost() { - return pipelineChannel.getDatanodes() - .get(pipelineChannel.getLeaderID()).getHostName(); - } - - /** - * Returns all machines that make up this pipeline. - * - * @return List of Machines. - */ - @JsonIgnore - public List<DatanodeID> getMachines() { - return new ArrayList<>(pipelineChannel.getDatanodes().values()); - } - - /** - * Returns all machines that make up this pipeline. - * - * @return List of Machines. - */ - public List<String> getDatanodeHosts() { - List<String> dataHosts = new ArrayList<>(); - for (DatanodeID id : pipelineChannel.getDatanodes().values()) { - dataHosts.add(id.getHostName()); - } - return dataHosts; - } - - /** - * Return a Protobuf Pipeline message from pipeline. - * - * @return Protobuf message - */ - @JsonIgnore - public OzoneProtos.Pipeline getProtobufMessage() { - OzoneProtos.Pipeline.Builder builder = - OzoneProtos.Pipeline.newBuilder(); - builder.setContainerName(this.containerName); - builder.setPipelineChannel(this.pipelineChannel.getProtobufMessage()); - return builder.build(); - } - - /** - * Returns containerName if available. - * - * @return String. - */ - public String getContainerName() { - return containerName; - } - - /** - * Returns private data that is set on this pipeline. - * - * @return blob, the user can interpret it any way they like. - */ - public byte[] getData() { - if (this.data != null) { - return Arrays.copyOf(this.data, this.data.length); - } else { - return null; - } - } - - @VisibleForTesting - public PipelineChannel getPipelineChannel() { - return pipelineChannel; - } - - /** - * Set private data on pipeline. - * - * @param data -- private data. - */ - public void setData(byte[] data) { - if (data != null) { - this.data = Arrays.copyOf(data, data.length); - } - } - - /** - * Gets the State of the pipeline. - * - * @return - LifeCycleStates. - */ - public OzoneProtos.LifeCycleState getLifeCycleState() { - return pipelineChannel.getLifeCycleState(); - } - - /** - * Gets the pipeline Name. - * - * @return - Name of the pipeline - */ - public String getPipelineName() { - return pipelineChannel.getName(); - } - - /** - * Returns the type. - * - * @return type - Standalone, Ratis, Chained. - */ - public OzoneProtos.ReplicationType getType() { - return pipelineChannel.getType(); - } - - @Override - public String toString() { - final StringBuilder b = new StringBuilder(getClass().getSimpleName()) - .append("["); - pipelineChannel.getDatanodes().keySet().stream() - .forEach(id -> b. - append(id.endsWith(pipelineChannel.getLeaderID()) ? "*" + id : id)); - b.append("] container:").append(containerName); - b.append(" name:").append(getPipelineName()); - if (getType() != null) { - b.append(" type:").append(getType().toString()); - } - if (getFactor() != null) { - b.append(" factor:").append(getFactor().toString()); - } - if (getLifeCycleState() != null) { - b.append(" State:").append(getLifeCycleState().toString()); - } - return b.toString(); - } - - /** - * Returns a JSON string of this object. - * - * @return String - json string - * @throws IOException - */ - public String toJsonString() throws IOException { - return WRITER.writeValueAsString(this); - } - - @JsonFilter(PIPELINE_INFO) - class MixIn { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java deleted file mode 100644 index 0af0534..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.container.common.helpers; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor; - -import java.util.Map; -import java.util.TreeMap; - -/** - * PipelineChannel information for a {@link Pipeline}. - */ -public class PipelineChannel { - @JsonIgnore - private String leaderID; - @JsonIgnore - private Map<String, DatanodeID> datanodes; - private LifeCycleState lifeCycleState; - private ReplicationType type; - private ReplicationFactor factor; - private String name; - - public PipelineChannel(String leaderID, LifeCycleState lifeCycleState, - ReplicationType replicationType, ReplicationFactor replicationFactor, - String name) { - this.leaderID = leaderID; - this.lifeCycleState = lifeCycleState; - this.type = replicationType; - this.factor = replicationFactor; - this.name = name; - datanodes = new TreeMap<>(); - } - - public String getLeaderID() { - return leaderID; - } - - public Map<String, DatanodeID> getDatanodes() { - return datanodes; - } - - public LifeCycleState getLifeCycleState() { - return lifeCycleState; - } - - public ReplicationType getType() { - return type; - } - - public ReplicationFactor getFactor() { - return factor; - } - - public String getName() { - return name; - } - - public void addMember(DatanodeID dataNodeId) { - datanodes.put(dataNodeId.getDatanodeUuid(), dataNodeId); - } - - @JsonIgnore - public OzoneProtos.PipelineChannel getProtobufMessage() { - OzoneProtos.PipelineChannel.Builder builder = - OzoneProtos.PipelineChannel.newBuilder(); - for (DatanodeID datanode : datanodes.values()) { - builder.addMembers(datanode.getProtoBufMessage()); - } - builder.setLeaderID(leaderID); - - if (this.getLifeCycleState() != null) { - builder.setState(this.getLifeCycleState()); - } - if (this.getType() != null) { - builder.setType(this.getType()); - } - - if (this.getFactor() != null) { - builder.setFactor(this.getFactor()); - } - return builder.build(); - } - - public static PipelineChannel getFromProtoBuf( - OzoneProtos.PipelineChannel transportProtos) { - Preconditions.checkNotNull(transportProtos); - PipelineChannel pipelineChannel = - new PipelineChannel(transportProtos.getLeaderID(), - transportProtos.getState(), - transportProtos.getType(), - transportProtos.getFactor(), - transportProtos.getName()); - - for (HdfsProtos.DatanodeIDProto dataID : transportProtos.getMembersList()) { - pipelineChannel.addMember(DatanodeID.getFromProtoBuf(dataID)); - } - return pipelineChannel; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java deleted file mode 100644 index 549fced..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.container.common.helpers; - -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos; - -import java.io.IOException; - -/** - * Exceptions thrown from the Storage Container. - */ -public class StorageContainerException extends IOException { - private ContainerProtos.Result result; - - /** - * Constructs an {@code IOException} with {@code null} - * as its error detail message. - */ - public StorageContainerException(ContainerProtos.Result result) { - this.result = result; - } - - /** - * Constructs an {@code IOException} with the specified detail message. - * - * @param message The detail message (which is saved for later retrieval by - * the {@link #getMessage()} method) - * @param result - The result code - */ - public StorageContainerException(String message, - ContainerProtos.Result result) { - super(message); - this.result = result; - } - - /** - * Constructs an {@code IOException} with the specified detail message - * and cause. - * <p> - * <p> Note that the detail message associated with {@code cause} is - * <i>not</i> automatically incorporated into this exception's detail - * message. - * - * @param message The detail message (which is saved for later retrieval by - * the {@link #getMessage()} method) - * - * @param cause The cause (which is saved for later retrieval by the {@link - * #getCause()} method). (A null value is permitted, and indicates that the - * cause is nonexistent or unknown.) - * - * @param result - The result code - * @since 1.6 - */ - public StorageContainerException(String message, Throwable cause, - ContainerProtos.Result result) { - super(message, cause); - this.result = result; - } - - /** - * Constructs an {@code IOException} with the specified cause and a - * detail message of {@code (cause==null ? null : cause.toString())} - * (which typically contains the class and detail message of {@code cause}). - * This constructor is useful for IO exceptions that are little more - * than wrappers for other throwables. - * - * @param cause The cause (which is saved for later retrieval by the {@link - * #getCause()} method). (A null value is permitted, and indicates that the - * cause is nonexistent or unknown.) - * @param result - The result code - * @since 1.6 - */ - public StorageContainerException(Throwable cause, ContainerProtos.Result - result) { - super(cause); - this.result = result; - } - - /** - * Returns Result. - * - * @return Result. - */ - public ContainerProtos.Result getResult() { - return result; - } - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java deleted file mode 100644 index 3fa9663..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.container.common.helpers; -/** - Contains protocol buffer helper classes and utilites used in - impl. - **/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java deleted file mode 100644 index ad24f98..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm; - -/** - * This package contains classes for the client of the storage container - * protocol. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java deleted file mode 100644 index 469dab5..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.protocol; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; - -import java.util.Set; - -/** - * Holds the nodes that currently host the container for an object key hash. - */ -@InterfaceAudience.Private -public final class LocatedContainer { - private final String key; - private final String matchedKeyPrefix; - private final String containerName; - private final Set<DatanodeInfo> locations; - private final DatanodeInfo leader; - - /** - * Creates a LocatedContainer. - * - * @param key object key - * @param matchedKeyPrefix prefix of key that was used to find the location - * @param containerName container name - * @param locations nodes that currently host the container - * @param leader node that currently acts as pipeline leader - */ - public LocatedContainer(String key, String matchedKeyPrefix, - String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) { - this.key = key; - this.matchedKeyPrefix = matchedKeyPrefix; - this.containerName = containerName; - this.locations = locations; - this.leader = leader; - } - - /** - * Returns the container name. - * - * @return container name - */ - public String getContainerName() { - return this.containerName; - } - - /** - * Returns the object key. - * - * @return object key - */ - public String getKey() { - return this.key; - } - - /** - * Returns the node that currently acts as pipeline leader. - * - * @return node that currently acts as pipeline leader - */ - public DatanodeInfo getLeader() { - return this.leader; - } - - /** - * Returns the nodes that currently host the container. - * - * @return Set<DatanodeInfo> nodes that currently host the container - */ - public Set<DatanodeInfo> getLocations() { - return this.locations; - } - - /** - * Returns the prefix of the key that was used to find the location. - * - * @return prefix of the key that was used to find the location - */ - public String getMatchedKeyPrefix() { - return this.matchedKeyPrefix; - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj == null) { - return false; - } - if (!(otherObj instanceof LocatedContainer)) { - return false; - } - LocatedContainer other = (LocatedContainer)otherObj; - return this.key == null ? other.key == null : this.key.equals(other.key); - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName() - + "{key=" + key - + "; matchedKeyPrefix=" + matchedKeyPrefix - + "; containerName=" + containerName - + "; locations=" + locations - + "; leader=" + leader - + "}"; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java deleted file mode 100644 index 9728554..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.protocol; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; -import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor; -import org.apache.hadoop.scm.ScmInfo; - -/** - * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes - * to read/write a block. - */ -public interface ScmBlockLocationProtocol { - - /** - * Find the set of nodes to read/write a block, as - * identified by the block key. This method supports batch lookup by - * passing multiple keys. - * - * @param keys batch of block keys to find - * @return allocated blocks for each block key - * @throws IOException if there is any failure - */ - Set<AllocatedBlock> getBlockLocations(Set<String> keys) throws IOException; - - /** - * Asks SCM where a block should be allocated. SCM responds with the - * set of datanodes that should be used creating this block. - * @param size - size of the block. - * @return allocated block accessing info (key, pipeline). - * @throws IOException - */ - AllocatedBlock allocateBlock(long size, ReplicationType type, - ReplicationFactor factor, String owner) throws IOException; - - /** - * Delete blocks for a set of object keys. - * - * @param keyBlocksInfoList Map of object key and its blocks. - * @return list of block deletion results. - * @throws IOException if there is any failure. - */ - List<DeleteBlockGroupResult> - deleteKeyBlocks(List<BlockGroup> keyBlocksInfoList) throws IOException; - - /** - * Gets the Clusterid and SCM Id from SCM. - */ - ScmInfo getScmInfo() throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java deleted file mode 100644 index 4e4b3d6..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.protocol; - -import java.util.List; -import java.util.stream.Collectors; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; - -/** - * Holds the nodes that currently host the block for a block key. - */ -@InterfaceAudience.Private -public final class ScmLocatedBlock { - private final String key; - private final List<DatanodeInfo> locations; - private final DatanodeInfo leader; - - /** - * Creates a ScmLocatedBlock. - * - * @param key object key - * @param locations nodes that currently host the block - * @param leader node that currently acts as pipeline leader - */ - public ScmLocatedBlock(final String key, final List<DatanodeInfo> locations, - final DatanodeInfo leader) { - this.key = key; - this.locations = locations; - this.leader = leader; - } - - /** - * Returns the object key. - * - * @return object key - */ - public String getKey() { - return this.key; - } - - /** - * Returns the node that currently acts as pipeline leader. - * - * @return node that currently acts as pipeline leader - */ - public DatanodeInfo getLeader() { - return this.leader; - } - - /** - * Returns the nodes that currently host the block. - * - * @return List<DatanodeInfo> nodes that currently host the block - */ - public List<DatanodeInfo> getLocations() { - return this.locations; - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj == null) { - return false; - } - if (!(otherObj instanceof ScmLocatedBlock)) { - return false; - } - ScmLocatedBlock other = (ScmLocatedBlock)otherObj; - return this.key == null ? other.key == null : this.key.equals(other.key); - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName() + "{key=" + key + "; locations=" - + locations.stream().map(loc -> loc.toString()).collect(Collectors - .joining(",")) + "; leader=" + leader + "}"; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java deleted file mode 100644 index ad78bd8..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.scm.protocol; - -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; - -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto; -import org.apache.hadoop.scm.ScmInfo; -import org.apache.hadoop.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; - -/** - * ContainerLocationProtocol is used by an HDFS node to find the set of nodes - * that currently host a container. - */ -public interface StorageContainerLocationProtocol { - /** - * Asks SCM where a container should be allocated. SCM responds with the - * set of datanodes that should be used creating this container. - * - */ - Pipeline allocateContainer(OzoneProtos.ReplicationType replicationType, - OzoneProtos.ReplicationFactor factor, String containerName, String owner) - throws IOException; - - /** - * Ask SCM the location of the container. SCM responds with a group of - * nodes where this container and its replicas are located. - * - * @param containerName - Name of the container. - * @return Pipeline - the pipeline where container locates. - * @throws IOException - */ - Pipeline getContainer(String containerName) throws IOException; - - /** - * Ask SCM a list of containers with a range of container names - * and the limit of count. - * Search container names between start name(exclusive), and - * use prefix name to filter the result. the max size of the - * searching range cannot exceed the value of count. - * - * @param startName start name, if null, start searching at the head. - * @param prefixName prefix name, if null, then filter is disabled. - * @param count count, if count < 0, the max size is unlimited.( - * Usually the count will be replace with a very big - * value instead of being unlimited in case the db is very big) - * - * @return a list of container. - * @throws IOException - */ - List<ContainerInfo> listContainer(String startName, String prefixName, - int count) throws IOException; - - /** - * Deletes a container in SCM. - * - * @param containerName - * @throws IOException - * if failed to delete the container mapping from db store - * or container doesn't exist. - */ - void deleteContainer(String containerName) throws IOException; - - /** - * Queries a list of Node Statuses. - * @param nodeStatuses - * @return List of Datanodes. - */ - OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses, - OzoneProtos.QueryScope queryScope, String poolName) throws IOException; - - /** - * Notify from client when begin or finish creating objects like pipeline - * or containers on datanodes. - * Container will be in Operational state after that. - * @param type object type - * @param name object name - * @param op operation type (e.g., create, close, delete) - * @param stage creation stage - */ - void notifyObjectStageChange( - ObjectStageChangeRequestProto.Type type, String name, - ObjectStageChangeRequestProto.Op op, - ObjectStageChangeRequestProto.Stage stage) throws IOException; - - /** - * Creates a replication pipeline of a specified type. - * @param type - replication type - * @param factor - factor 1 or 3 - * @param nodePool - optional machine list to build a pipeline. - * @throws IOException - */ - Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type, - OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool) - throws IOException; - - /** - * Returns information about SCM. - * - * @return {@link ScmInfo} - * @throws IOException - */ - ScmInfo getScmInfo() throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java deleted file mode 100644 index 274f859..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.protocol; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java deleted file mode 100644 index 02d973a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.scm.protocolPB; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtobufHelper; -import org.apache.hadoop.ipc.ProtocolTranslator; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; -import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksRequestProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockResponseProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksResponseProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsRequestProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsResponseProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.ScmLocatedBlockProto; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.KeyBlocks; -import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; -import org.apache.hadoop.scm.ScmInfo; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * This class is the client-side translator to translate the requests made on - * the {@link ScmBlockLocationProtocol} interface to the RPC server - * implementing {@link ScmBlockLocationProtocolPB}. - */ -@InterfaceAudience.Private -public final class ScmBlockLocationProtocolClientSideTranslatorPB - implements ScmBlockLocationProtocol, ProtocolTranslator, Closeable { - - /** - * RpcController is not used and hence is set to null. - */ - private static final RpcController NULL_RPC_CONTROLLER = null; - - private final ScmBlockLocationProtocolPB rpcProxy; - - /** - * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB. - * - * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy - */ - public ScmBlockLocationProtocolClientSideTranslatorPB( - ScmBlockLocationProtocolPB rpcProxy) { - this.rpcProxy = rpcProxy; - } - - /** - * Find the set of nodes to read/write a block, as - * identified by the block key. This method supports batch lookup by - * passing multiple keys. - * - * @param keys batch of block keys to find - * @return allocated blocks for each block key - * @throws IOException if there is any failure - */ - @Override - public Set<AllocatedBlock> getBlockLocations(Set<String> keys) - throws IOException { - GetScmBlockLocationsRequestProto.Builder req = - GetScmBlockLocationsRequestProto.newBuilder(); - for (String key : keys) { - req.addKeys(key); - } - final GetScmBlockLocationsResponseProto resp; - try { - resp = rpcProxy.getScmBlockLocations(NULL_RPC_CONTROLLER, - req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - Set<AllocatedBlock> locatedBlocks = - Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedBlocksCount()); - for (ScmLocatedBlockProto locatedBlock : resp.getLocatedBlocksList()) { - locatedBlocks.add(new AllocatedBlock.Builder() - .setKey(locatedBlock.getKey()) - .setPipeline(Pipeline.getFromProtoBuf(locatedBlock.getPipeline())) - .build()); - } - return locatedBlocks; - } - - /** - * Asks SCM where a block should be allocated. SCM responds with the - * set of datanodes that should be used creating this block. - * @param size - size of the block. - * @return allocated block accessing info (key, pipeline). - * @throws IOException - */ - @Override - public AllocatedBlock allocateBlock(long size, - OzoneProtos.ReplicationType type, OzoneProtos.ReplicationFactor factor, - String owner) throws IOException { - Preconditions.checkArgument(size > 0, "block size must be greater than 0"); - - AllocateScmBlockRequestProto request = - AllocateScmBlockRequestProto.newBuilder().setSize(size).setType(type) - .setFactor(factor).setOwner(owner).build(); - final AllocateScmBlockResponseProto response; - try { - response = rpcProxy.allocateScmBlock(NULL_RPC_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - if (response.getErrorCode() != - AllocateScmBlockResponseProto.Error.success) { - throw new IOException(response.hasErrorMessage() ? - response.getErrorMessage() : "Allocate block failed."); - } - AllocatedBlock.Builder builder = new AllocatedBlock.Builder() - .setKey(response.getKey()) - .setPipeline(Pipeline.getFromProtoBuf(response.getPipeline())) - .setShouldCreateContainer(response.getCreateContainer()); - return builder.build(); - } - - /** - * Delete the set of keys specified. - * - * @param keyBlocksInfoList batch of block keys to delete. - * @return list of block deletion results. - * @throws IOException if there is any failure. - * - */ - @Override - public List<DeleteBlockGroupResult> deleteKeyBlocks( - List<BlockGroup> keyBlocksInfoList) throws IOException { - List<KeyBlocks> keyBlocksProto = keyBlocksInfoList.stream() - .map(BlockGroup::getProto).collect(Collectors.toList()); - DeleteScmKeyBlocksRequestProto request = DeleteScmKeyBlocksRequestProto - .newBuilder().addAllKeyBlocks(keyBlocksProto).build(); - - final DeleteScmKeyBlocksResponseProto resp; - try { - resp = rpcProxy.deleteScmKeyBlocks(NULL_RPC_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - List<DeleteBlockGroupResult> results = - new ArrayList<>(resp.getResultsCount()); - results.addAll(resp.getResultsList().stream().map( - result -> new DeleteBlockGroupResult(result.getObjectKey(), - DeleteBlockGroupResult - .convertBlockResultProto(result.getBlockResultsList()))) - .collect(Collectors.toList())); - return results; - } - - /** - * Gets the cluster Id and Scm Id from SCM. - * @return ScmInfo - * @throws IOException - */ - @Override - public ScmInfo getScmInfo() throws IOException { - OzoneProtos.GetScmInfoRequestProto request = - OzoneProtos.GetScmInfoRequestProto.getDefaultInstance(); - OzoneProtos.GetScmInfoRespsonseProto resp; - try { - resp = rpcProxy.getScmInfo(NULL_RPC_CONTROLLER, request); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - ScmInfo.Builder builder = new ScmInfo.Builder() - .setClusterId(resp.getClusterId()) - .setScmId(resp.getScmId()); - return builder.build(); - } - - @Override - public Object getUnderlyingProxyObject() { - return rpcProxy; - } - - @Override - public void close() { - RPC.stopProxy(rpcProxy); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java deleted file mode 100644 index 256e735..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.protocolPB; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtocolInfo; -import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos - .ScmBlockLocationProtocolService; - -/** - * Protocol used from an HDFS node to StorageContainerManager. This extends the - * Protocol Buffers service interface to add Hadoop-specific annotations. - */ -@ProtocolInfo(protocolName = - "org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol", - protocolVersion = 1) -@InterfaceAudience.Private -public interface ScmBlockLocationProtocolPB - extends ScmBlockLocationProtocolService.BlockingInterface { -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
