http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java deleted file mode 100644 index 85f8d6f..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java deleted file mode 100644 index 93ed005..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cblock.kubernetes; - -import com.google.gson.reflect.TypeToken; -import com.squareup.okhttp.RequestBody; -import io.kubernetes.client.ApiClient; -import io.kubernetes.client.ApiException; -import io.kubernetes.client.Configuration; -import io.kubernetes.client.apis.CoreV1Api; -import io.kubernetes.client.models.V1ISCSIVolumeSource; -import io.kubernetes.client.models.V1ObjectMeta; -import io.kubernetes.client.models.V1ObjectReference; -import io.kubernetes.client.models.V1PersistentVolume; -import io.kubernetes.client.models.V1PersistentVolumeClaim; -import io.kubernetes.client.models.V1PersistentVolumeSpec; -import io.kubernetes.client.util.Config; -import io.kubernetes.client.util.Watch; -import okio.Buffer; -import org.apache.hadoop.cblock.cli.CBlockCli; -import org.apache.hadoop.cblock.exception.CBlockException; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.storage.StorageManager; -import org.apache.hadoop.conf.OzoneConfiguration; -import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.SocketTimeoutException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_IP; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_PORT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_CBLOCK_USER; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY; - -/** - * Kubernetes Dynamic Persistent Volume provisioner. - * - * Listens on the kubernetes feed and creates the appropriate cblock AND - * kubernetes PersistentVolume according to the created PersistentVolumeClaims. - */ -public class DynamicProvisioner implements Runnable{ - - protected static final Logger LOGGER = - LoggerFactory.getLogger(DynamicProvisioner.class); - - private static final String STORAGE_CLASS = "cblock"; - - private static final String PROVISIONER_ID = "hadoop.apache.org/cblock"; - private static final String KUBERNETES_PROVISIONER_KEY = - "volume.beta.kubernetes.io/storage-provisioner"; - private static final String KUBERNETES_BIND_COMPLETED_KEY = - "pv.kubernetes.io/bind-completed"; - - private boolean running = true; - - private final StorageManager storageManager; - - private String kubernetesConfigFile; - - private String externalIp; - - private int externalPort; - - private String cblockUser; - - private CoreV1Api api; - - private ApiClient client; - - private Thread watcherThread; - - public DynamicProvisioner(OzoneConfiguration ozoneConf, - StorageManager storageManager) throws IOException { - this.storageManager = storageManager; - - kubernetesConfigFile = ozoneConf - .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY); - - String jscsiServerAddress = ozoneConf - .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY, - DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT); - - externalIp = ozoneConf. - getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress); - - externalPort = ozoneConf. - getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT, - DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT); - - cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER, - DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT); - - - } - - public void init() throws IOException { - if (kubernetesConfigFile != null) { - client = Config.fromConfig(kubernetesConfigFile); - } else { - client = Config.fromCluster(); - } - client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS); - Configuration.setDefaultApiClient(client); - api = new CoreV1Api(); - - watcherThread = new Thread(this); - watcherThread.setName("DynamicProvisioner"); - watcherThread.setDaemon(true); - } - - @Override - public void run() { - LOGGER.info("Starting kubernetes dynamic provisioner."); - while (running) { - String resourceVersion = null; - try { - - Watch<V1PersistentVolumeClaim> watch = Watch.createWatch(client, - api.listPersistentVolumeClaimForAllNamespacesCall(null, - null, - false, - null, - null, - null, - resourceVersion, - null, - true, - null, - null), - new TypeToken<Watch.Response<V1PersistentVolumeClaim>>() { - }.getType()); - - - //check the new pvc resources, and create cblock + pv if needed - for (Watch.Response<V1PersistentVolumeClaim> item : watch) { - V1PersistentVolumeClaim claim = item.object; - - if (isPvMissingForPvc(claim)) { - - LOGGER.info("Provisioning volumes for PVC {}/{}", - claim.getMetadata().getNamespace(), - claim.getMetadata().getName()); - - if (LOGGER.isDebugEnabled()) { - RequestBody request = - api.getApiClient().serialize(claim, "application/json"); - - final Buffer buffer = new Buffer(); - request.writeTo(buffer); - LOGGER.debug("New PVC is detected: " + buffer.readUtf8()); - } - - String volumeName = createVolumeName(claim); - - long size = CBlockCli.parseSize( - claim.getSpec().getResources().getRequests().get("storage")); - - createCBlock(volumeName, size); - createPersistentVolumeFromPVC(item.object, volumeName); - } - } - } catch (Exception ex) { - if (ex.getCause() != null && ex - .getCause() instanceof SocketTimeoutException) { - //This is normal. We are connection to the kubernetes server and the - //connection should be reopened time to time... - LOGGER.debug("Time exception occured", ex); - } else { - LOGGER.error("Error on provisioning persistent volumes.", ex); - try { - //we can try again in the main loop - Thread.sleep(1000); - } catch (InterruptedException e) { - LOGGER.error("Error on sleeping after an error.", e); - } - } - } - } - } - - private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) { - - Map<String, String> annotations = claim.getMetadata().getAnnotations(); - - return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS - .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID - .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes" - .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY)); - } - - @VisibleForTesting - protected String createVolumeName(V1PersistentVolumeClaim claim) { - return claim.getMetadata().getName() + "-" + claim.getMetadata() - .getUid(); - } - - public void stop() { - running = false; - try { - watcherThread.join(60000); - } catch (InterruptedException e) { - LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e); - } - } - - private void createCBlock(String volumeName, long size) - throws CBlockException { - - MountVolumeResponse mountVolumeResponse = - storageManager.isVolumeValid(cblockUser, volumeName); - if (!mountVolumeResponse.getIsValid()) { - storageManager - .createVolume(cblockUser, volumeName, size, 4 * 1024); - } - } - - private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim, - String volumeName) throws ApiException, IOException { - - V1PersistentVolume v1PersistentVolume = - persitenceVolumeBuilder(claim, volumeName); - - if (LOGGER.isDebugEnabled()) { - RequestBody request = - api.getApiClient().serialize(v1PersistentVolume, "application/json"); - - final Buffer buffer = new Buffer(); - request.writeTo(buffer); - LOGGER.debug("Creating new PV: " + buffer.readUtf8()); - } - api.createPersistentVolume(v1PersistentVolume, null); - } - - protected V1PersistentVolume persitenceVolumeBuilder( - V1PersistentVolumeClaim claim, - String volumeName) { - - V1PersistentVolume v1PersistentVolume = new V1PersistentVolume(); - v1PersistentVolume.setKind("PersistentVolume"); - v1PersistentVolume.setApiVersion("v1"); - - V1ObjectMeta metadata = new V1ObjectMeta(); - metadata.setName(volumeName); - metadata.setNamespace(claim.getMetadata().getNamespace()); - metadata.setAnnotations(new HashMap<>()); - - metadata.getAnnotations() - .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID); - - metadata.getAnnotations() - .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS); - - v1PersistentVolume.setMetadata(metadata); - - V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec(); - - spec.setCapacity(new HashMap<>()); - spec.getCapacity().put("storage", - claim.getSpec().getResources().getRequests().get("storage")); - - spec.setAccessModes(new ArrayList<>()); - spec.getAccessModes().add("ReadWriteOnce"); - - V1ObjectReference claimRef = new V1ObjectReference(); - claimRef.setName(claim.getMetadata().getName()); - claimRef.setNamespace(claim.getMetadata().getNamespace()); - claimRef.setKind(claim.getKind()); - claimRef.setApiVersion(claim.getApiVersion()); - claimRef.setUid(claim.getMetadata().getUid()); - spec.setClaimRef(claimRef); - - spec.persistentVolumeReclaimPolicy("Delete"); - - V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource(); - iscsi.setIqn(cblockUser + ":" + volumeName); - iscsi.setLun(0); - iscsi.setFsType("ext4"); - String portal = externalIp + ":" + externalPort; - iscsi.setTargetPortal(portal); - iscsi.setPortals(new ArrayList<>()); - iscsi.getPortals().add(portal); - - spec.iscsi(iscsi); - v1PersistentVolume.setSpec(spec); - return v1PersistentVolume; - } - - - @VisibleForTesting - protected CoreV1Api getApi() { - return api; - } - - public void start() { - watcherThread.start(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java deleted file mode 100644 index 3ec5aab..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This package contains helper classes to run hadoop cluster in kubernetes - * environment. - */ -package org.apache.hadoop.cblock.kubernetes; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java deleted file mode 100644 index 00064a6..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.meta; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -/** - * - * The internal representation of a container maintained by CBlock server. - * Include enough information to exactly identify a container for read/write - * operation. - * - * NOTE that this class is work-in-progress. Depends on HDFS-7240 container - * implementation. Currently only to allow testing. - */ -public class ContainerDescriptor { - private final String containerID; - // the index of this container with in a volume - // on creation, there may be no way to know the index of the container - // as it is a volume specific information - private int containerIndex; - private Pipeline pipeline; - - public ContainerDescriptor(String containerID) { - this.containerID = containerID; - } - - public ContainerDescriptor(String containerID, int containerIndex) { - this.containerID = containerID; - this.containerIndex = containerIndex; - } - - public void setContainerIndex(int idx) { - this.containerIndex = idx; - } - - public String getContainerID() { - return containerID; - } - - public void setPipeline(Pipeline pipeline) { - this.pipeline = pipeline; - } - - public Pipeline getPipeline() { - return pipeline; - } - - public int getContainerIndex() { - return containerIndex; - } - - public long getUtilization() { - return 0; - } - - public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() { - CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder = - CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder(); - builder.setContainerID(containerID); - builder.setIndex(containerIndex); - if (pipeline != null) { - builder.setPipeline(pipeline.getProtobufMessage()); - } - return builder.build(); - } - - public static ContainerDescriptor fromProtobuf(byte[] data) - throws InvalidProtocolBufferException { - CBlockClientServerProtocolProtos.ContainerIDProto id = - CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data); - return new ContainerDescriptor(id.getContainerID(), - (int)id.getIndex()); - } - - @Override - public int hashCode() { - return containerID.hashCode()*37 + containerIndex; - } - - @Override - public boolean equals(Object o) { - if (o != null && o instanceof ContainerDescriptor) { - ContainerDescriptor other = (ContainerDescriptor)o; - return containerID.equals(other.containerID) && - containerIndex == other.containerIndex; - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java deleted file mode 100644 index 4f5930d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.meta; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * The internal representation maintained by CBlock server as the info for - * a volume. Contains the list of containers belonging to this volume. - * - * Many methods of this class is made such that the volume information ( - * including container list) can be easily transformed into a Json string - * that can be stored/parsed from a persistent store for cblock server - * persistence. - * - * This class is still work-in-progress. - */ -public class VolumeDescriptor { - // The main data structure is the container location map - // other thing are mainly just information - - // since only one operation at a time is allowed, no - // need to consider concurrency control here - - // key is container id - - private static final Logger LOG = - LoggerFactory.getLogger(VolumeDescriptor.class); - - private ConcurrentHashMap<String, ContainerDescriptor> containerMap; - private String userName; - private int blockSize; - private long volumeSize; - private String volumeName; - // this is essentially the ordered keys of containerMap - // which is kind of redundant information. But since we - // are likely to access it frequently based on ordering. - // keeping this copy to avoid having to sort the key every - // time - private List<String> containerIdOrdered; - - /** - * This is not being called explicitly, but this is necessary as - * it will be called by the parse method implicitly when - * reconstructing the object from json string. The get*() methods - * and set*() methods are for the same purpose also. - */ - public VolumeDescriptor() { - this(null, null, 0, 0); - } - - public VolumeDescriptor(String userName, String volumeName, long volumeSize, - int blockSize) { - this.containerMap = new ConcurrentHashMap<>(); - this.userName = userName; - this.volumeName = volumeName; - this.blockSize = blockSize; - this.volumeSize = volumeSize; - this.containerIdOrdered = new LinkedList<>(); - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getVolumeName() { - return volumeName; - } - - public void setVolumeName(String volumeName) { - this.volumeName = volumeName; - } - - public long getVolumeSize() { - return volumeSize; - } - - public void setVolumeSize(long volumeSize) { - this.volumeSize = volumeSize; - } - - public int getBlockSize() { - return blockSize; - } - - public void setBlockSize(int blockSize) { - this.blockSize = blockSize; - } - - public void setContainerIDs(ArrayList<String> containerIDs) { - containerIdOrdered.addAll(containerIDs); - } - - public void addContainer(ContainerDescriptor containerDescriptor) { - containerMap.put(containerDescriptor.getContainerID(), - containerDescriptor); - } - - - public HashMap<String, Pipeline> getPipelines() { - HashMap<String, Pipeline> pipelines = new HashMap<>(); - for (Map.Entry<String, ContainerDescriptor> entry : - containerMap.entrySet()) { - pipelines.put(entry.getKey(), entry.getValue().getPipeline()); - } - return pipelines; - } - - public boolean isEmpty() { - VolumeInfo info = getInfo(); - return info.getUsage() == 0; - } - - public VolumeInfo getInfo() { - // TODO : need to actually go through all containers of this volume and - // ask for their utilization. - long utilization = 0; - for (Map.Entry<String, ContainerDescriptor> entry : - containerMap.entrySet()) { - utilization += entry.getValue().getUtilization(); - } - return new VolumeInfo(this.userName, this.volumeName, - this.volumeSize, this.blockSize, - utilization * blockSize); - } - - public String[] getContainerIDs() { - //ArrayList<Long> ids = new ArrayList(containerMap.keySet()); - //return ids.toArray(new Long[ids.size()]); - return containerIdOrdered.toArray(new String[containerIdOrdered.size()]); - } - - public List<String> getContainerIDsList() { - return new ArrayList<>(containerIdOrdered); - } - - public List<Pipeline> getContainerPipelines() { - Map<String, Pipeline> tmp = getPipelines(); - List<Pipeline> pipelineList = new LinkedList<>(); - for (String containerIDString : containerIdOrdered) { - pipelineList.add(tmp.get(containerIDString)); - } - return pipelineList; - } - - @Override - public String toString() { - String string = ""; - string += "Username:" + userName + "\n"; - string += "VolumeName:" + volumeName + "\n"; - string += "VolumeSize:" + volumeSize + "\n"; - string += "blockSize:" + blockSize + "\n"; - string += "containerIds:" + containerIdOrdered + "\n"; - string += "containerIdsWithObject:" + containerMap.keySet(); - return string; - } - - public CBlockClientServerProtocolProtos.MountVolumeResponseProto - toProtobuf() { - CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume = - CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder(); - volume.setIsValid(true); - volume.setVolumeName(volumeName); - volume.setUserName(userName); - volume.setVolumeSize(volumeSize); - volume.setBlockSize(blockSize); - for (String containerIDString : containerIdOrdered) { - ContainerDescriptor containerDescriptor = containerMap.get( - containerIDString); - volume.addAllContainerIDs(containerDescriptor.toProtobuf()); - } - return volume.build(); - } - - public static VolumeDescriptor fromProtobuf(byte[] data) - throws InvalidProtocolBufferException { - CBlockClientServerProtocolProtos.MountVolumeResponseProto volume = - CBlockClientServerProtocolProtos.MountVolumeResponseProto - .parseFrom(data); - String userName = volume.getUserName(); - String volumeName = volume.getVolumeName(); - long volumeSize = volume.getVolumeSize(); - int blockSize = volume.getBlockSize(); - VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName, - volumeName, volumeSize, blockSize); - List<CBlockClientServerProtocolProtos.ContainerIDProto> containers - = volume.getAllContainerIDsList(); - - String[] containerOrdering = new String[containers.size()]; - - for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto : - containers) { - ContainerDescriptor containerDescriptor = new ContainerDescriptor( - containerProto.getContainerID(), - (int)containerProto.getIndex()); - if(containerProto.hasPipeline()) { - containerDescriptor.setPipeline( - Pipeline.getFromProtoBuf(containerProto.getPipeline())); - } - volumeDescriptor.addContainer(containerDescriptor); - containerOrdering[containerDescriptor.getContainerIndex()] = - containerDescriptor.getContainerID(); - } - volumeDescriptor.setContainerIDs( - new ArrayList<>(Arrays.asList(containerOrdering))); - return volumeDescriptor; - } - - @Override - public int hashCode() { - return userName.hashCode()*37 + volumeName.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (o != null && o instanceof VolumeDescriptor) { - VolumeDescriptor other = (VolumeDescriptor)o; - if (!userName.equals(other.getUserName()) || - !volumeName.equals(other.getVolumeName()) || - volumeSize != other.getVolumeSize() || - blockSize != other.getBlockSize()) { - return false; - } - if (containerIdOrdered.size() != other.containerIdOrdered.size() || - containerMap.size() != other.containerMap.size()) { - return false; - } - for (int i = 0; i<containerIdOrdered.size(); i++) { - if (!containerIdOrdered.get(i).equals( - other.containerIdOrdered.get(i))) { - return false; - } - } - return containerMap.equals(other.containerMap); - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java deleted file mode 100644 index 7f50c41..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.meta; - -/** - * A wrapper class that represents the information about a volume. Used in - * communication between CBlock client and CBlock server only. - */ -public class VolumeInfo { - private final String userName; - private final String volumeName; - private final long volumeSize; - private final long blockSize; - private final long usage; - - public VolumeInfo(String userName, String volumeName, long volumeSize, - long blockSize, long usage) { - this.userName = userName; - this.volumeName = volumeName; - this.volumeSize = volumeSize; - this.blockSize = blockSize; - this.usage = usage; - } - - // When listing volume, the usage will not be set. - public VolumeInfo(String userName, String volumeName, long volumeSize, - long blockSize) { - this.userName = userName; - this.volumeName = volumeName; - this.volumeSize = volumeSize; - this.blockSize = blockSize; - this.usage = -1; - } - - public long getVolumeSize() { - return volumeSize; - } - - public long getBlockSize() { - return blockSize; - } - - public long getUsage() { - return usage; - } - - public String getUserName() { - return userName; - } - - public String getVolumeName() { - return volumeName; - } - - @Override - public String toString() { - return " userName:" + userName + - " volumeName:" + volumeName + - " volumeSize:" + volumeSize + - " blockSize:" + blockSize + - " (sizeInBlocks:" + volumeSize/blockSize + ")" + - " usageInBlocks:" + usage; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java deleted file mode 100644 index a331d7a..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.meta; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java deleted file mode 100644 index a7d5d8b..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java deleted file mode 100644 index fc40cef..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; - -import org.apache.hadoop.cblock.meta.VolumeInfo; - -import java.io.IOException; -import java.util.List; - -/** - * The protocol that CBlock client side uses to talk to server side. CBlock - * client is the point where a volume is mounted. All the actual volume IO - * operations will go through CBlock client after the volume is mounted. - * - * When users mount a volume on CBlock client, CBlock client side uses this - * protocol to send mount request to CBlock server. - */ -public interface CBlockClientProtocol { - MountVolumeResponse mountVolume(String userName, String volumeName) - throws IOException; - - List<VolumeInfo> listVolumes() throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java deleted file mode 100644 index bf00bc0..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; - -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.classification.InterfaceAudience; - -import java.io.IOException; -import java.util.List; - -/** - * CBlock uses a separate command line tool to send volume management - * operations to CBlock server, including create/delete/info/list volumes. This - * is the protocol used by the command line tool to send these requests and get - * responses from CBlock server. - */ -@InterfaceAudience.Private -public interface CBlockServiceProtocol { - - void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws IOException; - - void deleteVolume(String userName, String volumeName, - boolean force) throws IOException; - - VolumeInfo infoVolume(String userName, - String volumeName) throws IOException; - - List<VolumeInfo> listVolume(String userName) throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java deleted file mode 100644 index 70ccd90..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; - -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -import java.util.HashMap; -import java.util.List; - -/** - * The response message of mounting a volume. Including enough information - * for the client to communicate (perform IO) with the volume containers - * directly. - */ -public class MountVolumeResponse { - private final boolean isValid; - private final String userName; - private final String volumeName; - private final long volumeSize; - private final int blockSize; - private List<Pipeline> containerList; - private HashMap<String, Pipeline> pipelineMap; - - public MountVolumeResponse(boolean isValid, String userName, - String volumeName, long volumeSize, int blockSize, - List<Pipeline> containerList, - HashMap<String, Pipeline> pipelineMap) { - this.isValid = isValid; - this.userName = userName; - this.volumeName = volumeName; - this.volumeSize = volumeSize; - this.blockSize = blockSize; - this.containerList = containerList; - this.pipelineMap = pipelineMap; - } - - public boolean getIsValid() { - return isValid; - } - - public String getUserName() { - return userName; - } - - public String getVolumeName() { - return volumeName; - } - - public long getVolumeSize() { - return volumeSize; - } - - public int getBlockSize() { - return blockSize; - } - - public List<Pipeline> getContainerList() { - return containerList; - } - - public HashMap<String, Pipeline> getPipelineMap() { - return pipelineMap; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java deleted file mode 100644 index 33438ec..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java deleted file mode 100644 index 99f3110..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtocolInfo; - -/** - * This is the protocol CBlock client uses to talk to CBlock server. - * CBlock client is the mounting point of a volume. When a user mounts a - * volume, the cBlock client running on the local node will use this protocol - * to talk to CBlock server to mount the volume. - */ -@ProtocolInfo(protocolName = - "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol", - protocolVersion = 1) -@InterfaceAudience.Private -public interface CBlockClientServerProtocolPB extends - CBlockClientServerProtocolProtos - .CBlockClientServerProtocolService.BlockingInterface { -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java deleted file mode 100644 index bfe2130..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockClientProtocol; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.stream.Collectors; - -/** - * The server side implementation of cblock client to server protocol. - */ -@InterfaceAudience.Private -public class CBlockClientServerProtocolServerSideTranslatorPB implements - CBlockClientServerProtocolPB { - - private final CBlockClientProtocol impl; - - public CBlockClientServerProtocolServerSideTranslatorPB( - CBlockClientProtocol impl) { - this.impl = impl; - } - - @Override - public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume( - RpcController controller, - CBlockClientServerProtocolProtos.MountVolumeRequestProto request) - throws ServiceException { - String userName = request.getUserName(); - String volumeName = request.getVolumeName(); - CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder - resp = - CBlockClientServerProtocolProtos - .MountVolumeResponseProto.newBuilder(); - try { - MountVolumeResponse result = impl.mountVolume(userName, volumeName); - boolean isValid = result.getIsValid(); - resp.setIsValid(isValid); - if (isValid) { - resp.setUserName(result.getUserName()); - resp.setVolumeName(result.getVolumeName()); - resp.setVolumeSize(result.getVolumeSize()); - resp.setBlockSize(result.getBlockSize()); - List<Pipeline> containers = result.getContainerList(); - HashMap<String, Pipeline> pipelineMap = result.getPipelineMap(); - - for (int i=0; i<containers.size(); i++) { - CBlockClientServerProtocolProtos.ContainerIDProto.Builder id = - CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder(); - String containerName = containers.get(i).getContainerName(); - id.setContainerID(containerName); - id.setIndex(i); - if (pipelineMap.containsKey(containerName)) { - id.setPipeline(pipelineMap.get(containerName).getProtobufMessage()); - } - resp.addAllContainerIDs(id.build()); - } - } - } catch (IOException e) { - throw new ServiceException(e); - } - return resp.build(); - } - - @Override - public CBlockClientServerProtocolProtos.ListVolumesResponseProto listVolumes( - RpcController controller, - CBlockClientServerProtocolProtos.ListVolumesRequestProto request) - throws ServiceException { - try { - CBlockClientServerProtocolProtos.ListVolumesResponseProto.Builder resp = - CBlockClientServerProtocolProtos.ListVolumesResponseProto - .newBuilder(); - List<VolumeInfo> volumeInfos = impl.listVolumes(); - List<CBlockServiceProtocolProtos.VolumeInfoProto> convertedInfos = - volumeInfos.stream().map( - volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto - .newBuilder().setUserName(volumeInfo.getUserName()) - .setBlockSize(volumeInfo.getBlockSize()) - .setVolumeName(volumeInfo.getVolumeName()) - .setVolumeSize(volumeInfo.getVolumeSize()) - .setUsage(volumeInfo.getUsage()).build()) - .collect(Collectors.toList()); - resp.addAllVolumeEntry(convertedInfos); - return resp.build(); - } catch (IOException e) { - throw new ServiceException(e); - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java deleted file mode 100644 index 282d6cd..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtocolInfo; - -/** - * Users use a independent command line tool to talk to CBlock server for - * volume operations (create/delete/info/list). This is the protocol used by - * the the command line tool to send these requests to CBlock server. - */ -@ProtocolInfo(protocolName = - "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol", - protocolVersion = 1) -@InterfaceAudience.Private -public interface CBlockServiceProtocolPB extends - CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface { -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java deleted file mode 100644 index 8924a0c..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockServiceProtocol; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT; - -/** - * Server side implementation of the protobuf service. - */ -@InterfaceAudience.Private -public class CBlockServiceProtocolServerSideTranslatorPB - implements CBlockServiceProtocolPB { - - private final CBlockServiceProtocol impl; - private static final Logger LOG = - LoggerFactory.getLogger( - CBlockServiceProtocolServerSideTranslatorPB.class); - - @Override - public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume( - RpcController controller, - CBlockServiceProtocolProtos.CreateVolumeRequestProto request) - throws ServiceException { - if (LOG.isDebugEnabled()) { - LOG.debug("createVolume called! volume size: " + request.getVolumeSize() - + " block size: " + request.getBlockSize()); - } - try { - if (request.hasBlockSize()) { - impl.createVolume(request.getUserName(), request.getVolumeName(), - request.getVolumeSize(), request.getBlockSize()); - } else{ - impl.createVolume(request.getUserName(), request.getVolumeName(), - request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT); - } - } catch (IOException e) { - throw new ServiceException(e); - } - return CBlockServiceProtocolProtos.CreateVolumeResponseProto - .newBuilder().build(); - } - - @Override - public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume( - RpcController controller, - CBlockServiceProtocolProtos.DeleteVolumeRequestProto request) - throws ServiceException { - if (LOG.isDebugEnabled()) { - LOG.debug("deleteVolume called! volume name: " + request.getVolumeName() - + " force:" + request.getForce()); - } - try { - if (request.hasForce()) { - impl.deleteVolume(request.getUserName(), request.getVolumeName(), - request.getForce()); - } else { - impl.deleteVolume(request.getUserName(), request.getVolumeName(), - false); - } - } catch (IOException e) { - throw new ServiceException(e); - } - return CBlockServiceProtocolProtos.DeleteVolumeResponseProto - .newBuilder().build(); - } - - @Override - public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume( - RpcController controller, - CBlockServiceProtocolProtos.InfoVolumeRequestProto request - ) throws ServiceException { - if (LOG.isDebugEnabled()) { - LOG.debug("infoVolume called! volume name: " + request.getVolumeName()); - } - CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp = - CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder(); - CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto = - CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder(); - VolumeInfo volumeInfo; - try { - volumeInfo = impl.infoVolume(request.getUserName(), - request.getVolumeName()); - } catch (IOException e) { - throw new ServiceException(e); - } - - volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize()); - volumeInfoProto.setBlockSize(volumeInfo.getBlockSize()); - volumeInfoProto.setUsage(volumeInfo.getUsage()); - volumeInfoProto.setUserName(volumeInfo.getUserName()); - volumeInfoProto.setVolumeName(volumeInfo.getVolumeName()); - resp.setVolumeInfo(volumeInfoProto); - return resp.build(); - } - - @Override - public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume( - RpcController controller, - CBlockServiceProtocolProtos.ListVolumeRequestProto request - ) throws ServiceException { - CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp = - CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder(); - String userName = null; - if (request.hasUserName()) { - userName = request.getUserName(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("list volume received for :" + userName); - } - List<VolumeInfo> volumes; - try { - volumes = impl.listVolume(userName); - } catch (IOException e) { - throw new ServiceException(e); - } - for (VolumeInfo volume : volumes) { - CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto - = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder(); - volumeEntryProto.setUserName(volume.getUserName()); - volumeEntryProto.setVolumeName(volume.getVolumeName()); - volumeEntryProto.setVolumeSize(volume.getVolumeSize()); - volumeEntryProto.setBlockSize(volume.getBlockSize()); - resp.addVolumeEntry(volumeEntryProto.build()); - } - return resp.build(); - } - - public CBlockServiceProtocolServerSideTranslatorPB( - CBlockServiceProtocol impl) { - this.impl = impl; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java deleted file mode 100644 index 5e03a92..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java deleted file mode 100644 index 865f3b2..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.storage; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.cblock.CBlockConfigKeys; -import org.apache.hadoop.cblock.exception.CBlockException; -import org.apache.hadoop.cblock.meta.ContainerDescriptor; -import org.apache.hadoop.cblock.meta.VolumeDescriptor; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.util.KeyUtil; -import org.apache.hadoop.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos; -import org.apache.hadoop.scm.client.ScmClient; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class maintains the key space of CBlock, more specifically, the - * volume to container mapping. The core data structure - * is a map from users to their volumes info, where volume info is a handler - * to a volume, containing information for IO on that volume and a storage - * client responsible for talking to the SCM. - */ -public class StorageManager { - private static final Logger LOGGER = - LoggerFactory.getLogger(StorageManager.class); - private final ScmClient storageClient; - private final int numThreads; - private static final int MAX_THREADS = - Runtime.getRuntime().availableProcessors() * 2; - private static final int MAX_QUEUE_CAPACITY = 1024; - private final String cblockId; - - /** - * We will NOT have the situation where same kv pair getting - * processed, but it is possible to have multiple kv pair being - * processed at same time. - * - * So using just ConcurrentHashMap should be sufficient - * - * Again since currently same user accessing from multiple places - * is not allowed, no need to consider concurrency of volume map - * within one user - */ - private ConcurrentHashMap<String, HashMap<String, VolumeDescriptor>> - user2VolumeMap; - // size of an underlying container. - // TODO : assuming all containers are of the same size - private long containerSizeB; - - public StorageManager(ScmClient storageClient, - OzoneConfiguration ozoneConfig, String cblockId) throws IOException { - this.storageClient = storageClient; - this.user2VolumeMap = new ConcurrentHashMap<>(); - this.containerSizeB = storageClient.getContainerSize(null); - this.numThreads = - ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE, - CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT); - this.cblockId = cblockId; - } - - /** - * This call will put the volume into in-memory map. - * - * more specifically, make the volume discoverable on jSCSI server - * and keep it's reference in-memory for look up. - * @param userName the user name of the volume. - * @param volumeName the name of the volume, - * @param volume a {@link VolumeDescriptor} object encapsulating the - * information about the volume. - */ - private void makeVolumeReady(String userName, String volumeName, - VolumeDescriptor volume) { - HashMap<String, VolumeDescriptor> userVolumes; - if (user2VolumeMap.containsKey(userName)) { - userVolumes = user2VolumeMap.get(userName); - } else { - userVolumes = new HashMap<>(); - user2VolumeMap.put(userName, userVolumes); - } - userVolumes.put(volumeName, volume); - } - - /** - * Called by CBlockManager to add volumes read from persistent store into - * memory, need to contact SCM to setup the reference to the containers given - * their id. - * - * Only for failover process where container meta info is read from - * persistent store, and containers themselves are alive. - * - * TODO : Currently, this method is not being called as failover process - * is not implemented yet. - * - * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating - * the information about a volume. - * @throws IOException when adding the volume failed. e.g. volume already - * exist, or no more container available. - */ - public synchronized void addVolume(VolumeDescriptor volumeDescriptor) - throws IOException{ - String userName = volumeDescriptor.getUserName(); - String volumeName = volumeDescriptor.getVolumeName(); - LOGGER.info("addVolume:" + userName + ":" + volumeName); - if (user2VolumeMap.containsKey(userName) - && user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Volume already exist for " - + userName + ":" + volumeName); - } - // the container ids are read from levelDB, setting up the - // container handlers here. - String[] containerIds = volumeDescriptor.getContainerIDs(); - - for (String containerId : containerIds) { - try { - Pipeline pipeline = storageClient.getContainer(containerId); - ContainerDescriptor containerDescriptor = - new ContainerDescriptor(containerId); - containerDescriptor.setPipeline(pipeline); - volumeDescriptor.addContainer(containerDescriptor); - } catch (IOException e) { - LOGGER.error("Getting container failed! Container:{} error:{}", - containerId, e); - throw e; - } - } - // now ready to put into in-memory map. - makeVolumeReady(userName, volumeName, volumeDescriptor); - } - - private class CreateContainerTask implements Runnable { - private final VolumeDescriptor volume; - private final int containerIdx; - private final ArrayList<String> containerIds; - private final AtomicInteger numFailed; - - CreateContainerTask(VolumeDescriptor volume, int containerIdx, - ArrayList<String> containerIds, - AtomicInteger numFailed) { - this.volume = volume; - this.containerIdx = containerIdx; - this.containerIds = containerIds; - this.numFailed = numFailed; - } - - /** - * When an object implementing interface <code>Runnable</code> is used - * to create a thread, starting the thread causes the object's - * <code>run</code> method to be called in that separately executing - * thread. - * <p> - * The general contract of the method <code>run</code> is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - public void run() { - ContainerDescriptor container = null; - try { - Pipeline pipeline = storageClient.createContainer( - OzoneProtos.ReplicationType.STAND_ALONE, - OzoneProtos.ReplicationFactor.ONE, - KeyUtil.getContainerName(volume.getUserName(), - volume.getVolumeName(), containerIdx), cblockId); - - container = new ContainerDescriptor(pipeline.getContainerName()); - - container.setPipeline(pipeline); - container.setContainerIndex(containerIdx); - volume.addContainer(container); - containerIds.set(containerIdx, container.getContainerID()); - } catch (Exception e) { - numFailed.incrementAndGet(); - if (container != null) { - LOGGER.error("Error creating container Container:{}:" + - " index:{} error:{}", container.getContainerID(), - containerIdx, e); - } else { - LOGGER.error("Error creating container.", e); - } - } - } - } - - private boolean createVolumeContainers(VolumeDescriptor volume) { - ArrayList<String> containerIds = new ArrayList<>(); - ThreadPoolExecutor executor = new ThreadPoolExecutor( - Math.min(numThreads, MAX_THREADS), - MAX_THREADS, 1, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY), - new ThreadPoolExecutor.CallerRunsPolicy()); - - AtomicInteger numFailedCreates = new AtomicInteger(0); - long allocatedSize = 0; - int containerIdx = 0; - while (allocatedSize < volume.getVolumeSize()) { - // adding null to allocate space in ArrayList - containerIds.add(containerIdx, null); - Runnable task = new CreateContainerTask(volume, containerIdx, - containerIds, numFailedCreates); - executor.submit(task); - allocatedSize += containerSizeB; - containerIdx += 1; - } - - // issue the command and then wait for it to finish - executor.shutdown(); - try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); - } catch (InterruptedException e) { - LOGGER.error("Error creating volume:{} error:{}", - volume.getVolumeName(), e); - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - - volume.setContainerIDs(containerIds); - return numFailedCreates.get() == 0; - } - - private void deleteContainer(String containerID, boolean force) { - try { - Pipeline pipeline = storageClient.getContainer(containerID); - storageClient.deleteContainer(pipeline, force); - } catch (Exception e) { - LOGGER.error("Error deleting container Container:{} error:{}", - containerID, e); - } - } - - private void deleteVolumeContainers(List<String> containers, boolean force) - throws CBlockException { - ThreadPoolExecutor executor = new ThreadPoolExecutor( - Math.min(numThreads, MAX_THREADS), - MAX_THREADS, 1, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY), - new ThreadPoolExecutor.CallerRunsPolicy()); - - for (String deleteContainer : containers) { - if (deleteContainer != null) { - Runnable task = () -> deleteContainer(deleteContainer, force); - executor.submit(task); - } - } - - // issue the command and then wait for it to finish - executor.shutdown(); - try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); - } catch (InterruptedException e) { - LOGGER.error("Error deleting containers error:{}", e); - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - - /** - * Called by CBlock server when creating a fresh volume. The core - * logic is adding needed information into in-memory meta data. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume. - * @param volumeSize the size of the volume. - * @param blockSize the block size of the volume. - * @throws CBlockException when the volume can not be created. - */ - public synchronized void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws CBlockException { - LOGGER.debug("createVolume:" + userName + ":" + volumeName); - if (user2VolumeMap.containsKey(userName) - && user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Volume already exist for " - + userName + ":" + volumeName); - } - if (volumeSize < blockSize) { - throw new CBlockException("Volume size smaller than block size? " + - "volume size:" + volumeSize + " block size:" + blockSize); - } - VolumeDescriptor volume - = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize); - boolean success = createVolumeContainers(volume); - if (!success) { - // cleanup the containers and throw the exception - deleteVolumeContainers(volume.getContainerIDsList(), true); - throw new CBlockException("Error when creating volume:" + volumeName); - } - makeVolumeReady(userName, volumeName, volume); - } - - /** - * Called by CBlock server to delete a specific volume. Mainly - * to check whether it can be deleted, and remove it from in-memory meta - * data. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume. - * @param force if set to false, only delete volume it is empty, otherwise - * throw exception. if set to true, delete regardless. - * @throws CBlockException when the volume can not be deleted. - */ - public synchronized void deleteVolume(String userName, String volumeName, - boolean force) throws CBlockException { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Deleting non-exist volume " - + userName + ":" + volumeName); - } - if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) { - throw new CBlockException("Deleting a non-empty volume without force!"); - } - VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName); - deleteVolumeContainers(volume.getContainerIDsList(), force); - if (user2VolumeMap.get(userName).size() == 0) { - user2VolumeMap.remove(userName); - } - } - - /** - * Called by CBlock server to get information of a specific volume. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume. - * @return a {@link VolumeInfo} object encapsulating the information of the - * volume. - * @throws CBlockException when the information can not be retrieved. - */ - public synchronized VolumeInfo infoVolume(String userName, String volumeName) - throws CBlockException { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Getting info for non-exist volume " - + userName + ":" + volumeName); - } - return user2VolumeMap.get(userName).get(volumeName).getInfo(); - } - - /** - * Called by CBlock server to check whether the given volume can be - * mounted, i.e. whether it can be found in the meta data. - * - * return a {@link MountVolumeResponse} with isValid flag to indicate - * whether the volume can be mounted or not. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume - * @return a {@link MountVolumeResponse} object encapsulating whether the - * volume is valid, and if yes, the requried information for client to - * read/write the volume. - */ - public synchronized MountVolumeResponse isVolumeValid( - String userName, String volumeName) { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - // in the case of invalid volume, no need to set any value other than - // isValid flag. - return new MountVolumeResponse(false, null, null, 0, 0, null, null); - } - VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName); - return new MountVolumeResponse(true, userName, - volumeName, volume.getVolumeSize(), volume.getBlockSize(), - volume.getContainerPipelines(), volume.getPipelines()); - } - - /** - * Called by CBlock manager to list all volumes. - * - * @param userName the userName whose volume to be listed, if set to null, - * all volumes will be listed. - * @return a list of {@link VolumeDescriptor} representing all volumes - * requested. - */ - public synchronized List<VolumeDescriptor> getAllVolume(String userName) { - ArrayList<VolumeDescriptor> allVolumes = new ArrayList<>(); - if (userName == null) { - for (Map.Entry<String, HashMap<String, VolumeDescriptor>> entry - : user2VolumeMap.entrySet()) { - allVolumes.addAll(entry.getValue().values()); - } - } else { - if (user2VolumeMap.containsKey(userName)) { - allVolumes.addAll(user2VolumeMap.get(userName).values()); - } - } - return allVolumes; - } - - /** - * Only for testing the behavior of create/delete volumes. - */ - @VisibleForTesting - public VolumeDescriptor getVolume(String userName, String volumeName) { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - return null; - } - return user2VolumeMap.get(userName).get(volumeName); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java deleted file mode 100644 index 4426e6d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.storage; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java deleted file mode 100644 index beb9e32..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.util; - -/** - * A simply class that generates key mappings. (e.g. from (userName, volumeName) - * pair to a single string volumeKey. - */ -public final class KeyUtil { - public static String getVolumeKey(String userName, String volumeName) { - return userName + ":" + volumeName; - } - - public static String getContainerName(String userName, String volumeName, - int containerID) { - return getVolumeKey(userName, volumeName) + "#" + containerID; - } - - public static String getUserNameFromVolumeKey(String key) { - return key.split(":")[0]; - } - - public static String getVolumeFromVolumeKey(String key) { - return key.split(":")[1]; - } - - public static boolean isValidVolumeKey(String key) { - return key.contains(":"); - } - - private KeyUtil() { - - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java deleted file mode 100644 index 5b9aa0c..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.util; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 72f39fc..4c94e38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -36,8 +36,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -73,7 +71,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -1497,23 +1494,6 @@ public class DFSUtil { return password; } - public static boolean isOzoneEnabled(Configuration conf) { - String securityEnabled = conf.get(CommonConfigurationKeysPublic - .HADOOP_SECURITY_AUTHENTICATION, - "simple"); - boolean securityAuthorizationEnabled = conf.getBoolean( - CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, - false); - - if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) { - LOG.error("Ozone is not supported in a security enabled cluster. "); - return false; - } else { - return conf.getBoolean(OZONE_ENABLED, - OZONE_ENABLED_DEFAULT); - } - } - /** * Converts a Date into an ISO-8601 formatted datetime string. */ --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
