http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java deleted file mode 100644 index 860386d..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.protocolPB; - -/** - * This package contains classes for the Protocol Buffers binding of Ozone - * protocols. - */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java deleted file mode 100644 index 909873f..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.web.utils; - -import java.io.IOException; -import java.util.List; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.type.CollectionType; - -/** - * JSON Utility functions used in ozone. - */ -public final class JsonUtils { - - // Reuse ObjectMapper instance for improving performance. - // ObjectMapper is thread safe as long as we always configure instance - // before use. - private static final ObjectMapper MAPPER = new ObjectMapper(); - private static final ObjectReader READER = MAPPER.readerFor(Object.class); - private static final ObjectWriter WRITTER = - MAPPER.writerWithDefaultPrettyPrinter(); - - private JsonUtils() { - // Never constructed - } - - public static String toJsonStringWithDefaultPrettyPrinter(String jsonString) - throws IOException { - Object json = READER.readValue(jsonString); - return WRITTER.writeValueAsString(json); - } - - public static String toJsonString(Object obj) throws IOException { - return MAPPER.writeValueAsString(obj); - } - - /** - * Deserialize a list of elements from a given string, - * each element in the list is in the given type. - * - * @param str json string. - * @param elementType element type. - * @return List of elements of type elementType - * @throws IOException - */ - public static List<?> toJsonList(String str, Class<?> elementType) - throws IOException { - CollectionType type = MAPPER.getTypeFactory() - .constructCollectionType(List.class, elementType); - return MAPPER.readValue(str, type); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java deleted file mode 100644 index 9d15359..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * This class contains constants for configuration keys used in SCM. - */ -@InterfaceAudience.Public -@InterfaceStability.Unstable -public final class ScmConfigKeys { - - public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY = - "scm.container.client.idle.threshold"; - public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT = - "10s"; - - public static final String SCM_CONTAINER_CLIENT_MAX_SIZE_KEY = - "scm.container.client.max.size"; - public static final int SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT = - 256; - - public static final String SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS = - "scm.container.client.max.outstanding.requests"; - public static final int SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS_DEFAULT - = 100; - - public static final String DFS_CONTAINER_RATIS_ENABLED_KEY - = "dfs.container.ratis.enabled"; - public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT - = false; - public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY - = "dfs.container.ratis.rpc.type"; - public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT - = "GRPC"; - public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY - = "dfs.container.ratis.num.write.chunk.threads"; - public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT - = 60; - public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY = - "dfs.container.ratis.segment.size"; - public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT = - 1 * 1024 * 1024 * 1024; - public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY = - "dfs.container.ratis.segment.preallocated.size"; - public static final int - DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024; - - // TODO : this is copied from OzoneConsts, may need to move to a better place - public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size"; - // 16 MB by default - public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024; - public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024; - - public static final String OZONE_SCM_CLIENT_PORT_KEY = - "ozone.scm.client.port"; - public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860; - - public static final String OZONE_SCM_DATANODE_PORT_KEY = - "ozone.scm.datanode.port"; - public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861; - - // OZONE_KSM_PORT_DEFAULT = 9862 - public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY = - "ozone.scm.block.client.port"; - public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863; - - // Container service client - public static final String OZONE_SCM_CLIENT_ADDRESS_KEY = - "ozone.scm.client.address"; - public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY = - "ozone.scm.client.bind.host"; - public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT = - "0.0.0.0"; - - // Block service client - public static final String OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY = - "ozone.scm.block.client.address"; - public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY = - "ozone.scm.block.client.bind.host"; - public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT = - "0.0.0.0"; - - public static final String OZONE_SCM_DATANODE_ADDRESS_KEY = - "ozone.scm.datanode.address"; - public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY = - "ozone.scm.datanode.bind.host"; - public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT = - "0.0.0.0"; - - public static final String OZONE_SCM_HTTP_ENABLED_KEY = - "ozone.scm.http.enabled"; - public static final String OZONE_SCM_HTTP_BIND_HOST_KEY = - "ozone.scm.http-bind-host"; - public static final String OZONE_SCM_HTTPS_BIND_HOST_KEY = - "ozone.scm.https-bind-host"; - public static final String OZONE_SCM_HTTP_ADDRESS_KEY = - "ozone.scm.http-address"; - public static final String OZONE_SCM_HTTPS_ADDRESS_KEY = - "ozone.scm.https-address"; - public static final String OZONE_SCM_KEYTAB_FILE = - "ozone.scm.keytab.file"; - public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; - public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876; - public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877; - - public static final String HDSL_REST_HTTP_ADDRESS_KEY = - "hdsl.rest.http-address"; - public static final String HDSL_REST_HTTP_ADDRESS_DEFAULT = "0.0.0.0:9880"; - public static final String HDSL_REST_CSRF_ENABLED_KEY = - "hdsl.rest.rest-csrf.enabled"; - public static final boolean HDSL_REST_CSRF_ENABLED_DEFAULT = false; - public static final String HDSL_REST_NETTY_HIGH_WATERMARK = - "hdsl.rest.netty.high.watermark"; - public static final int HDSL_REST_NETTY_HIGH_WATERMARK_DEFAULT = 65536; - public static final int HDSL_REST_NETTY_LOW_WATERMARK_DEFAULT = 32768; - public static final String HDSL_REST_NETTY_LOW_WATERMARK = - "hdsl.rest.netty.low.watermark"; - - public static final String OZONE_SCM_HANDLER_COUNT_KEY = - "ozone.scm.handler.count.key"; - public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10; - - public static final String OZONE_SCM_HEARTBEAT_INTERVAL = - "ozone.scm.heartbeat.interval"; - public static final String OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT = - "30s"; - - public static final String OZONE_SCM_DEADNODE_INTERVAL = - "ozone.scm.dead.node.interval"; - public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT = - "10m"; - - public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS = - "ozone.scm.max.hb.count.to.process"; - public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000; - - public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL = - "ozone.scm.heartbeat.thread.interval"; - public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT = - "3s"; - - public static final String OZONE_SCM_STALENODE_INTERVAL = - "ozone.scm.stale.node.interval"; - public static final String OZONE_SCM_STALENODE_INTERVAL_DEFAULT = - "90s"; - - public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT = - "ozone.scm.heartbeat.rpc-timeout"; - public static final long OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT = - 1000; - - /** - * Defines how frequently we will log the missing of heartbeat to a specific - * SCM. In the default case we will write a warning message for each 10 - * sequential heart beats that we miss to a specific SCM. This is to avoid - * overrunning the log with lots of HB missed Log statements. - */ - public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT = - "ozone.scm.heartbeat.log.warn.interval.count"; - public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT = - 10; - - // ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT. - // Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777 - // - // If this key is not specified datanodes will not be able to find - // SCM. The SCM membership can be dynamic, so this key should contain - // all possible SCM names. Once the SCM leader is discovered datanodes will - // get the right list of SCMs to heartbeat to from the leader. - // While it is good for the datanodes to know the names of all SCM nodes, - // it is sufficient to actually know the name of on working SCM. That SCM - // will be able to return the information about other SCMs that are part of - // the SCM replicated Log. - // - //In case of a membership change, any one of the SCM machines will be - // able to send back a new list to the datanodes. - public static final String OZONE_SCM_NAMES = "ozone.scm.names"; - - public static final int OZONE_SCM_DEFAULT_PORT = - OZONE_SCM_DATANODE_PORT_DEFAULT; - // File Name and path where datanode ID is to written to. - // if this value is not set then container startup will fail. - public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id"; - - public static final String OZONE_SCM_DATANODE_ID_PATH_DEFAULT = "datanode.id"; - - public static final String OZONE_SCM_DB_CACHE_SIZE_MB = - "ozone.scm.db.cache.size.mb"; - public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128; - - public static final String OZONE_SCM_CONTAINER_SIZE_GB = - "ozone.scm.container.size.gb"; - public static final int OZONE_SCM_CONTAINER_SIZE_DEFAULT = 5; - - public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY = - "ozone.scm.container.placement.impl"; - - public static final String OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE = - "ozone.scm.container.provision_batch_size"; - public static final int OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT = 20; - - public static final String OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY = - "ozone.scm.container.deletion-choosing.policy"; - - public static final String OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT = - "ozone.scm.container.creation.lease.timeout"; - - public static final String - OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s"; - - /** - * Don't start processing a pool if we have not had a minimum number of - * seconds from the last processing. - */ - public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL = - "ozone.scm.container.report.processing.interval"; - public static final String - OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s"; - - /** - * This determines the total number of pools to be processed in parallel. - */ - public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS = - "ozone.scm.max.nodepool.processing.threads"; - public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1; - /** - * These 2 settings control the number of threads in executor pool and time - * outs for thw container reports from all nodes. - */ - public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS = - "ozone.scm.max.container.report.threads"; - public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100; - public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT = - "ozone.scm.container.reports.wait.timeout"; - public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT = - "5m"; - - public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY = - "ozone.scm.block.deletion.max.retry"; - public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096; - - // Once a container usage crosses this threshold, it is eligible for - // closing. - public static final String OZONE_SCM_CONTAINER_CLOSE_THRESHOLD = - "ozone.scm.container.close.threshold"; - public static final float OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f; - /** - * Never constructed. - */ - private ScmConfigKeys() { - - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmInfo.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmInfo.java deleted file mode 100644 index e442fe2..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/ScmInfo.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm; - -/** - * ScmInfo wraps the result returned from SCM#getScmInfo which - * contains clusterId and the SCM Id. - */ -public final class ScmInfo { - private String clusterId; - private String scmId; - - /** - * Builder for ScmInfo. - */ - public static class Builder { - private String clusterId; - private String scmId; - - /** - * sets the cluster id. - * @param cid clusterId to be set - * @return Builder for ScmInfo - */ - public Builder setClusterId(String cid) { - this.clusterId = cid; - return this; - } - - /** - * sets the scmId. - * @param id scmId - * @return Builder for scmInfo - */ - public Builder setScmId(String id) { - this.scmId = id; - return this; - } - - public ScmInfo build() { - return new ScmInfo(clusterId, scmId); - } - } - - private ScmInfo(String clusterId, String scmId) { - this.clusterId = clusterId; - this.scmId = scmId; - } - - /** - * Gets the clusterId from the Version file. - * @return ClusterId - */ - public String getClusterId() { - return clusterId; - } - - /** - * Gets the SCM Id from the Version file. - * @return SCM Id - */ - public String getScmId() { - return scmId; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java deleted file mode 100644 index 49817d3..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hdsl.protocol.DatanodeDetails; -import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos - .ContainerCommandResponseProto; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * A Client for the storageContainer protocol. - */ -public abstract class XceiverClientSpi implements Closeable { - - final private AtomicInteger referenceCount; - private boolean isEvicted; - - XceiverClientSpi() { - this.referenceCount = new AtomicInteger(0); - this.isEvicted = false; - } - - void incrementReference() { - this.referenceCount.incrementAndGet(); - } - - void decrementReference() { - this.referenceCount.decrementAndGet(); - cleanup(); - } - - void setEvicted() { - isEvicted = true; - cleanup(); - } - - // close the xceiverClient only if, - // 1) there is no refcount on the client - // 2) it has been evicted from the cache. - private void cleanup() { - if (referenceCount.get() == 0 && isEvicted) { - close(); - } - } - - @VisibleForTesting - public int getRefcount() { - return referenceCount.get(); - } - - /** - * Connects to the leader in the pipeline. - */ - public abstract void connect() throws Exception; - - @Override - public abstract void close(); - - /** - * Returns the pipeline of machines that host the container used by this - * client. - * - * @return pipeline of machines that host the container - */ - public abstract Pipeline getPipeline(); - - /** - * Sends a given command to server and gets the reply back. - * @param request Request - * @return Response to the command - * @throws IOException - */ - public abstract ContainerCommandResponseProto sendCommand( - ContainerCommandRequestProto request) throws IOException; - - /** - * Sends a given command to server gets a waitable future back. - * - * @param request Request - * @return Response to the command - * @throws IOException - */ - public abstract CompletableFuture<ContainerCommandResponseProto> - sendCommandAsync(ContainerCommandRequestProto request) - throws IOException, ExecutionException, InterruptedException; - - /** - * Create a pipeline. - * - * @param pipelineID - Name of the pipeline. - * @param datanodes - Datanodes - */ - public abstract void createPipeline(String pipelineID, - List<DatanodeDetails> datanodes) throws IOException; - - /** - * Returns pipeline Type. - * - * @return - {Stand_Alone, Ratis or Chained} - */ - public abstract HdslProtos.ReplicationType getPipelineType(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/ScmClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/ScmClient.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/ScmClient.java deleted file mode 100644 index 79c6029..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/ScmClient.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.client; - -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.ContainerData; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos; -import org.apache.hadoop.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; - -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; - -/** - * The interface to call into underlying container layer. - * - * Written as interface to allow easy testing: implement a mock container layer - * for standalone testing of CBlock API without actually calling into remote - * containers. Actual container layer can simply re-implement this. - * - * NOTE this is temporarily needed class. When SCM containers are full-fledged, - * this interface will likely be removed. - */ -@InterfaceStability.Unstable -public interface ScmClient { - /** - * Creates a Container on SCM and returns the pipeline. - * @param containerId - String container ID - * @return Pipeline - * @throws IOException - */ - Pipeline createContainer(String containerId, String owner) throws IOException; - - /** - * Gets a container by Name -- Throws if the container does not exist. - * @param containerId - String Container ID - * @return Pipeline - * @throws IOException - */ - Pipeline getContainer(String containerId) throws IOException; - - /** - * Close a container by name. - * - * @param pipeline the container to be closed. - * @throws IOException - */ - void closeContainer(Pipeline pipeline) throws IOException; - - /** - * Deletes an existing container. - * @param pipeline - Pipeline that represents the container. - * @param force - true to forcibly delete the container. - * @throws IOException - */ - void deleteContainer(Pipeline pipeline, boolean force) throws IOException; - - /** - * Lists a range of containers and get their info. - * - * @param startName start name, if null, start searching at the head. - * @param prefixName prefix name, if null, then filter is disabled. - * @param count count, if count < 0, the max size is unlimited.( - * Usually the count will be replace with a very big - * value instead of being unlimited in case the db is very big) - * - * @return a list of pipeline. - * @throws IOException - */ - List<ContainerInfo> listContainer(String startName, String prefixName, - int count) throws IOException; - - /** - * Read meta data from an existing container. - * @param pipeline - Pipeline that represents the container. - * @return ContainerInfo - * @throws IOException - */ - ContainerData readContainer(Pipeline pipeline) throws IOException; - - - /** - * Gets the container size -- Computed by SCM from Container Reports. - * @param pipeline - Pipeline - * @return number of bytes used by this container. - * @throws IOException - */ - long getContainerSize(Pipeline pipeline) throws IOException; - - /** - * Creates a Container on SCM and returns the pipeline. - * @param type - Replication Type. - * @param replicationFactor - Replication Factor - * @param containerId - Container ID - * @return Pipeline - * @throws IOException - in case of error. - */ - Pipeline createContainer(HdslProtos.ReplicationType type, - HdslProtos.ReplicationFactor replicationFactor, String containerId, - String owner) throws IOException; - - /** - * Returns a set of Nodes that meet a query criteria. - * @param nodeStatuses - A set of criteria that we want the node to have. - * @param queryScope - Query scope - Cluster or pool. - * @param poolName - if it is pool, a pool name is required. - * @return A set of nodes that meet the requested criteria. - * @throws IOException - */ - HdslProtos.NodePool queryNode(EnumSet<HdslProtos.NodeState> nodeStatuses, - HdslProtos.QueryScope queryScope, String poolName) throws IOException; - - /** - * Creates a specified replication pipeline. - * @param type - Type - * @param factor - Replication factor - * @param nodePool - Set of machines. - * @throws IOException - */ - Pipeline createReplicationPipeline(HdslProtos.ReplicationType type, - HdslProtos.ReplicationFactor factor, HdslProtos.NodePool nodePool) - throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/package-info.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/package-info.java deleted file mode 100644 index e85e542..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/client/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.client; - -/** - * This package contains classes for the client of the storage container - * protocol. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/ContainerID.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/ContainerID.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/ContainerID.java deleted file mode 100644 index a51d3b7..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/ContainerID.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package org.apache.hadoop.ozone.scm.container.ContainerStates; - -import com.google.common.base.Preconditions; -import org.apache.commons.math3.util.MathUtils; - -/** - * Container ID is an integer that is a value between 1..MAX_CONTAINER ID. - * <p> - * We are creating a specific type for this to avoid mixing this with - * normal integers in code. - */ -public class ContainerID implements Comparable { - - private final long id; - - /** - * Constructs ContainerID. - * - * @param id int - */ - public ContainerID(long id) { - Preconditions.checkState(id > 0, - "Container ID should be a positive int"); - this.id = id; - } - - /** - * Factory method for creation of ContainerID. - * @param containerID long - * @return ContainerID. - */ - public static ContainerID valueof(long containerID) { - Preconditions.checkState(containerID > 0); - return new ContainerID(containerID); - } - - /** - * Returns int representation of ID. - * - * @return int - */ - public long getId() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - ContainerID that = (ContainerID) o; - - return id == that.id; - } - - @Override - public int hashCode() { - return MathUtils.hash(id); - } - - @Override - public int compareTo(Object o) { - Preconditions.checkNotNull(o); - if (o instanceof ContainerID) { - return Long.compare(((ContainerID) o).getId(), this.getId()); - } - throw new IllegalArgumentException("Object O, should be an instance " + - "of ContainerID"); - } - - @Override - public String toString() { - return "id=" + id; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/package-info.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/package-info.java deleted file mode 100644 index 61f5609..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/ContainerStates/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -/** - * Container States. - */ -package org.apache.hadoop.ozone.scm.container.ContainerStates; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java deleted file mode 100644 index f51336f..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - -/** - * Allocated block wraps the result returned from SCM#allocateBlock which - * contains a Pipeline and the key. - */ -public final class AllocatedBlock { - private Pipeline pipeline; - private String key; - // Indicates whether the client should create container before writing block. - private boolean shouldCreateContainer; - - /** - * Builder for AllocatedBlock. - */ - public static class Builder { - private Pipeline pipeline; - private String key; - private boolean shouldCreateContainer; - - public Builder setPipeline(Pipeline p) { - this.pipeline = p; - return this; - } - - public Builder setKey(String k) { - this.key = k; - return this; - } - - public Builder setShouldCreateContainer(boolean shouldCreate) { - this.shouldCreateContainer = shouldCreate; - return this; - } - - public AllocatedBlock build() { - return new AllocatedBlock(pipeline, key, shouldCreateContainer); - } - } - - private AllocatedBlock(Pipeline pipeline, String key, - boolean shouldCreateContainer) { - this.pipeline = pipeline; - this.key = key; - this.shouldCreateContainer = shouldCreateContainer; - } - - public Pipeline getPipeline() { - return pipeline; - } - - public String getKey() { - return key; - } - - public boolean getCreateContainer() { - return shouldCreateContainer; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java deleted file mode 100644 index 8e6fdbb..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - -import com.google.common.base.Preconditions; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos; -import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID; -import org.apache.hadoop.util.Time; - -import java.util.Comparator; - -/** - * Class wraps ozone container info. - */ -public class ContainerInfo - implements Comparator<ContainerInfo>, Comparable<ContainerInfo> { - private HdslProtos.LifeCycleState state; - private Pipeline pipeline; - // Bytes allocated by SCM for clients. - private long allocatedBytes; - // Actual container usage, updated through heartbeat. - private long usedBytes; - private long numberOfKeys; - private long lastUsed; - // The wall-clock ms since the epoch at which the current state enters. - private long stateEnterTime; - private String owner; - private String containerName; - private long containerID; - ContainerInfo( - long containerID, - final String containerName, - HdslProtos.LifeCycleState state, - Pipeline pipeline, - long allocatedBytes, - long usedBytes, - long numberOfKeys, - long stateEnterTime, - String owner) { - this.containerID = containerID; - this.containerName = containerName; - this.pipeline = pipeline; - this.allocatedBytes = allocatedBytes; - this.usedBytes = usedBytes; - this.numberOfKeys = numberOfKeys; - this.lastUsed = Time.monotonicNow(); - this.state = state; - this.stateEnterTime = stateEnterTime; - this.owner = owner; - } - - /** - * Needed for serialization findbugs. - */ - public ContainerInfo() { - } - - public static ContainerInfo fromProtobuf(HdslProtos.SCMContainerInfo info) { - ContainerInfo.Builder builder = new ContainerInfo.Builder(); - builder.setPipeline(Pipeline.getFromProtoBuf(info.getPipeline())); - builder.setAllocatedBytes(info.getAllocatedBytes()); - builder.setUsedBytes(info.getUsedBytes()); - builder.setNumberOfKeys(info.getNumberOfKeys()); - builder.setState(info.getState()); - builder.setStateEnterTime(info.getStateEnterTime()); - builder.setOwner(info.getOwner()); - builder.setContainerName(info.getContainerName()); - builder.setContainerID(info.getContainerID()); - return builder.build(); - } - - public long getContainerID() { - return containerID; - } - - public String getContainerName() { - return containerName; - } - - public HdslProtos.LifeCycleState getState() { - return state; - } - - public void setState(HdslProtos.LifeCycleState state) { - this.state = state; - } - - public long getStateEnterTime() { - return stateEnterTime; - } - - public Pipeline getPipeline() { - return pipeline; - } - - public long getAllocatedBytes() { - return allocatedBytes; - } - - /** - * Set Allocated bytes. - * - * @param size - newly allocated bytes -- negative size is case of deletes - * can be used. - */ - public void updateAllocatedBytes(long size) { - this.allocatedBytes += size; - } - - public long getUsedBytes() { - return usedBytes; - } - - public long getNumberOfKeys() { - return numberOfKeys; - } - - public ContainerID containerID() { - return new ContainerID(getContainerID()); - } - - /** - * Gets the last used time from SCM's perspective. - * - * @return time in milliseconds. - */ - public long getLastUsed() { - return lastUsed; - } - - public void updateLastUsedTime() { - lastUsed = Time.monotonicNow(); - } - - public void allocate(long size) { - // should we also have total container size in ContainerInfo - // and check before allocating? - allocatedBytes += size; - } - - public HdslProtos.SCMContainerInfo getProtobuf() { - HdslProtos.SCMContainerInfo.Builder builder = - HdslProtos.SCMContainerInfo.newBuilder(); - builder.setPipeline(getPipeline().getProtobufMessage()); - builder.setAllocatedBytes(getAllocatedBytes()); - builder.setUsedBytes(getUsedBytes()); - builder.setNumberOfKeys(getNumberOfKeys()); - builder.setState(state); - builder.setStateEnterTime(stateEnterTime); - builder.setContainerID(getContainerID()); - - if (getOwner() != null) { - builder.setOwner(getOwner()); - } - builder.setContainerName(getContainerName()); - return builder.build(); - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - @Override - public String toString() { - return "ContainerInfo{" - + "state=" + state - + ", pipeline=" + pipeline - + ", stateEnterTime=" + stateEnterTime - + ", owner=" + owner - + ", containerName='" + containerName - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - ContainerInfo that = (ContainerInfo) o; - - return new EqualsBuilder() - .append(pipeline.getContainerName(), that.pipeline.getContainerName()) - - // TODO : Fix this later. If we add these factors some tests fail. - // So Commenting this to continue and will enforce this with - // Changes in pipeline where we remove Container Name to - // SCMContainerinfo from Pipeline. - // .append(pipeline.getFactor(), that.pipeline.getFactor()) - // .append(pipeline.getType(), that.pipeline.getType()) - .append(owner, that.owner) - .isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(11, 811) - .append(pipeline.getContainerName()) - .append(pipeline.getFactor()) - .append(pipeline.getType()) - .append(owner) - .toHashCode(); - } - - /** - * Compares its two arguments for order. Returns a negative integer, zero, or - * a positive integer as the first argument is less than, equal to, or greater - * than the second.<p> - * - * @param o1 the first object to be compared. - * @param o2 the second object to be compared. - * @return a negative integer, zero, or a positive integer as the first - * argument is less than, equal to, or greater than the second. - * @throws NullPointerException if an argument is null and this comparator - * does not permit null arguments - * @throws ClassCastException if the arguments' types prevent them from - * being compared by this comparator. - */ - @Override - public int compare(ContainerInfo o1, ContainerInfo o2) { - return Long.compare(o1.getLastUsed(), o2.getLastUsed()); - } - - /** - * Compares this object with the specified object for order. Returns a - * negative integer, zero, or a positive integer as this object is less than, - * equal to, or greater than the specified object. - * - * @param o the object to be compared. - * @return a negative integer, zero, or a positive integer as this object is - * less than, equal to, or greater than the specified object. - * @throws NullPointerException if the specified object is null - * @throws ClassCastException if the specified object's type prevents it - * from being compared to this object. - */ - @Override - public int compareTo(ContainerInfo o) { - return this.compare(this, o); - } - - /** - * Builder class for ContainerInfo. - */ - public static class Builder { - private HdslProtos.LifeCycleState state; - private Pipeline pipeline; - private long allocated; - private long used; - private long keys; - private long stateEnterTime; - private String owner; - private String containerName; - private long containerID; - - public Builder setContainerID(long id) { - Preconditions.checkState(id >= 0); - this.containerID = id; - return this; - } - - public Builder setState(HdslProtos.LifeCycleState lifeCycleState) { - this.state = lifeCycleState; - return this; - } - - public Builder setPipeline(Pipeline containerPipeline) { - this.pipeline = containerPipeline; - return this; - } - - public Builder setAllocatedBytes(long bytesAllocated) { - this.allocated = bytesAllocated; - return this; - } - - public Builder setUsedBytes(long bytesUsed) { - this.used = bytesUsed; - return this; - } - - public Builder setNumberOfKeys(long keyCount) { - this.keys = keyCount; - return this; - } - - public Builder setStateEnterTime(long time) { - this.stateEnterTime = time; - return this; - } - - public Builder setOwner(String containerOwner) { - this.owner = containerOwner; - return this; - } - - public Builder setContainerName(String container) { - this.containerName = container; - return this; - } - - public ContainerInfo build() { - return new - ContainerInfo(containerID, containerName, state, pipeline, - allocated, used, keys, stateEnterTime, owner); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java deleted file mode 100644 index dceaa6c..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - - -import static org.apache.hadoop.hdsl.protocol.proto - .ScmBlockLocationProtocolProtos.DeleteScmBlockResult; - -/** - * Class wraps storage container manager block deletion results. - */ -public class DeleteBlockResult { - private String key; - private DeleteScmBlockResult.Result result; - - public DeleteBlockResult(final String key, - final DeleteScmBlockResult.Result result) { - this.key = key; - this.result = result; - } - - /** - * Get key deleted. - * @return key name. - */ - public String getKey() { - return key; - } - - /** - * Get key deletion result. - * @return key deletion result. - */ - public DeleteScmBlockResult.Result getResult() { - return result; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java deleted file mode 100644 index 9f2d1f4..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.container.common.helpers; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonFilter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.ser.FilterProvider; -import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; -import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdsl.protocol.DatanodeDetails; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * A pipeline represents the group of machines over which a container lives. - */ -public class Pipeline { - static final String PIPELINE_INFO = "PIPELINE_INFO_FILTER"; - private static final ObjectWriter WRITER; - - static { - ObjectMapper mapper = new ObjectMapper(); - String[] ignorableFieldNames = {"data"}; - FilterProvider filters = new SimpleFilterProvider() - .addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter - .serializeAllExcept(ignorableFieldNames)); - mapper.setVisibility(PropertyAccessor.FIELD, - JsonAutoDetect.Visibility.ANY); - mapper.addMixIn(Object.class, MixIn.class); - - WRITER = mapper.writer(filters); - } - - private String containerName; - private PipelineChannel pipelineChannel; - /** - * Allows you to maintain private data on pipelines. This is not serialized - * via protobuf, just allows us to maintain some private data. - */ - @JsonIgnore - private byte[] data; - /** - * Constructs a new pipeline data structure. - * - * @param containerName - Container - * @param pipelineChannel - transport information for this container - */ - public Pipeline(String containerName, PipelineChannel pipelineChannel) { - this.containerName = containerName; - this.pipelineChannel = pipelineChannel; - data = null; - } - - /** - * Gets pipeline object from protobuf. - * - * @param pipeline - ProtoBuf definition for the pipeline. - * @return Pipeline Object - */ - public static Pipeline getFromProtoBuf(HdslProtos.Pipeline pipeline) { - Preconditions.checkNotNull(pipeline); - PipelineChannel pipelineChannel = - PipelineChannel.getFromProtoBuf(pipeline.getPipelineChannel()); - return new Pipeline(pipeline.getContainerName(), pipelineChannel); - } - - public HdslProtos.ReplicationFactor getFactor() { - return pipelineChannel.getFactor(); - } - - /** - * Returns the first machine in the set of datanodes. - * - * @return First Machine. - */ - @JsonIgnore - public DatanodeDetails getLeader() { - return pipelineChannel.getDatanodes().get(pipelineChannel.getLeaderID()); - } - - /** - * Returns the leader host. - * - * @return First Machine. - */ - public String getLeaderHost() { - return pipelineChannel.getDatanodes() - .get(pipelineChannel.getLeaderID()).getHostName(); - } - - /** - * Returns all machines that make up this pipeline. - * - * @return List of Machines. - */ - @JsonIgnore - public List<DatanodeDetails> getMachines() { - return new ArrayList<>(pipelineChannel.getDatanodes().values()); - } - - /** - * Returns all machines that make up this pipeline. - * - * @return List of Machines. - */ - public List<String> getDatanodeHosts() { - List<String> dataHosts = new ArrayList<>(); - for (DatanodeDetails id : pipelineChannel.getDatanodes().values()) { - dataHosts.add(id.getHostName()); - } - return dataHosts; - } - - /** - * Return a Protobuf Pipeline message from pipeline. - * - * @return Protobuf message - */ - @JsonIgnore - public HdslProtos.Pipeline getProtobufMessage() { - HdslProtos.Pipeline.Builder builder = - HdslProtos.Pipeline.newBuilder(); - builder.setContainerName(this.containerName); - builder.setPipelineChannel(this.pipelineChannel.getProtobufMessage()); - return builder.build(); - } - - /** - * Returns containerName if available. - * - * @return String. - */ - public String getContainerName() { - return containerName; - } - - /** - * Returns private data that is set on this pipeline. - * - * @return blob, the user can interpret it any way they like. - */ - public byte[] getData() { - if (this.data != null) { - return Arrays.copyOf(this.data, this.data.length); - } else { - return null; - } - } - - @VisibleForTesting - public PipelineChannel getPipelineChannel() { - return pipelineChannel; - } - - /** - * Set private data on pipeline. - * - * @param data -- private data. - */ - public void setData(byte[] data) { - if (data != null) { - this.data = Arrays.copyOf(data, data.length); - } - } - - /** - * Gets the State of the pipeline. - * - * @return - LifeCycleStates. - */ - public HdslProtos.LifeCycleState getLifeCycleState() { - return pipelineChannel.getLifeCycleState(); - } - - /** - * Gets the pipeline Name. - * - * @return - Name of the pipeline - */ - public String getPipelineName() { - return pipelineChannel.getName(); - } - - /** - * Returns the type. - * - * @return type - Standalone, Ratis, Chained. - */ - public HdslProtos.ReplicationType getType() { - return pipelineChannel.getType(); - } - - @Override - public String toString() { - final StringBuilder b = new StringBuilder(getClass().getSimpleName()) - .append("["); - pipelineChannel.getDatanodes().keySet().stream() - .forEach(id -> b. - append(id.endsWith(pipelineChannel.getLeaderID()) ? "*" + id : id)); - b.append("] container:").append(containerName); - b.append(" name:").append(getPipelineName()); - if (getType() != null) { - b.append(" type:").append(getType().toString()); - } - if (getFactor() != null) { - b.append(" factor:").append(getFactor().toString()); - } - if (getLifeCycleState() != null) { - b.append(" State:").append(getLifeCycleState().toString()); - } - return b.toString(); - } - - /** - * Returns a JSON string of this object. - * - * @return String - json string - * @throws IOException - */ - public String toJsonString() throws IOException { - return WRITER.writeValueAsString(this); - } - - @JsonFilter(PIPELINE_INFO) - class MixIn { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java deleted file mode 100644 index 1937968..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.container.common.helpers; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdsl.protocol.DatanodeDetails; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.LifeCycleState; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationType; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationFactor; - -import java.util.Map; -import java.util.TreeMap; - -/** - * PipelineChannel information for a {@link Pipeline}. - */ -public class PipelineChannel { - @JsonIgnore - private String leaderID; - @JsonIgnore - private Map<String, DatanodeDetails> datanodes; - private LifeCycleState lifeCycleState; - private ReplicationType type; - private ReplicationFactor factor; - private String name; - - public PipelineChannel(String leaderID, LifeCycleState lifeCycleState, - ReplicationType replicationType, ReplicationFactor replicationFactor, - String name) { - this.leaderID = leaderID; - this.lifeCycleState = lifeCycleState; - this.type = replicationType; - this.factor = replicationFactor; - this.name = name; - datanodes = new TreeMap<>(); - } - - public String getLeaderID() { - return leaderID; - } - - public Map<String, DatanodeDetails> getDatanodes() { - return datanodes; - } - - public LifeCycleState getLifeCycleState() { - return lifeCycleState; - } - - public ReplicationType getType() { - return type; - } - - public ReplicationFactor getFactor() { - return factor; - } - - public String getName() { - return name; - } - - public void addMember(DatanodeDetails datanodeDetails) { - datanodes.put(datanodeDetails.getUuid().toString(), - datanodeDetails); - } - - @JsonIgnore - public HdslProtos.PipelineChannel getProtobufMessage() { - HdslProtos.PipelineChannel.Builder builder = - HdslProtos.PipelineChannel.newBuilder(); - for (DatanodeDetails datanode : datanodes.values()) { - builder.addMembers(datanode.getProtoBufMessage()); - } - builder.setLeaderID(leaderID); - - if (this.getLifeCycleState() != null) { - builder.setState(this.getLifeCycleState()); - } - if (this.getType() != null) { - builder.setType(this.getType()); - } - - if (this.getFactor() != null) { - builder.setFactor(this.getFactor()); - } - return builder.build(); - } - - public static PipelineChannel getFromProtoBuf( - HdslProtos.PipelineChannel transportProtos) { - Preconditions.checkNotNull(transportProtos); - PipelineChannel pipelineChannel = - new PipelineChannel(transportProtos.getLeaderID(), - transportProtos.getState(), - transportProtos.getType(), - transportProtos.getFactor(), - transportProtos.getName()); - - for (HdslProtos.DatanodeDetailsProto dataID : - transportProtos.getMembersList()) { - pipelineChannel.addMember(DatanodeDetails.getFromProtoBuf(dataID)); - } - return pipelineChannel; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java deleted file mode 100644 index e285aec..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.container.common.helpers; - -import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos; - -import java.io.IOException; - -/** - * Exceptions thrown from the Storage Container. - */ -public class StorageContainerException extends IOException { - private ContainerProtos.Result result; - - /** - * Constructs an {@code IOException} with {@code null} - * as its error detail message. - */ - public StorageContainerException(ContainerProtos.Result result) { - this.result = result; - } - - /** - * Constructs an {@code IOException} with the specified detail message. - * - * @param message The detail message (which is saved for later retrieval by - * the {@link #getMessage()} method) - * @param result - The result code - */ - public StorageContainerException(String message, - ContainerProtos.Result result) { - super(message); - this.result = result; - } - - /** - * Constructs an {@code IOException} with the specified detail message - * and cause. - * <p> - * <p> Note that the detail message associated with {@code cause} is - * <i>not</i> automatically incorporated into this exception's detail - * message. - * - * @param message The detail message (which is saved for later retrieval by - * the {@link #getMessage()} method) - * - * @param cause The cause (which is saved for later retrieval by the {@link - * #getCause()} method). (A null value is permitted, and indicates that the - * cause is nonexistent or unknown.) - * - * @param result - The result code - * @since 1.6 - */ - public StorageContainerException(String message, Throwable cause, - ContainerProtos.Result result) { - super(message, cause); - this.result = result; - } - - /** - * Constructs an {@code IOException} with the specified cause and a - * detail message of {@code (cause==null ? null : cause.toString())} - * (which typically contains the class and detail message of {@code cause}). - * This constructor is useful for IO exceptions that are little more - * than wrappers for other throwables. - * - * @param cause The cause (which is saved for later retrieval by the {@link - * #getCause()} method). (A null value is permitted, and indicates that the - * cause is nonexistent or unknown.) - * @param result - The result code - * @since 1.6 - */ - public StorageContainerException(Throwable cause, ContainerProtos.Result - result) { - super(cause); - this.result = result; - } - - /** - * Returns Result. - * - * @return Result. - */ - public ContainerProtos.Result getResult() { - return result; - } - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java deleted file mode 100644 index 3fa9663..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.container.common.helpers; -/** - Contains protocol buffer helper classes and utilites used in - impl. - **/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/package-info.java deleted file mode 100644 index ad24f98..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm; - -/** - * This package contains classes for the client of the storage container - * protocol. - */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java deleted file mode 100644 index 469dab5..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.protocol; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; - -import java.util.Set; - -/** - * Holds the nodes that currently host the container for an object key hash. - */ -@InterfaceAudience.Private -public final class LocatedContainer { - private final String key; - private final String matchedKeyPrefix; - private final String containerName; - private final Set<DatanodeInfo> locations; - private final DatanodeInfo leader; - - /** - * Creates a LocatedContainer. - * - * @param key object key - * @param matchedKeyPrefix prefix of key that was used to find the location - * @param containerName container name - * @param locations nodes that currently host the container - * @param leader node that currently acts as pipeline leader - */ - public LocatedContainer(String key, String matchedKeyPrefix, - String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) { - this.key = key; - this.matchedKeyPrefix = matchedKeyPrefix; - this.containerName = containerName; - this.locations = locations; - this.leader = leader; - } - - /** - * Returns the container name. - * - * @return container name - */ - public String getContainerName() { - return this.containerName; - } - - /** - * Returns the object key. - * - * @return object key - */ - public String getKey() { - return this.key; - } - - /** - * Returns the node that currently acts as pipeline leader. - * - * @return node that currently acts as pipeline leader - */ - public DatanodeInfo getLeader() { - return this.leader; - } - - /** - * Returns the nodes that currently host the container. - * - * @return Set<DatanodeInfo> nodes that currently host the container - */ - public Set<DatanodeInfo> getLocations() { - return this.locations; - } - - /** - * Returns the prefix of the key that was used to find the location. - * - * @return prefix of the key that was used to find the location - */ - public String getMatchedKeyPrefix() { - return this.matchedKeyPrefix; - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj == null) { - return false; - } - if (!(otherObj instanceof LocatedContainer)) { - return false; - } - LocatedContainer other = (LocatedContainer)otherObj; - return this.key == null ? other.key == null : this.key.equals(other.key); - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName() - + "{key=" + key - + "; matchedKeyPrefix=" + matchedKeyPrefix - + "; containerName=" + containerName - + "; locations=" + locations - + "; leader=" + leader - + "}"; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java deleted file mode 100644 index 0bb84ac..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm.protocol; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -import org.apache.hadoop.ozone.common.DeleteBlockGroupResult; -import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationType; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationFactor; -import org.apache.hadoop.scm.ScmInfo; - -/** - * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes - * to read/write a block. - */ -public interface ScmBlockLocationProtocol { - - /** - * Find the set of nodes to read/write a block, as - * identified by the block key. This method supports batch lookup by - * passing multiple keys. - * - * @param keys batch of block keys to find - * @return allocated blocks for each block key - * @throws IOException if there is any failure - */ - Set<AllocatedBlock> getBlockLocations(Set<String> keys) throws IOException; - - /** - * Asks SCM where a block should be allocated. SCM responds with the - * set of datanodes that should be used creating this block. - * @param size - size of the block. - * @return allocated block accessing info (key, pipeline). - * @throws IOException - */ - AllocatedBlock allocateBlock(long size, ReplicationType type, - ReplicationFactor factor, String owner) throws IOException; - - /** - * Delete blocks for a set of object keys. - * - * @param keyBlocksInfoList Map of object key and its blocks. - * @return list of block deletion results. - * @throws IOException if there is any failure. - */ - List<DeleteBlockGroupResult> - deleteKeyBlocks(List<BlockGroup> keyBlocksInfoList) throws IOException; - - /** - * Gets the Clusterid and SCM Id from SCM. - */ - ScmInfo getScmInfo() throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java deleted file mode 100644 index 4e4b3d6..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.scm.protocol; - -import java.util.List; -import java.util.stream.Collectors; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; - -/** - * Holds the nodes that currently host the block for a block key. - */ -@InterfaceAudience.Private -public final class ScmLocatedBlock { - private final String key; - private final List<DatanodeInfo> locations; - private final DatanodeInfo leader; - - /** - * Creates a ScmLocatedBlock. - * - * @param key object key - * @param locations nodes that currently host the block - * @param leader node that currently acts as pipeline leader - */ - public ScmLocatedBlock(final String key, final List<DatanodeInfo> locations, - final DatanodeInfo leader) { - this.key = key; - this.locations = locations; - this.leader = leader; - } - - /** - * Returns the object key. - * - * @return object key - */ - public String getKey() { - return this.key; - } - - /** - * Returns the node that currently acts as pipeline leader. - * - * @return node that currently acts as pipeline leader - */ - public DatanodeInfo getLeader() { - return this.leader; - } - - /** - * Returns the nodes that currently host the block. - * - * @return List<DatanodeInfo> nodes that currently host the block - */ - public List<DatanodeInfo> getLocations() { - return this.locations; - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj == null) { - return false; - } - if (!(otherObj instanceof ScmLocatedBlock)) { - return false; - } - ScmLocatedBlock other = (ScmLocatedBlock)otherObj; - return this.key == null ? other.key == null : this.key.equals(other.key); - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public String toString() { - return getClass().getSimpleName() + "{key=" + key + "; locations=" - + locations.stream().map(loc -> loc.toString()).collect(Collectors - .joining(",")) + "; leader=" + leader + "}"; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/651a05a1/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java deleted file mode 100644 index 01fc075..0000000 --- a/hadoop-hdsl/common/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.scm.protocol; - -import java.io.IOException; -import java.util.EnumSet; -import java.util.List; - -import org.apache.hadoop.hdsl.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto; -import org.apache.hadoop.scm.ScmInfo; -import org.apache.hadoop.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdsl.protocol.proto.HdslProtos; - -/** - * ContainerLocationProtocol is used by an HDFS node to find the set of nodes - * that currently host a container. - */ -public interface StorageContainerLocationProtocol { - /** - * Asks SCM where a container should be allocated. SCM responds with the - * set of datanodes that should be used creating this container. - * - */ - Pipeline allocateContainer(HdslProtos.ReplicationType replicationType, - HdslProtos.ReplicationFactor factor, String containerName, String owner) - throws IOException; - - /** - * Ask SCM the location of the container. SCM responds with a group of - * nodes where this container and its replicas are located. - * - * @param containerName - Name of the container. - * @return Pipeline - the pipeline where container locates. - * @throws IOException - */ - Pipeline getContainer(String containerName) throws IOException; - - /** - * Ask SCM a list of containers with a range of container names - * and the limit of count. - * Search container names between start name(exclusive), and - * use prefix name to filter the result. the max size of the - * searching range cannot exceed the value of count. - * - * @param startName start name, if null, start searching at the head. - * @param prefixName prefix name, if null, then filter is disabled. - * @param count count, if count < 0, the max size is unlimited.( - * Usually the count will be replace with a very big - * value instead of being unlimited in case the db is very big) - * - * @return a list of container. - * @throws IOException - */ - List<ContainerInfo> listContainer(String startName, String prefixName, - int count) throws IOException; - - /** - * Deletes a container in SCM. - * - * @param containerName - * @throws IOException - * if failed to delete the container mapping from db store - * or container doesn't exist. - */ - void deleteContainer(String containerName) throws IOException; - - /** - * Queries a list of Node Statuses. - * @param nodeStatuses - * @return List of Datanodes. - */ - HdslProtos.NodePool queryNode(EnumSet<HdslProtos.NodeState> nodeStatuses, - HdslProtos.QueryScope queryScope, String poolName) throws IOException; - - /** - * Notify from client when begin or finish creating objects like pipeline - * or containers on datanodes. - * Container will be in Operational state after that. - * @param type object type - * @param name object name - * @param op operation type (e.g., create, close, delete) - * @param stage creation stage - */ - void notifyObjectStageChange( - ObjectStageChangeRequestProto.Type type, String name, - ObjectStageChangeRequestProto.Op op, - ObjectStageChangeRequestProto.Stage stage) throws IOException; - - /** - * Creates a replication pipeline of a specified type. - * @param type - replication type - * @param factor - factor 1 or 3 - * @param nodePool - optional machine list to build a pipeline. - * @throws IOException - */ - Pipeline createReplicationPipeline(HdslProtos.ReplicationType type, - HdslProtos.ReplicationFactor factor, HdslProtos.NodePool nodePool) - throws IOException; - - /** - * Returns information about SCM. - * - * @return {@link ScmInfo} - * @throws IOException - */ - ScmInfo getScmInfo() throws IOException; -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
