http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java index 1496da2,0000000..cfacd5f mode 100644,000000..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerMap.java @@@ -1,124 -1,0 +1,130 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.storagecontainer; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; +import org.apache.hadoop.util.GSet; + ++import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * Maps a storage container to its location on datanodes. Similar to + * {@link org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap} + */ +public class StorageContainerMap implements GSet<Block, BlockInfo> { + + private Map<Long, BitWiseTrieContainerMap> containerPrefixMap + = new HashMap<Long, BitWiseTrieContainerMap>(); + private int size; + public static final int PREFIX_LENGTH = 28; + + @Override + public int size() { + // TODO: update size when new containers created + return size; + } + + @Override + public boolean contains(Block key) { + return getBlockInfoContiguous(key.getBlockId()) != null; + } + + @Override + public BlockInfoContiguous get(Block key) { + return getBlockInfoContiguous(key.getBlockId()); + } + + @Override + public BlockInfoContiguous put(BlockInfo element) { + BlockInfoContiguous info = getBlockInfoContiguous(element.getBlockId()); + if (info == null) { + throw new IllegalStateException( + "The containers are created by splitting"); + } + // TODO: replace + return info; + } + + @Override + public BlockInfoContiguous remove(Block key) { + // It doesn't remove + return getBlockInfoContiguous(key.getBlockId()); + } + + @Override + public void clear() { + containerPrefixMap.clear(); + } + + @Override ++ public Collection<BlockInfo> values() { ++ return null; ++ } ++ ++ @Override + public Iterator<BlockInfo> iterator() { + // TODO : Support iteration + throw new UnsupportedOperationException(""); + } + + /** + * Initialize a new trie for a new bucket. + */ + public synchronized void initPrefix(long prefix) { + Preconditions.checkArgument((prefix >>> PREFIX_LENGTH) == 0, + "Prefix shouldn't be longer than "+PREFIX_LENGTH+" bits"); + if (getTrieMap(prefix << (64 - PREFIX_LENGTH)) != null) { + // Already initialized + return; + } + BitWiseTrieContainerMap newTrie = new BitWiseTrieContainerMap(prefix, + PREFIX_LENGTH); + containerPrefixMap.put(prefix, newTrie); + } + + @VisibleForTesting + synchronized BitWiseTrieContainerMap getTrieMap(long containerId) { + long prefix = containerId >>> (64 - PREFIX_LENGTH); + return containerPrefixMap.get(prefix); + } + + @VisibleForTesting + BlockInfoContiguous getBlockInfoContiguous(long containerId) { + BitWiseTrieContainerMap map = getTrieMap(containerId); + if (map == null) { + return null; + } + return map.get(containerId); + } + + public void splitContainer(long key) { + BitWiseTrieContainerMap map = getTrieMap(key); + if (map == null) { + throw new IllegalArgumentException("No container exists"); + } + map.addBit(key); + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java index 7d1eb66,0000000..60e9c20 mode 100644,000000..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java @@@ -1,162 -1,0 +1,170 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.storagecontainer; + +import org.apache.hadoop.hdfs.protocol.Block; ++import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.namenode.CacheManager; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.Namesystem; ++import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; +import org.apache.hadoop.ipc.StandbyException; +import org.apache.hadoop.security.AccessControlException; + +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Namesystem implementation to be used by StorageContainerManager. + */ +public class StorageContainerNameService implements Namesystem { + + private ReentrantReadWriteLock coarseLock = new ReentrantReadWriteLock(); + private String blockPoolId; + private volatile boolean serviceRunning = true; + + public void shutdown() { + serviceRunning = false; + } + + @Override + public boolean isRunning() { + return serviceRunning; + } + + @Override + public void checkSuperuserPrivilege() throws AccessControlException { + // TBD + } + + @Override + public String getBlockPoolId() { + return blockPoolId; + } + + public void setBlockPoolId(String id) { + this.blockPoolId = id; + } + + @Override + public boolean isInStandbyState() { + // HA mode is not supported + return false; + } + + @Override + public boolean isGenStampInFuture(Block block) { + // HA mode is not supported + return false; + } + + @Override ++ public BlockCollection getBlockCollection(long id) { ++ return null; ++ } ++ ++ @Override + public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal) { + // TBD + } + + @Override + public void checkOperation(NameNode.OperationCategory read) + throws StandbyException { + // HA mode is not supported + } + + @Override + public boolean isInSnapshot(BlockInfo blockInfo) { + // Snapshots not supported + return false; + } + + @Override + public CacheManager getCacheManager() { + // Cache Management is not supported + return null; + } + + @Override ++ public HAContext getHAContext() { ++ return null; ++ } ++ ++ @Override + public void readLock() { + coarseLock.readLock().lock(); + } + + @Override + public void readUnlock() { + coarseLock.readLock().unlock(); + } + + @Override + public boolean hasReadLock() { + return coarseLock.getReadHoldCount() > 0 || hasWriteLock(); + } + + @Override + public void writeLock() { + coarseLock.writeLock().lock(); + } + + @Override + public void writeLockInterruptibly() throws InterruptedException { + coarseLock.writeLock().lockInterruptibly(); + } + + @Override + public void writeUnlock() { + coarseLock.writeLock().unlock(); + } + + @Override + public boolean hasWriteLock() { + return coarseLock.isWriteLockedByCurrentThread(); + } + + @Override + public void checkSafeMode() { + // TBD + } + + @Override + public boolean isInSafeMode() { + return false; + } + + @Override + public boolean isInStartupSafeMode() { + return false; + } + - @Override - public boolean isPopulatingReplQueues() { - return false; - } + + @Override + public void incrementSafeBlockCount(int replication) { + // Do nothing + } + + @Override + public void decrementSafeBlockCount(BlockInfo b) { + // Do nothing + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto index 0000000,3b60e51..f2cc36c mode 000000,100644..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto @@@ -1,0 -1,201 +1,209 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /** + * These .proto interfaces are private and stable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *stable* .proto interface. + */ + + // This file contains protocol buffers that are used throughout HDFS -- i.e. + // by the client, server, and data transfer protocols. + + + option java_package = "org.apache.hadoop.hdfs.protocol.proto"; + option java_outer_classname = "HdfsServerProtos"; + option java_generate_equals_and_hash = true; + package hadoop.hdfs; + + import "hdfs.proto"; + + /** + * A list of storage IDs. + */ + message StorageUuidsProto { + repeated string storageUuids = 1; + } + + /** + * Block access token information + */ + message BlockKeyProto { + required uint32 keyId = 1; // Key identifier + required uint64 expiryDate = 2; // Expiry time in milliseconds + optional bytes keyBytes = 3; // Key secret + } + + /** + * Current key and set of block keys at the namenode. + */ + message ExportedBlockKeysProto { + required bool isBlockTokenEnabled = 1; + required uint64 keyUpdateInterval = 2; + required uint64 tokenLifeTime = 3; + required BlockKeyProto currentKey = 4; + repeated BlockKeyProto allKeys = 5; + } + + /** + * Block and datanodes where is it located + */ + message BlockWithLocationsProto { + required BlockProto block = 1; // Block + repeated string datanodeUuids = 2; // Datanodes with replicas of the block + repeated string storageUuids = 3; // Storages with replicas of the block + repeated StorageTypeProto storageTypes = 4; + } + + /** + * List of block with locations + */ + message BlocksWithLocationsProto { + repeated BlockWithLocationsProto blocks = 1; + } + + /** + * Editlog information with available transactions + */ + message RemoteEditLogProto { + required uint64 startTxId = 1; // Starting available edit log transaction + required uint64 endTxId = 2; // Ending available edit log transaction + optional bool isInProgress = 3 [default = false]; + } + + /** + * Enumeration of editlogs available on a remote namenode + */ + message RemoteEditLogManifestProto { + repeated RemoteEditLogProto logs = 1; + } + + /** + * Namespace information that describes namespace on a namenode + */ + message NamespaceInfoProto { + required string buildVersion = 1; // Software revision version (e.g. an svn or git revision) + required uint32 unused = 2; // Retained for backward compatibility + required string blockPoolID = 3; // block pool used by the namespace + required StorageInfoProto storageInfo = 4;// Node information + required string softwareVersion = 5; // Software version number (e.g. 2.0.0) + optional uint64 capabilities = 6 [default = 0]; // feature flags + } + + /** + * State of a block replica at a datanode + */ + enum ReplicaStateProto { + FINALIZED = 0; // State of a replica when it is not modified + RBW = 1; // State of replica that is being written to + RWR = 2; // State of replica that is waiting to be recovered + RUR = 3; // State of replica that is under recovery + TEMPORARY = 4; // State of replica that is created for replication + } + + /** + * Block that needs to be recovered with at a given location + */ + message RecoveringBlockProto { + required uint64 newGenStamp = 1; // New genstamp post recovery + required LocatedBlockProto block = 2; // Block to be recovered + optional BlockProto truncateBlock = 3; // New block for recovery (truncate) + } + + /** + * Unique signature to identify checkpoint transactions. + */ + message CheckpointSignatureProto { + required string blockPoolId = 1; + required uint64 mostRecentCheckpointTxId = 2; + required uint64 curSegmentTxId = 3; + required StorageInfoProto storageInfo = 4; + } + + /** + * Command returned from primary to checkpointing namenode. + * This command has checkpoint signature that identifies + * checkpoint transaction and is needed for further + * communication related to checkpointing. + */ + message CheckpointCommandProto { + // Unique signature to identify checkpoint transation + required CheckpointSignatureProto signature = 1; + + // If true, return transfer image to primary upon the completion of checkpoint + required bool needToReturnImage = 2; + } + + /** + * Command sent from one namenode to another namenode. + */ + message NamenodeCommandProto { + enum Type { + NamenodeCommand = 0; // Base command + CheckPointCommand = 1; // Check point command + } + required uint32 action = 1; + required Type type = 2; + optional CheckpointCommandProto checkpointCmd = 3; + } + + /** + * void request + */ + message VersionRequestProto { + } + + /** + * Version response from namenode. + */ + message VersionResponseProto { + required NamespaceInfoProto info = 1; + } + + /** + * Common node information shared by all the nodes in the cluster + */ + message StorageInfoProto { + required uint32 layoutVersion = 1; // Layout version of the file system + required uint32 namespceID = 2; // File system namespace ID + required string clusterID = 3; // ID of the cluster + required uint64 cTime = 4; // File system creation time ++ ++ enum NodeTypeProto { ++ NAME_NODE = 1; ++ DATA_NODE = 2; ++ JOURNAL_NODE = 3; ++ STORAGE_CONTAINER_SERVICE = 4; ++ } ++ optional NodeTypeProto nodeType = 5; + } + + /** + * Information sent by a namenode to identify itself to the primary namenode. + */ + message NamenodeRegistrationProto { + required string rpcAddress = 1; // host:port of the namenode RPC address + required string httpAddress = 2; // host:port of the namenode http server + enum NamenodeRoleProto { + NAMENODE = 1; + BACKUP = 2; + CHECKPOINT = 3; + } + required StorageInfoProto storageInfo = 3; // Node information + optional NamenodeRoleProto role = 4 [default = NAMENODE]; // Namenode role + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java ----------------------------------------------------------------------