HDDS-357. Use DBStore and TableStore for OzoneManager non-background service. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff036e49 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff036e49 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff036e49 Branch: refs/heads/HDFS-12090 Commit: ff036e49ff967d5dacf4b2d9d5376e57578ef391 Parents: eed8415 Author: Anu Engineer <aengin...@apache.org> Authored: Sun Sep 2 11:47:32 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Sun Sep 2 11:47:32 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/ozone/OzoneConsts.java | 6 +- .../org/apache/hadoop/utils/RocksDBStore.java | 2 +- .../org/apache/hadoop/utils/db/DBStore.java | 22 + .../org/apache/hadoop/utils/db/RDBStore.java | 26 +- .../common/src/main/resources/ozone-default.xml | 2 +- .../apache/hadoop/hdds/server/ServerUtils.java | 5 + .../ozone/client/io/ChunkGroupOutputStream.java | 4 +- .../hadoop/ozone/om/helpers/OpenKeySession.java | 6 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 11 +- ...neManagerProtocolClientSideTranslatorPB.java | 8 +- .../src/main/proto/OzoneManagerProtocol.proto | 6 +- .../rpc/TestCloseContainerHandlingByClient.java | 37 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 4 + .../apache/hadoop/ozone/om/TestOmSQLCli.java | 7 +- .../hadoop/ozone/om/TestOzoneManager.java | 37 +- .../hadoop/ozone/web/client/TestVolume.java | 6 + .../hadoop/ozone/om/BucketManagerImpl.java | 57 ++- .../org/apache/hadoop/ozone/om/KeyManager.java | 6 +- .../apache/hadoop/ozone/om/KeyManagerImpl.java | 276 +++++----- .../hadoop/ozone/om/OMMetadataManager.java | 222 ++++---- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 509 +++++++++++-------- .../apache/hadoop/ozone/om/OzoneManager.java | 209 ++++---- .../hadoop/ozone/om/VolumeManagerImpl.java | 156 +++--- ...neManagerProtocolServerSideTranslatorPB.java | 7 +- .../hadoop/ozone/om/TestBucketManagerImpl.java | 208 ++++---- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 12 +- 26 files changed, 978 insertions(+), 873 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 15366fb..8ea4d7f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -92,7 +92,6 @@ public final class OzoneConsts { public static final String CONTAINER_DB_SUFFIX = "container.db"; public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX; public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX; - public static final String BLOCK_DB = "block.db"; public static final String OPEN_CONTAINERS_DB = "openContainers.db"; public static final String DELETED_BLOCK_DB = "deletedBlock.db"; public static final String OM_DB_NAME = "om.db"; @@ -113,8 +112,6 @@ public final class OzoneConsts { public static final String DELETING_KEY_PREFIX = "#deleting#"; public static final String DELETED_KEY_PREFIX = "#deleted#"; public static final String DELETE_TRANSACTION_KEY_PREFIX = "#delTX#"; - public static final String OPEN_KEY_PREFIX = "#open#"; - public static final String OPEN_KEY_ID_DELIMINATOR = "#"; /** * OM LevelDB prefixes. @@ -138,8 +135,7 @@ public final class OzoneConsts { * | #deleting#/volumeName/bucketName/keyName | KeyInfo | * ---------------------------------------------------------- */ - public static final String OM_VOLUME_PREFIX = "/#"; - public static final String OM_BUCKET_PREFIX = "/#"; + public static final String OM_KEY_PREFIX = "/"; public static final String OM_USER_PREFIX = "$"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java index b243e3d..379d9e9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/RocksDBStore.java @@ -94,7 +94,7 @@ public class RocksDBStore implements MetadataStore { } } - private IOException toIOException(String msg, RocksDBException e) { + public static IOException toIOException(String msg, RocksDBException e) { String statusCode = e.getStatus() == null ? "N/A" : e.getStatus().getCodeString(); String errMessage = e.getMessage() == null ? "Unknown error" : http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java index a817f4f..6947a83 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java @@ -20,6 +20,7 @@ package org.apache.hadoop.utils.db; import org.apache.hadoop.classification.InterfaceStability; +import org.rocksdb.WriteBatch; import java.io.IOException; import java.util.ArrayList; @@ -83,11 +84,32 @@ public interface DBStore extends AutoCloseable { throws IOException; /** + * Moves a key from the Source Table to the destination Table and updates the + * destination with the new key name and value. + * This is similar to deleting an entry in one table and adding an entry in + * another table, here it is done atomically. + * + * @param sourceKey - Key to move. + * @param destKey - Destination key name. + * @param value - new value to write to the destination table. + * @param source - Source Table. + * @param dest - Destination Table. + * @throws IOException on Failure + */ + void move(byte[] sourceKey, byte[] destKey, byte[] value, + Table source, Table dest) throws IOException; + + /** * Returns an estimated count of keys in this DB. * * @return long, estimate of keys in the DB. */ long getEstimatedKeyCount() throws IOException; + /** + * Writes a transaction into the DB using the default write Options. + * @param batch - Batch to write. + */ + void write(WriteBatch batch) throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java index 85508d5..5078b3e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java @@ -189,9 +189,16 @@ public class RDBStore implements DBStore { } } + @Override public void move(byte[] key, byte[] value, Table source, Table dest) throws IOException { + move(key, key, value, source, dest); + } + + @Override + public void move(byte[] sourceKey, byte[] destKey, byte[] value, Table source, + Table dest) throws IOException { RDBTable sourceTable; RDBTable destTable; if (source instanceof RDBTable) { @@ -210,13 +217,13 @@ public class RDBStore implements DBStore { + "RocksDBTable."); } try (WriteBatch batch = new WriteBatch()) { - batch.put(destTable.getHandle(), key, value); - batch.delete(sourceTable.getHandle(), key); + batch.put(destTable.getHandle(), destKey, value); + batch.delete(sourceTable.getHandle(), sourceKey); db.write(writeOptions, batch); } catch (RocksDBException rockdbException) { - LOG.error("Move of key failed. Key:{}", DFSUtil.bytes2String(key)); - throw toIOException("Unable to move key: " + DFSUtil.bytes2String(key), - rockdbException); + LOG.error("Move of key failed. Key:{}", DFSUtil.bytes2String(sourceKey)); + throw toIOException("Unable to move key: " + + DFSUtil.bytes2String(sourceKey), rockdbException); } } @@ -229,6 +236,15 @@ public class RDBStore implements DBStore { } } + @Override + public void write(WriteBatch batch) throws IOException { + try { + db.write(writeOptions, batch); + } catch (RocksDBException e) { + throw toIOException("Unable to write the batch.", e); + } + } + @VisibleForTesting protected ObjectName getStatMBeanName() { return statMBeanName; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index d3ec4a5..6f296c6 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1101,7 +1101,7 @@ <property> <name>hdds.db.profile</name> - <value>DBProfile.SSD</value> + <value>SSD</value> <tag>OZONE, OM, PERFORMANCE, REQUIRED</tag> <description>This property allows user to pick a configuration that tunes the RocksDB settings for the hardware it is running http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java index a0e78dc..c6d85d8 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java @@ -136,4 +136,9 @@ public final class ServerUtils { return dirPath; } + public static void setOzoneMetaDirPath(OzoneConfiguration conf, + String path) { + conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, path); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java index 00624d5..c632df6 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java @@ -74,7 +74,7 @@ public class ChunkGroupOutputStream extends OutputStream { private final StorageContainerLocationProtocolClientSideTranslatorPB scmClient; private final OmKeyArgs keyArgs; - private final int openID; + private final long openID; private final XceiverClientManager xceiverClientManager; private final int chunkSize; private final String requestID; @@ -115,7 +115,7 @@ public class ChunkGroupOutputStream extends OutputStream { } @VisibleForTesting - public int getOpenID() { + public long getOpenID() { return openID; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java index bc364e6..11ee622 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java @@ -23,14 +23,14 @@ package org.apache.hadoop.ozone.om.helpers; * that servers can recognize this client, and thus know how to close the key. */ public class OpenKeySession { - private final int id; + private final long id; private final OmKeyInfo keyInfo; // the version of the key when it is being opened in this session. // a block that has a create version equals to open version means it will // be committed only when this open session is closed. private long openVersion; - public OpenKeySession(int id, OmKeyInfo info, long version) { + public OpenKeySession(long id, OmKeyInfo info, long version) { this.id = id; this.keyInfo = info; this.openVersion = version; @@ -44,7 +44,7 @@ public class OpenKeySession { return keyInfo; } - public int getId() { + public long getId() { return id; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index b7a099d..edb260a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -148,7 +148,7 @@ public interface OzoneManagerProtocol { * @param clientID the client identification * @throws IOException */ - void commitKey(OmKeyArgs args, int clientID) throws IOException; + void commitKey(OmKeyArgs args, long clientID) throws IOException; /** * Allocate a new block, it is assumed that the client is having an open key @@ -159,7 +159,7 @@ public interface OzoneManagerProtocol { * @return an allocated block * @throws IOException */ - OmKeyLocationInfo allocateBlock(OmKeyArgs args, int clientID) + OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID) throws IOException; /** @@ -172,9 +172,10 @@ public interface OzoneManagerProtocol { OmKeyInfo lookupKey(OmKeyArgs args) throws IOException; /** - * Rename an existing key within a bucket + * Rename an existing key within a bucket. * @param args the args of the key. * @param toKeyName New name to be used for the Key + * @throws IOException */ void renameKey(OmKeyArgs args, String toKeyName) throws IOException; @@ -214,7 +215,7 @@ public interface OzoneManagerProtocol { * @throws IOException */ List<OmBucketInfo> listBuckets(String volumeName, - String startBucketName, String bucketPrefix, int maxNumOfBuckets) + String startBucketName, String bucketPrefix, int maxNumOfBuckets) throws IOException; /** @@ -239,7 +240,7 @@ public interface OzoneManagerProtocol { * @throws IOException */ List<OmKeyInfo> listKeys(String volumeName, - String bucketName, String startKeyName, String keyPrefix, int maxKeys) + String bucketName, String startKeyName, String keyPrefix, int maxKeys) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index e557ac5..c0829fa 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -488,7 +488,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB */ @Override public List<OmBucketInfo> listBuckets(String volumeName, - String startKey, String prefix, int count) throws IOException { + String startKey, String prefix, int count) throws IOException { List<OmBucketInfo> buckets = new ArrayList<>(); ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.newBuilder(); reqBuilder.setVolumeName(volumeName); @@ -554,7 +554,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB } @Override - public OmKeyLocationInfo allocateBlock(OmKeyArgs args, int clientID) + public OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID) throws IOException { AllocateBlockRequest.Builder req = AllocateBlockRequest.newBuilder(); KeyArgs keyArgs = KeyArgs.newBuilder() @@ -579,7 +579,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB } @Override - public void commitKey(OmKeyArgs args, int clientID) + public void commitKey(OmKeyArgs args, long clientID) throws IOException { CommitKeyRequest.Builder req = CommitKeyRequest.newBuilder(); List<OmKeyLocationInfo> locationInfoList = args.getLocationInfoList(); @@ -708,7 +708,7 @@ public final class OzoneManagerProtocolClientSideTranslatorPB */ @Override public List<OmKeyInfo> listKeys(String volumeName, String bucketName, - String startKey, String prefix, int maxKeys) throws IOException { + String startKey, String prefix, int maxKeys) throws IOException { List<OmKeyInfo> keys = new ArrayList<>(); ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder(); reqBuilder.setVolumeName(volumeName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index 51a0a7f..242e3b5 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -273,7 +273,7 @@ message LocateKeyResponse { optional KeyInfo keyInfo = 2; // clients' followup request may carry this ID for stateful operations (similar // to a cookie). - optional uint32 ID = 3; + optional uint64 ID = 3; // TODO : allow specifiying a particular version to read. optional uint64 openVersion = 4; } @@ -319,7 +319,7 @@ message ListKeysResponse { message AllocateBlockRequest { required KeyArgs keyArgs = 1; - required uint32 clientID = 2; + required uint64 clientID = 2; } message AllocateBlockResponse { @@ -329,7 +329,7 @@ message AllocateBlockResponse { message CommitKeyRequest { required KeyArgs keyArgs = 1; - required uint32 clientID = 2; + required uint64 clientID = 2; } message CommitKeyResponse { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index ffdba7e..50d7ec5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -1,19 +1,18 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ package org.apache.hadoop.ozone.client.rpc; @@ -69,7 +68,6 @@ public class TestCloseContainerHandlingByClient { private static String bucketName; private static String keyString; - /** * Create a MiniDFSCluster for testing. * <p> @@ -80,7 +78,7 @@ public class TestCloseContainerHandlingByClient { @BeforeClass public static void init() throws Exception { conf = new OzoneConfiguration(); - chunkSize = (int)OzoneConsts.MB; + chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize); conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4)); @@ -108,7 +106,7 @@ public class TestCloseContainerHandlingByClient { } private static String fixedLengthString(String string, int length) { - return String.format("%1$"+length+ "s", string); + return String.format("%1$" + length + "s", string); } @Test @@ -288,13 +286,13 @@ public class TestCloseContainerHandlingByClient { ChunkGroupOutputStream groupOutputStream = (ChunkGroupOutputStream) outputStream.getOutputStream(); - int clientId = groupOutputStream.getOpenID(); + long clientId = groupOutputStream.getOpenID(); OMMetadataManager metadataManager = cluster.getOzoneManager().getMetadataManager(); - String objectKey = - metadataManager.getKeyWithDBPrefix(volumeName, bucketName, keyName); - byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientId); - byte[] openKeyData = metadataManager.get(openKey); + byte[] openKey = + metadataManager.getOpenKeyBytes( + volumeName, bucketName, keyName, clientId); + byte[] openKeyData = metadataManager.getOpenKeyTable().get(openKey); OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf( OzoneManagerProtocolProtos.KeyInfo.parseFrom(openKeyData)); List<OmKeyLocationInfo> locationInfoList = @@ -361,7 +359,6 @@ public class TestCloseContainerHandlingByClient { is.close(); } - @Test public void testBlockWriteViaRatis() throws Exception { String keyName = "ratis"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java index 45b3843..f8ad32e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java @@ -51,6 +51,7 @@ import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -601,6 +602,9 @@ public class TestOzoneRpcClient { Assert.assertEquals(toKeyName, key.getName()); } + // Listing all volumes in the cluster feature has to be fixed after HDDS-357. + // TODO: fix this + @Ignore @Test public void testListVolume() throws IOException, OzoneException { String volBase = "vol-" + RandomStringUtils.randomNumeric(3); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java index ab26c00..a3ff6c8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -82,7 +83,8 @@ public class TestOmSQLCli { @Parameterized.Parameters public static Collection<Object[]> data() { return Arrays.asList(new Object[][] { - {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, + // Uncomment the below line if we support leveldb in future. + //{OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB}, {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB} }); } @@ -161,6 +163,9 @@ public class TestOmSQLCli { } } + // After HDDS-357, we have to fix SQLCli. + // TODO: fix SQLCli + @Ignore @Test public void testOmDB() throws Exception { String dbOutPath = GenericTestUtils.getTempPath( http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java index 4908c4d..b6ade60 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java @@ -56,8 +56,8 @@ import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.ListKeys; import org.apache.hadoop.ozone.web.response.ListVolumes; import org.apache.hadoop.util.Time; -import org.apache.hadoop.utils.MetadataKeyFilters; -import org.apache.hadoop.utils.MetadataStore; +import org.apache.hadoop.utils.db.Table; +import org.apache.hadoop.utils.db.TableIterator; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -75,7 +75,6 @@ import java.nio.file.Paths; import java.net.InetSocketAddress; import java.text.ParseException; import java.util.LinkedList; -import java.util.Map; import java.util.Random; import java.util.Set; import java.util.List; @@ -83,8 +82,8 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; -import static org.apache.hadoop.ozone.OzoneConsts.DELETING_KEY_PREFIX; +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_CLIENT_ADDRESS_KEY; @@ -631,13 +630,16 @@ public class TestOzoneManager { storageHandler.deleteKey(keyArgs); Assert.assertEquals(1 + numKeyDeletes, omMetrics.getNumKeyDeletes()); - // Make sure the deleted key has been renamed. - MetadataStore store = cluster.getOzoneManager(). - getMetadataManager().getStore(); - List<Map.Entry<byte[], byte[]>> list = store.getRangeKVs(null, 10, - new MetadataKeyFilters.KeyPrefixFilter() - .addFilter(DELETING_KEY_PREFIX)); - Assert.assertEquals(1, list.size()); + // Make sure the deleted key has been moved to the deleted table. + OMMetadataManager manager = cluster.getOzoneManager(). + getMetadataManager(); + + try(TableIterator<Table.KeyValue> iter = + manager.getDeletedTable().iterator()) { + iter.seekToFirst(); + Table.KeyValue kv = iter.next(); + Assert.assertNotNull(kv); + } // Delete the key again to test deleting non-existing key. try { @@ -1016,13 +1018,14 @@ public class TestOzoneManager { storageHandler.createVolume(createVolumeArgs); } - // Test list all volumes + // Test list all volumes - Removed Support for this operation for time + // being. TODO: we will need to bring this back if needed. UserArgs userArgs0 = new UserArgs(user0, OzoneUtils.getRequestID(), null, null, null, null); - listVolumeArgs = new ListArgs(userArgs0, "Vol-testListVolumes", 100, null); - listVolumeArgs.setRootScan(true); - volumes = storageHandler.listVolumes(listVolumeArgs); - Assert.assertEquals(20, volumes.getVolumes().size()); + //listVolumeArgs = new ListArgs(userArgs0,"Vol-testListVolumes", 100, null); + // listVolumeArgs.setRootScan(true); + // volumes = storageHandler.listVolumes(listVolumeArgs); + // Assert.assertEquals(20, volumes.getVolumes().size()); // Test list all volumes belongs to an user listVolumeArgs = new ListArgs(userArgs0, null, 100, null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java index 31f9214..3765bc8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java @@ -221,6 +221,9 @@ public class TestVolume { assertTrue(newVol.getCreationTime() > 0); } + // Listing all volumes in the cluster feature has to be fixed after HDDS-357. + // TODO: fix this + @Ignore @Test public void testListVolume() throws OzoneException, IOException { runTestListVolume(client); @@ -305,6 +308,9 @@ public class TestVolume { assertEquals(volCount / step, pagecount); } + // Listing all volumes in the cluster feature has to be fixed after HDDS-357. + // TODO: fix this + @Ignore @Test public void testListVolumes() throws Exception { runTestListVolumes(client); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java index 4bbce81..d54addd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java @@ -18,12 +18,11 @@ package org.apache.hadoop.ozone.om; import com.google.common.base.Preconditions; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.exceptions.OMException; -import org.apache.hadoop.ozone.protocol.proto - .OzoneManagerProtocolProtos.BucketInfo; -import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo; import org.apache.hadoop.util.Time; import org.iq80.leveldb.DBException; import org.slf4j.Logger; @@ -46,9 +45,10 @@ public class BucketManagerImpl implements BucketManager { /** * Constructs BucketManager. + * * @param metadataManager */ - public BucketManagerImpl(OMMetadataManager metadataManager){ + public BucketManagerImpl(OMMetadataManager metadataManager) { this.metadataManager = metadataManager; } @@ -73,6 +73,7 @@ public class BucketManagerImpl implements BucketManager { /** * Creates a bucket. + * * @param bucketInfo - OmBucketInfo. */ @Override @@ -86,13 +87,13 @@ public class BucketManagerImpl implements BucketManager { byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName); //Check if the volume exists - if (metadataManager.get(volumeKey) == null) { + if (metadataManager.getVolumeTable().get(volumeKey) == null) { LOG.debug("volume: {} not found ", volumeName); throw new OMException("Volume doesn't exist", OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND); } //Check if bucket already exists - if (metadataManager.get(bucketKey) != null) { + if (metadataManager.getBucketTable().get(bucketKey) != null) { LOG.debug("bucket: {} already exists ", bucketName); throw new OMException("Bucket already exist", OMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS); @@ -106,7 +107,8 @@ public class BucketManagerImpl implements BucketManager { .setIsVersionEnabled(bucketInfo.getIsVersionEnabled()) .setCreationTime(Time.now()) .build(); - metadataManager.put(bucketKey, omBucketInfo.getProtobuf().toByteArray()); + metadataManager.getBucketTable().put(bucketKey, + omBucketInfo.getProtobuf().toByteArray()); LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName); } catch (IOException | DBException ex) { @@ -134,7 +136,7 @@ public class BucketManagerImpl implements BucketManager { metadataManager.readLock().lock(); try { byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName); - byte[] value = metadataManager.get(bucketKey); + byte[] value = metadataManager.getBucketTable().get(bucketKey); if (value == null) { LOG.debug("bucket: {} not found in volume: {}.", bucketName, volumeName); @@ -155,8 +157,9 @@ public class BucketManagerImpl implements BucketManager { /** * Sets bucket property from args. + * * @param args - BucketArgs. - * @throws IOException + * @throws IOException - On Failure. */ @Override public void setBucketProperty(OmBucketArgs args) throws IOException { @@ -167,15 +170,15 @@ public class BucketManagerImpl implements BucketManager { try { byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName); //Check if volume exists - if(metadataManager.get(metadataManager.getVolumeKey(volumeName)) == - null) { + if (metadataManager.getVolumeTable() + .get(metadataManager.getVolumeKey(volumeName)) == null) { LOG.debug("volume: {} not found ", volumeName); throw new OMException("Volume doesn't exist", OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND); } - byte[] value = metadataManager.get(bucketKey); + byte[] value = metadataManager.getBucketTable().get(bucketKey); //Check if bucket exist - if(value == null) { + if (value == null) { LOG.debug("bucket: {} not found ", bucketName); throw new OMException("Bucket doesn't exist", OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND); @@ -187,7 +190,7 @@ public class BucketManagerImpl implements BucketManager { .setBucketName(oldBucketInfo.getBucketName()); //Check ACLs to update - if(args.getAddAcls() != null || args.getRemoveAcls() != null) { + if (args.getAddAcls() != null || args.getRemoveAcls() != null) { bucketInfoBuilder.setAcls(getUpdatedAclList(oldBucketInfo.getAcls(), args.getRemoveAcls(), args.getAddAcls())); LOG.debug("Updating ACLs for bucket: {} in volume: {}", @@ -218,7 +221,7 @@ public class BucketManagerImpl implements BucketManager { } bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime()); - metadataManager.put(bucketKey, + metadataManager.getBucketTable().put(bucketKey, bucketInfoBuilder.build().getProtobuf().toByteArray()); } catch (IOException | DBException ex) { if (!(ex instanceof OMException)) { @@ -242,10 +245,10 @@ public class BucketManagerImpl implements BucketManager { */ private List<OzoneAcl> getUpdatedAclList(List<OzoneAcl> existingAcls, List<OzoneAcl> removeAcls, List<OzoneAcl> addAcls) { - if(removeAcls != null && !removeAcls.isEmpty()) { + if (removeAcls != null && !removeAcls.isEmpty()) { existingAcls.removeAll(removeAcls); } - if(addAcls != null && !addAcls.isEmpty()) { + if (addAcls != null && !addAcls.isEmpty()) { addAcls.stream().filter(acl -> !existingAcls.contains(acl)).forEach( existingAcls::add); } @@ -254,9 +257,10 @@ public class BucketManagerImpl implements BucketManager { /** * Deletes an existing empty bucket from volume. + * * @param volumeName - Name of the volume. * @param bucketName - Name of the bucket. - * @throws IOException + * @throws IOException - on Failure. */ public void deleteBucket(String volumeName, String bucketName) throws IOException { @@ -264,16 +268,17 @@ public class BucketManagerImpl implements BucketManager { Preconditions.checkNotNull(bucketName); metadataManager.writeLock().lock(); try { - byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName); //Check if volume exists - if (metadataManager.get(metadataManager.getVolumeKey(volumeName)) - == null) { + if (metadataManager.getVolumeTable() + .get(metadataManager.getVolumeKey(volumeName)) == null) { LOG.debug("volume: {} not found ", volumeName); throw new OMException("Volume doesn't exist", OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND); } - //Check if bucket exist - if (metadataManager.get(bucketKey) == null) { + + //Check if bucket exists + byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + if (metadataManager.getBucketTable().get(bucketKey) == null) { LOG.debug("bucket: {} not found ", bucketName); throw new OMException("Bucket doesn't exist", OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND); @@ -284,7 +289,7 @@ public class BucketManagerImpl implements BucketManager { throw new OMException("Bucket is not empty", OMException.ResultCodes.FAILED_BUCKET_NOT_EMPTY); } - metadataManager.delete(bucketKey); + metadataManager.getBucketTable().delete(bucketKey); } catch (IOException ex) { if (!(ex instanceof OMException)) { LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName, @@ -301,7 +306,7 @@ public class BucketManagerImpl implements BucketManager { */ @Override public List<OmBucketInfo> listBuckets(String volumeName, - String startBucket, String bucketPrefix, int maxNumOfBuckets) + String startBucket, String bucketPrefix, int maxNumOfBuckets) throws IOException { Preconditions.checkNotNull(volumeName); metadataManager.readLock().lock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 226c07d..a512d7b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -49,7 +49,7 @@ public interface KeyManager { * @param clientID the client that is committing. * @throws IOException */ - void commitKey(OmKeyArgs args, int clientID) throws IOException; + void commitKey(OmKeyArgs args, long clientID) throws IOException; /** * A client calls this on an open key, to request to allocate a new block, @@ -60,7 +60,7 @@ public interface KeyManager { * @return the reference to the new block. * @throws IOException */ - OmKeyLocationInfo allocateBlock(OmKeyArgs args, int clientID) + OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID) throws IOException; /** * Given the args of a key to put, write an open key entry to meta data. @@ -128,7 +128,7 @@ public interface KeyManager { * @throws IOException */ List<OmKeyInfo> listKeys(String volumeName, - String bucketName, String startKey, String keyPrefix, int maxKeys) + String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index d0561d6..d585523 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -17,24 +17,25 @@ package org.apache.hadoop.ozone.om; import com.google.common.base.Preconditions; -import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; +import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; +import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; -import org.apache.hadoop.ozone.protocol.proto - .OzoneManagerProtocolProtos.KeyInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; -import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; import org.apache.hadoop.util.Time; -import org.apache.hadoop.utils.BatchOperation; -import org.iq80.leveldb.DBException; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,25 +43,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Random; - -import static org.apache.hadoop.ozone - .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT; -import static org.apache.hadoop.ozone - .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY; -import static org.apache.hadoop.ozone - .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE; -import static org.apache.hadoop.ozone - .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT; -import static org.apache.hadoop.ozone - .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; -import static org.apache.hadoop.ozone - .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB; -import org.apache.hadoop.hdds.protocol - .proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.protocol - .proto.HddsProtos.ReplicationFactor; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB; /** * Implementation of keyManager. @@ -78,13 +67,12 @@ public class KeyManagerImpl implements KeyManager { private final boolean useRatis; private final long preallocateMax; - private final Random random; private final String omId; public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, - OMMetadataManager metadataManager, - OzoneConfiguration conf, - String omId) { + OMMetadataManager metadataManager, + OzoneConfiguration conf, + String omId) { this.scmBlockClient = scmBlockClient; this.metadataManager = metadataManager; this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB, @@ -94,11 +82,9 @@ public class KeyManagerImpl implements KeyManager { this.preallocateMax = conf.getLong( OZONE_KEY_PREALLOCATION_MAXSIZE, OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT); - random = new Random(); this.omId = omId; } - @Override public void start() { } @@ -113,13 +99,13 @@ public class KeyManagerImpl implements KeyManager { byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName); //Check if the volume exists - if(metadataManager.get(volumeKey) == null) { + if (metadataManager.getVolumeTable().get(volumeKey) == null) { LOG.error("volume not found: {}", volumeName); throw new OMException("Volume not found", OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND); } //Check if bucket already exists - if(metadataManager.get(bucketKey) == null) { + if (metadataManager.getBucketTable().get(bucketKey) == null) { LOG.error("bucket not found: {}/{} ", volumeName, bucketName); throw new OMException("Bucket not found", OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND); @@ -127,7 +113,7 @@ public class KeyManagerImpl implements KeyManager { } @Override - public OmKeyLocationInfo allocateBlock(OmKeyArgs args, int clientID) + public OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID) throws IOException { Preconditions.checkNotNull(args); metadataManager.writeLock().lock(); @@ -137,13 +123,13 @@ public class KeyManagerImpl implements KeyManager { try { validateBucket(volumeName, bucketName); - String objectKey = metadataManager.getKeyWithDBPrefix( - volumeName, bucketName, keyName); - byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientID); - byte[] keyData = metadataManager.get(openKey); + byte[] openKey = metadataManager.getOpenKeyBytes( + volumeName, bucketName, keyName, clientID); + + byte[] keyData = metadataManager.getOpenKeyTable().get(openKey); if (keyData == null) { - LOG.error("Allocate block for a key not in open status in meta store " + - objectKey + " with ID " + clientID); + LOG.error("Allocate block for a key not in open status in meta store" + + " /{}/{}/{} with ID {}", volumeName, bucketName, keyName, clientID); throw new OMException("Open Key not found", OMException.ResultCodes.FAILED_KEY_NOT_FOUND); } @@ -162,7 +148,8 @@ public class KeyManagerImpl implements KeyManager { // the same version keyInfo.appendNewBlocks(Collections.singletonList(info)); keyInfo.updateModifcationTime(); - metadataManager.put(openKey, keyInfo.getProtobuf().toByteArray()); + metadataManager.getOpenKeyTable().put(openKey, + keyInfo.getProtobuf().toByteArray()); return info; } finally { metadataManager.writeLock().unlock(); @@ -172,28 +159,30 @@ public class KeyManagerImpl implements KeyManager { @Override public OpenKeySession openKey(OmKeyArgs args) throws IOException { Preconditions.checkNotNull(args); - metadataManager.writeLock().lock(); String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); + validateBucket(volumeName, bucketName); + + metadataManager.writeLock().lock(); String keyName = args.getKeyName(); ReplicationFactor factor = args.getFactor(); ReplicationType type = args.getType(); + long currentTime = Time.monotonicNowNanos(); // If user does not specify a replication strategy or // replication factor, OM will use defaults. - if(factor == null) { - factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE; + if (factor == null) { + factor = useRatis ? ReplicationFactor.THREE : ReplicationFactor.ONE; } - if(type == null) { + if (type == null) { type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE; } try { - validateBucket(volumeName, bucketName); long requestedSize = Math.min(preallocateMax, args.getDataSize()); List<OmKeyLocationInfo> locations = new ArrayList<>(); - String objectKey = metadataManager.getKeyWithDBPrefix( + byte[] objectKey = metadataManager.getOzoneKeyBytes( volumeName, bucketName, keyName); // requested size is not required but more like a optimization: // SCM looks at the requested, if it 0, no block will be allocated at @@ -218,9 +207,7 @@ public class KeyManagerImpl implements KeyManager { // value, then this value is used, otherwise, we allocate a single block // which is the current size, if read by the client. long size = args.getDataSize() >= 0 ? args.getDataSize() : scmBlockSize; - byte[] keyKey = metadataManager.getDBKeyBytes( - volumeName, bucketName, keyName); - byte[] value = metadataManager.get(keyKey); + byte[] value = metadataManager.getKeyTable().get(objectKey); OmKeyInfo keyInfo; long openVersion; if (value != null) { @@ -233,7 +220,7 @@ public class KeyManagerImpl implements KeyManager { } else { // the key does not exist, create a new object, the new blocks are the // version 0 - long currentTime = Time.now(); + keyInfo = new OmKeyInfo.Builder() .setVolumeName(args.getVolumeName()) .setBucketName(args.getBucketName()) @@ -248,31 +235,31 @@ public class KeyManagerImpl implements KeyManager { .build(); openVersion = 0; } - // Generate a random ID which is not already in meta db. - int id = -1; - // in general this should finish in a couple times at most. putting some - // arbitrary large number here to avoid dead loop. - for (int j = 0; j < 10000; j++) { - id = random.nextInt(); - byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, id); - if (metadataManager.get(openKey) == null) { - metadataManager.put(openKey, keyInfo.getProtobuf().toByteArray()); - break; - } - } - if (id == -1) { - throw new IOException("Failed to find a usable id for " + objectKey); + byte[] openKey = metadataManager.getOpenKeyBytes( + volumeName, bucketName, keyName, currentTime); + if (metadataManager.getOpenKeyTable().get(openKey) != null) { + // This should not happen. If this condition is satisfied, it means + // that we have generated a same openKeyId (i.e. currentTime) for two + // different client who are trying to write the same key at the same + // time. The chance of this happening is very, very minimal. + + // Do we really need this check? Can we avoid this to gain some + // minor performance improvement? + LOG.warn("Cannot allocate key. The generated open key id is already" + + "used for the same key which is currently being written."); + throw new OMException("Cannot allocate key. Not able to get a valid" + + "open key id.", OMException.ResultCodes.FAILED_KEY_ALLOCATION); } + metadataManager.getOpenKeyTable().put(openKey, + keyInfo.getProtobuf().toByteArray()); LOG.debug("Key {} allocated in volume {} bucket {}", keyName, volumeName, bucketName); - return new OpenKeySession(id, keyInfo, openVersion); + return new OpenKeySession(currentTime, keyInfo, openVersion); } catch (OMException e) { throw e; } catch (IOException ex) { - if (!(ex instanceof OMException)) { - LOG.error("Key open failed for volume:{} bucket:{} key:{}", - volumeName, bucketName, keyName, ex); - } + LOG.error("Key open failed for volume:{} bucket:{} key:{}", + volumeName, bucketName, keyName, ex); throw new OMException(ex.getMessage(), OMException.ResultCodes.FAILED_KEY_ALLOCATION); } finally { @@ -281,7 +268,7 @@ public class KeyManagerImpl implements KeyManager { } @Override - public void commitKey(OmKeyArgs args, int clientID) throws IOException { + public void commitKey(OmKeyArgs args, long clientID) throws IOException { Preconditions.checkNotNull(args); metadataManager.writeLock().lock(); String volumeName = args.getVolumeName(); @@ -289,15 +276,14 @@ public class KeyManagerImpl implements KeyManager { String keyName = args.getKeyName(); try { validateBucket(volumeName, bucketName); - String objectKey = metadataManager.getKeyWithDBPrefix( + byte[] openKey = metadataManager.getOpenKeyBytes(volumeName, bucketName, + keyName, clientID); + byte[] objectKey = metadataManager.getOzoneKeyBytes( volumeName, bucketName, keyName); - byte[] objectKeyBytes = metadataManager.getDBKeyBytes(volumeName, - bucketName, keyName); - byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientID); - byte[] openKeyData = metadataManager.get(openKey); + byte[] openKeyData = metadataManager.getOpenKeyTable().get(openKey); if (openKeyData == null) { throw new OMException("Commit a key without corresponding entry " + - DFSUtil.bytes2String(openKey), ResultCodes.FAILED_KEY_NOT_FOUND); + DFSUtil.bytes2String(objectKey), ResultCodes.FAILED_KEY_NOT_FOUND); } OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData)); @@ -305,12 +291,13 @@ public class KeyManagerImpl implements KeyManager { keyInfo.setModificationTime(Time.now()); List<OmKeyLocationInfo> locationInfoList = args.getLocationInfoList(); Preconditions.checkNotNull(locationInfoList); + //update the block length for each block keyInfo.updateLocationInfoList(locationInfoList); - BatchOperation batch = new BatchOperation(); - batch.delete(openKey); - batch.put(objectKeyBytes, keyInfo.getProtobuf().toByteArray()); - metadataManager.writeBatch(batch); + metadataManager.getStore().move(openKey, objectKey, + keyInfo.getProtobuf().toByteArray(), + metadataManager.getOpenKeyTable(), + metadataManager.getKeyTable()); } catch (OMException e) { throw e; } catch (IOException ex) { @@ -331,9 +318,9 @@ public class KeyManagerImpl implements KeyManager { String bucketName = args.getBucketName(); String keyName = args.getKeyName(); try { - byte[] keyKey = metadataManager.getDBKeyBytes( + byte[] keyBytes = metadataManager.getOzoneKeyBytes( volumeName, bucketName, keyName); - byte[] value = metadataManager.get(keyKey); + byte[] value = metadataManager.getKeyTable().get(keyBytes); if (value == null) { LOG.debug("volume:{} bucket:{} Key:{} not found", volumeName, bucketName, keyName); @@ -341,7 +328,7 @@ public class KeyManagerImpl implements KeyManager { OMException.ResultCodes.FAILED_KEY_NOT_FOUND); } return OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value)); - } catch (DBException ex) { + } catch (IOException ex) { LOG.error("Get key failed for volume:{} bucket:{} key:{}", volumeName, bucketName, keyName, ex); throw new OMException(ex.getMessage(), @@ -368,9 +355,9 @@ public class KeyManagerImpl implements KeyManager { metadataManager.writeLock().lock(); try { // fromKeyName should exist - byte[] fromKey = metadataManager.getDBKeyBytes( + byte[] fromKey = metadataManager.getOzoneKeyBytes( volumeName, bucketName, fromKeyName); - byte[] fromKeyValue = metadataManager.get(fromKey); + byte[] fromKeyValue = metadataManager.getKeyTable().get(fromKey); if (fromKeyValue == null) { // TODO: Add support for renaming open key LOG.error( @@ -381,10 +368,20 @@ public class KeyManagerImpl implements KeyManager { OMException.ResultCodes.FAILED_KEY_NOT_FOUND); } + // A rename is a no-op if the target and source name is same. + // TODO: Discuss if we need to throw?. + // TODO: Define the semantics of rename more clearly. Today this code + // will allow rename of a Key across volumes. This should *not* be + // allowed. The documentation of Ozone says that rename is permitted only + // within a volume. + if (fromKeyName.equals(toKeyName)) { + return; + } + // toKeyName should not exist byte[] toKey = - metadataManager.getDBKeyBytes(volumeName, bucketName, toKeyName); - byte[] toKeyValue = metadataManager.get(toKey); + metadataManager.getOzoneKeyBytes(volumeName, bucketName, toKeyName); + byte[] toKeyValue = metadataManager.getKeyTable().get(toKey); if (toKeyValue != null) { LOG.error( "Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}. " @@ -394,19 +391,18 @@ public class KeyManagerImpl implements KeyManager { OMException.ResultCodes.FAILED_KEY_ALREADY_EXISTS); } - if (fromKeyName.equals(toKeyName)) { - return; - } OmKeyInfo newKeyInfo = OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(fromKeyValue)); newKeyInfo.setKeyName(toKeyName); newKeyInfo.updateModifcationTime(); - BatchOperation batch = new BatchOperation(); - batch.delete(fromKey); - batch.put(toKey, newKeyInfo.getProtobuf().toByteArray()); - metadataManager.writeBatch(batch); - } catch (DBException ex) { + try (WriteBatch batch = new WriteBatch()) { + batch.delete(metadataManager.getKeyTable().getHandle(), fromKey); + batch.put(metadataManager.getKeyTable().getHandle(), toKey, + newKeyInfo.getProtobuf().toByteArray()); + metadataManager.getStore().write(batch); + } + } catch (RocksDBException | IOException ex) { LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}", volumeName, bucketName, fromKeyName, toKeyName, ex); throw new OMException(ex.getMessage(), @@ -424,19 +420,19 @@ public class KeyManagerImpl implements KeyManager { String bucketName = args.getBucketName(); String keyName = args.getKeyName(); try { - byte[] objectKey = metadataManager.getDBKeyBytes( + byte[] objectKey = metadataManager.getOzoneKeyBytes( volumeName, bucketName, keyName); - byte[] objectValue = metadataManager.get(objectKey); + byte[] objectValue = metadataManager.getKeyTable().get(objectKey); if (objectValue == null) { throw new OMException("Key not found", OMException.ResultCodes.FAILED_KEY_NOT_FOUND); } - byte[] deletingKey = metadataManager.getDeletedKeyName(objectKey); - BatchOperation batch = new BatchOperation(); - batch.put(deletingKey, objectValue); - batch.delete(objectKey); - metadataManager.writeBatch(batch); - } catch (DBException ex) { + metadataManager.getStore().move(objectKey, + metadataManager.getKeyTable(), + metadataManager.getDeletedTable()); + } catch (OMException ex) { + throw ex; + } catch (IOException ex) { LOG.error(String.format("Delete key failed for volume:%s " + "bucket:%s key:%s", volumeName, bucketName, keyName), ex); throw new OMException(ex.getMessage(), ex, @@ -448,53 +444,30 @@ public class KeyManagerImpl implements KeyManager { @Override public List<OmKeyInfo> listKeys(String volumeName, String bucketName, - String startKey, String keyPrefix, + String startKey, String keyPrefix, int maxKeys) throws IOException { Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); - metadataManager.readLock().lock(); - try { - return metadataManager.listKeys(volumeName, bucketName, - startKey, keyPrefix, maxKeys); - } finally { - metadataManager.readLock().unlock(); - } + // We don't take a lock in this path, since we walk the + // underlying table using an iterator. That automatically creates a + // snapshot of the data, so we don't need these locks at a higher level + // when we iterate. + return metadataManager.listKeys(volumeName, bucketName, + startKey, keyPrefix, maxKeys); } @Override public List<BlockGroup> getPendingDeletionKeys(final int count) throws IOException { - metadataManager.readLock().lock(); - try { - return metadataManager.getPendingDeletionKeys(count); - } finally { - metadataManager.readLock().unlock(); - } + //TODO: Fix this in later patches. + return null; } @Override public void deletePendingDeletionKey(String objectKeyName) - throws IOException{ - Preconditions.checkNotNull(objectKeyName); - if (!objectKeyName.startsWith(OzoneConsts.DELETING_KEY_PREFIX)) { - throw new IllegalArgumentException("Invalid key name," - + " the name should be the key name with deleting prefix"); - } - - // Simply removes the entry from OM DB. - metadataManager.writeLock().lock(); - try { - byte[] pendingDelKey = DFSUtil.string2Bytes(objectKeyName); - byte[] delKeyValue = metadataManager.get(pendingDelKey); - if (delKeyValue == null) { - throw new IOException("Failed to delete key " + objectKeyName - + " because it is not found in DB"); - } - metadataManager.delete(pendingDelKey); - } finally { - metadataManager.writeLock().unlock(); - } + throws IOException { + // TODO : Fix in later patches. } @Override @@ -510,23 +483,6 @@ public class KeyManagerImpl implements KeyManager { @Override public void deleteExpiredOpenKey(String objectKeyName) throws IOException { Preconditions.checkNotNull(objectKeyName); - if (!objectKeyName.startsWith(OzoneConsts.OPEN_KEY_PREFIX)) { - throw new IllegalArgumentException("Invalid key name," - + " the name should be the key name with open key prefix"); - } - - // Simply removes the entry from OM DB. - metadataManager.writeLock().lock(); - try { - byte[] openKey = DFSUtil.string2Bytes(objectKeyName); - byte[] delKeyValue = metadataManager.get(openKey); - if (delKeyValue == null) { - throw new IOException("Failed to delete key " + objectKeyName - + " because it is not found in DB"); - } - metadataManager.delete(openKey); - } finally { - metadataManager.writeLock().unlock(); - } + // TODO: Fix this in later patches. } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff036e49/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index f2e78e6..0e9ae42 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.om; import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.common.BlockGroup; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; -import org.apache.hadoop.utils.BatchOperation; -import org.apache.hadoop.utils.MetadataStore; +import org.apache.hadoop.utils.db.DBStore; +import org.apache.hadoop.utils.db.Table; import java.io.IOException; import java.util.List; @@ -40,68 +40,47 @@ public interface OMMetadataManager { /** * Stop metadata manager. */ - void stop() throws IOException; + void stop() throws Exception; /** * Get metadata store. + * * @return metadata store. */ @VisibleForTesting - MetadataStore getStore(); + DBStore getStore(); /** * Returns the read lock used on Metadata DB. + * * @return readLock */ Lock readLock(); /** * Returns the write lock used on Metadata DB. + * * @return writeLock */ Lock writeLock(); /** - * Returns the value associated with this key. - * @param key - key - * @return value - */ - byte[] get(byte[] key) throws IOException; - - /** - * Puts a Key into Metadata DB. - * @param key - key - * @param value - value - */ - void put(byte[] key, byte[] value) throws IOException; - - /** - * Deletes a Key from Metadata DB. - * @param key - key - */ - void delete(byte[] key) throws IOException; - - /** - * Atomic write a batch of operations. - * @param batch - * @throws IOException - */ - void writeBatch(BatchOperation batch) throws IOException; - - /** * Given a volume return the corresponding DB key. + * * @param volume - Volume name */ byte[] getVolumeKey(String volume); /** * Given a user return the corresponding DB key. + * * @param user - User name */ byte[] getUserKey(String user); /** * Given a volume and bucket, return the corresponding DB key. + * * @param volume - User name * @param bucket - Bucket name */ @@ -109,131 +88,103 @@ public interface OMMetadataManager { /** * Given a volume, bucket and a key, return the corresponding DB key. + * * @param volume - volume name * @param bucket - bucket name * @param key - key name * @return bytes of DB key. */ - byte[] getDBKeyBytes(String volume, String bucket, String key); - - /** - * Returns the DB key name of a deleted key in OM metadata store. - * The name for a deleted key has prefix #deleting# followed by - * the actual key name. - * @param keyName - key name - * @return bytes of DB key. - */ - byte[] getDeletedKeyName(byte[] keyName); + byte[] getOzoneKeyBytes(String volume, String bucket, String key); /** - * Returns the DB key name of a open key in OM metadata store. - * Should be #open# prefix followed by actual key name. - * @param keyName - key name + * Returns the DB key name of a open key in OM metadata store. Should be + * #open# prefix followed by actual key name. + * + * @param volume - volume name + * @param bucket - bucket name + * @param key - key name * @param id - the id for this open * @return bytes of DB key. */ - byte[] getOpenKeyNameBytes(String keyName, int id); + byte[] getOpenKeyBytes(String volume, String bucket, String key, long id); /** - * Returns the full name of a key given volume name, bucket name and key name. - * Generally done by padding certain delimiters. + * Given a volume, check if it is empty, i.e there are no buckets inside it. * - * @param volumeName - volume name - * @param bucketName - bucket name - * @param keyName - key name - * @return the full key name. - */ - String getKeyWithDBPrefix(String volumeName, String bucketName, - String keyName); - - /** - * Given a volume, check if it is empty, - * i.e there are no buckets inside it. * @param volume - Volume name */ boolean isVolumeEmpty(String volume) throws IOException; /** - * Given a volume/bucket, check if it is empty, - * i.e there are no keys inside it. + * Given a volume/bucket, check if it is empty, i.e there are no keys inside + * it. + * * @param volume - Volume name - * @param bucket - Bucket name + * @param bucket - Bucket name * @return true if the bucket is empty */ boolean isBucketEmpty(String volume, String bucket) throws IOException; /** - * Returns a list of buckets represented by {@link OmBucketInfo} - * in the given volume. - * - * @param volumeName - * the name of the volume. This argument is required, - * this method returns buckets in this given volume. - * @param startBucket - * the start bucket name. Only the buckets whose name is - * after this value will be included in the result. - * This key is excluded from the result. - * @param bucketPrefix - * bucket name prefix. Only the buckets whose name has - * this prefix will be included in the result. - * @param maxNumOfBuckets - * the maximum number of buckets to return. It ensures - * the size of the result will not exceed this limit. + * Returns a list of buckets represented by {@link OmBucketInfo} in the given + * volume. + * + * @param volumeName the name of the volume. This argument is required, this + * method returns buckets in this given volume. + * @param startBucket the start bucket name. Only the buckets whose name is + * after this value will be included in the result. This key is excluded from + * the result. + * @param bucketPrefix bucket name prefix. Only the buckets whose name has + * this prefix will be included in the result. + * @param maxNumOfBuckets the maximum number of buckets to return. It ensures + * the size of the result will not exceed this limit. * @return a list of buckets. * @throws IOException */ List<OmBucketInfo> listBuckets(String volumeName, String startBucket, - String bucketPrefix, int maxNumOfBuckets) throws IOException; - - /** - * Returns a list of keys represented by {@link OmKeyInfo} - * in the given bucket. - * - * @param volumeName - * the name of the volume. - * @param bucketName - * the name of the bucket. - * @param startKey - * the start key name, only the keys whose name is - * after this value will be included in the result. - * This key is excluded from the result. - * @param keyPrefix - * key name prefix, only the keys whose name has - * this prefix will be included in the result. - * @param maxKeys - * the maximum number of keys to return. It ensures - * the size of the result will not exceed this limit. + String bucketPrefix, int maxNumOfBuckets) + throws IOException; + + /** + * Returns a list of keys represented by {@link OmKeyInfo} in the given + * bucket. + * + * @param volumeName the name of the volume. + * @param bucketName the name of the bucket. + * @param startKey the start key name, only the keys whose name is after this + * value will be included in the result. This key is excluded from the + * result. + * @param keyPrefix key name prefix, only the keys whose name has this prefix + * will be included in the result. + * @param maxKeys the maximum number of keys to return. It ensures the size of + * the result will not exceed this limit. * @return a list of keys. * @throws IOException */ List<OmKeyInfo> listKeys(String volumeName, - String bucketName, String startKey, String keyPrefix, int maxKeys) + String bucketName, String startKey, String keyPrefix, int maxKeys) throws IOException; /** - * Returns a list of volumes owned by a given user; if user is null, - * returns all volumes. + * Returns a list of volumes owned by a given user; if user is null, returns + * all volumes. * - * @param userName - * volume owner - * @param prefix - * the volume prefix used to filter the listing result. - * @param startKey - * the start volume name determines where to start listing from, - * this key is excluded from the result. - * @param maxKeys - * the maximum number of volumes to return. + * @param userName volume owner + * @param prefix the volume prefix used to filter the listing result. + * @param startKey the start volume name determines where to start listing + * from, this key is excluded from the result. + * @param maxKeys the maximum number of volumes to return. * @return a list of {@link OmVolumeArgs} * @throws IOException */ List<OmVolumeArgs> listVolumes(String userName, String prefix, - String startKey, int maxKeys) throws IOException; + String startKey, int maxKeys) throws IOException; /** * Returns a list of pending deletion key info that ups to the given count. - * Each entry is a {@link BlockGroup}, which contains the info about the - * key name and all its associated block IDs. A pending deletion key is - * stored with #deleting# prefix in OM DB. + * Each entry is a {@link BlockGroup}, which contains the info about the key + * name and all its associated block IDs. A pending deletion key is stored + * with #deleting# prefix in OM DB. * * @param count max number of keys to return. * @return a list of {@link BlockGroup} represent keys and blocks. @@ -250,4 +201,47 @@ public interface OMMetadataManager { * @throws IOException */ List<BlockGroup> getExpiredOpenKeys() throws IOException; + + /** + * Returns the user Table. + * + * @return UserTable. + */ + Table getUserTable(); + + /** + * Returns the Volume Table. + * + * @return VolumeTable. + */ + Table getVolumeTable(); + + /** + * Returns the BucketTable. + * + * @return BucketTable. + */ + Table getBucketTable(); + + /** + * Returns the KeyTable. + * + * @return KeyTable. + */ + Table getKeyTable(); + + /** + * Get Deleted Table. + * + * @return Deleted Table. + */ + Table getDeletedTable(); + + /** + * Gets the OpenKeyTable. + * + * @return Table. + */ + Table getOpenKeyTable(); + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org