This is an automated email from the ASF dual-hosted git repository.
jitendra pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fa067aa HDDS-936. Need a tool to map containers to ozone objects.
Contributed by Sarun Singla
fa067aa is described below
commit fa067aa157d497e780a7f064924921fe6b3e3a7f
Author: Jitendra Pandey <jiten...@apache.org>
AuthorDate: Wed Feb 13 15:43:14 2019 -0800
HDDS-936. Need a tool to map containers to ozone objects. Contributed by
Sarun Singla
---
hadoop-ozone/tools/pom.xml | 5 +
.../apache/hadoop/ozone/fsck/BlockIdDetails.java | 66 ++++++++++
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 134 +++++++++++++++++++++
.../org/apache/hadoop/ozone/fsck/package-info.java | 44 +++++++
.../hadoop/ozone/fsck/TestContainerMapper.java | 117 ++++++++++++++++++
.../org/apache/hadoop/ozone/fsck/package-info.java | 44 +++++++
6 files changed, 410 insertions(+)
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index 1af72c9..95bef70 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -78,6 +78,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<version>2.15.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-ozone-manager</artifactId>
+ <version>0.4.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/BlockIdDetails.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/BlockIdDetails.java
new file mode 100644
index 0000000..4f7b8a1
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/BlockIdDetails.java
@@ -0,0 +1,66 @@
+package org.apache.hadoop.ozone.fsck;
+
+import java.util.Objects;
+
+/**
+ * Getter and Setter for BlockDetails.
+ */
+
+public class BlockIdDetails {
+
+ private String bucketName;
+ private String blockVol;
+ private String keyName;
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public void setBucketName(String bucketName) {
+ this.bucketName = bucketName;
+ }
+
+ public String getBlockVol() {
+ return blockVol;
+ }
+
+ public void setBlockVol(String blockVol) {
+ this.blockVol = blockVol;
+ }
+
+ public String getKeyName() {
+ return keyName;
+ }
+
+ public void setKeyName(String keyName) {
+ this.keyName = keyName;
+ }
+
+ @Override
+ public String toString() {
+ return "BlockIdDetails{" +
+ "bucketName='" + bucketName + '\'' +
+ ", blockVol='" + blockVol + '\'' +
+ ", keyName='" + keyName + '\'' +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ BlockIdDetails that = (BlockIdDetails) o;
+ return Objects.equals(bucketName, that.bucketName) &&
+ Objects.equals(blockVol, that.blockVol) &&
+ Objects.equals(keyName, that.keyName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(bucketName, blockVol, keyName);
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java
new file mode 100644
index 0000000..29a06be
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.fsck;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.utils.db.Table;
+import org.apache.hadoop.utils.db.TableIterator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
+
+
+/**
+ * Generates Container Id to Blocks and BlockDetails mapping.
+ */
+
+public class ContainerMapper {
+
+
+ private static Table getMetaTable(OzoneConfiguration configuration)
+ throws IOException {
+ OmMetadataManagerImpl metadataManager =
+ new OmMetadataManagerImpl(configuration);
+ return metadataManager.getKeyTable();
+ }
+
+ public static void main(String[] args) throws IOException {
+ String path = args[0];
+ if (path == null) {
+ throw new IOException("Path cannot be null");
+ }
+
+ OzoneConfiguration configuration = new OzoneConfiguration();
+ configuration.set(OZONE_OM_DB_DIRS, path);
+
+ ContainerMapper containerMapper = new ContainerMapper();
+ Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
+ containerMapper.parseOmDB(configuration);
+
+
+ ObjectMapper mapper = new ObjectMapper();
+ System.out.println(mapper.writeValueAsString(dataMap));
+
+ }
+
+ /**
+ * Generates Container Id to Blocks and BlockDetails mapping.
+ * @param configuration @{@link OzoneConfiguration}
+ * @return Map<Long, List<Map<Long, @BlockDetails>>>
+ * Map of ContainerId -> (Block, Block info)
+ * @throws IOException
+ */
+ public Map<Long, List<Map<Long, BlockIdDetails>>>
+ parseOmDB(OzoneConfiguration configuration) throws IOException {
+ String path = configuration.get(OZONE_OM_DB_DIRS);
+ if (path == null || path.isEmpty()) {
+ throw new IOException(OZONE_OM_DB_DIRS + "should be set ");
+ } else {
+ Table keyTable = getMetaTable(configuration);
+ Map<Long, List<Map<Long, BlockIdDetails>>> dataMap = new HashMap<>();
+
+ if (keyTable != null) {
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ keyValueTableIterator = keyTable.iterator()) {
+ while (keyValueTableIterator.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> keyValue =
+ keyValueTableIterator.next();
+ OmKeyInfo omKeyInfo = keyValue.getValue();
+ byte[] value = omKeyInfo.getProtobuf().toByteArray();
+ OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(
+ OzoneManagerProtocolProtos.KeyInfo.parseFrom(value));
+ for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo
+ .getKeyLocationVersions()) {
+ List<OmKeyLocationInfo> keyLocationInfo = keyLocationInfoGroup
+ .getLocationList();
+ for (OmKeyLocationInfo keyLocation : keyLocationInfo) {
+ BlockIdDetails blockIdDetails = new BlockIdDetails();
+ Map<Long, BlockIdDetails> innerMap = new HashMap<>();
+
+ long containerID = keyLocation.getBlockID().getContainerID();
+ long blockID = keyLocation.getBlockID().getLocalID();
+ blockIdDetails.setBucketName(keyInfo.getBucketName());
+ blockIdDetails.setBlockVol(keyInfo.getVolumeName());
+ blockIdDetails.setKeyName(keyInfo.getKeyName());
+
+ List<Map<Long, BlockIdDetails>> innerList = new ArrayList<>();
+ innerMap.put(blockID, blockIdDetails);
+
+ if (dataMap.containsKey(containerID)) {
+ innerList = dataMap.get(containerID);
+ }
+
+ innerList.add(innerMap);
+ dataMap.put(containerID, innerList);
+ }
+ }
+ }
+ }
+ }
+
+ return dataMap;
+
+ }
+ }
+}
+
+
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/package-info.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/package-info.java
new file mode 100644
index 0000000..432d65c
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/package-info.java
@@ -0,0 +1,44 @@
+/**
+ * Package info.
+ * <p>
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * <p>
+ * fsck tool.
+ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * fsck tool.
+ */
+
+
+package org.apache.hadoop.ozone.fsck;
\ No newline at end of file
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java
new file mode 100644
index 0000000..112674a
--- /dev/null
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.fsck;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import
org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.ratis.util.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
+import static org.junit.Assert.*;
+
+/**
+ * Test cases for ContainerMapper.
+ */
+
+public class TestContainerMapper {
+
+ private static MiniOzoneCluster cluster = null;
+ private static OzoneClient ozClient = null;
+ private static ObjectStore store = null;
+ private static OzoneManager ozoneManager;
+ private static StorageContainerLocationProtocolClientSideTranslatorPB
+ storageContainerLocationClient;
+ private static final String SCM_ID = UUID.randomUUID().toString();
+ private static String volName = UUID.randomUUID().toString();
+ private static String bucketName = UUID.randomUUID().toString();
+ private static OzoneConfiguration conf;
+ private static List<String> keyList = new ArrayList<>();
+ private static String dbPath;
+
+
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ dbPath = GenericTestUtils.getRandomizedTempPath();
+ conf.set(OZONE_OM_DB_DIRS, dbPath);
+ conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, "100MB");
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .setScmId(SCM_ID)
+ .build();
+ cluster.waitForClusterToBeReady();
+ ozClient = OzoneClientFactory.getRpcClient(conf);
+ store = ozClient.getObjectStore();
+ storageContainerLocationClient =
+ cluster.getStorageContainerLocationClient();
+ ozoneManager = cluster.getOzoneManager();
+ store.createVolume(volName);
+ OzoneVolume volume = store.getVolume(volName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+ byte[] data = generateData(10 * 1024 * 1024, (byte)98);
+
+ for (int i = 0; i < 20; i++) {
+ String key = UUID.randomUUID().toString();
+ keyList.add(key);
+ OzoneOutputStream out = bucket.createKey(key, data.length,
+ ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
+ new HashMap<String, String>());
+ out.write(data, 0, data.length);
+ out.close();
+ }
+ cluster.stop();
+ }
+
+ @Test
+ public void testContainerMapper() throws Exception {
+ ContainerMapper containerMapper = new ContainerMapper();
+ Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
+ containerMapper.parseOmDB(conf);
+ // As we have created 20 keys with 10 MB size, and each
+ // container max size is 100 MB, it should create 3 containers because
+ // containers are closing before reaching the threshold
+ assertEquals(3, dataMap.size());
+ }
+
+ private static byte[] generateData(int size, byte val) {
+ byte[] chars = new byte[size];
+ Arrays.fill(chars, val);
+ return chars;
+ }
+
+ @AfterClass
+ public static void shutdown() throws IOException {
+ cluster.shutdown();
+ FileUtils.deleteFully(new File(dbPath));
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/package-info.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/package-info.java
new file mode 100644
index 0000000..432d65c
--- /dev/null
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/package-info.java
@@ -0,0 +1,44 @@
+/**
+ * Package info.
+ * <p>
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * <p>
+ * fsck tool.
+ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * fsck tool.
+ */
+
+
+package org.apache.hadoop.ozone.fsck;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
