subject:"\[hadoop\] 04\/05\: HDFS\-16128. \[FGL\] Added support for saving\/loading an FS Image for PartitionedGSet. Contributed by Xing Lin. \(#3201\)"

[hadoop] 04/05: HDFS-16128. [FGL] Added support for saving/loading an FS Image for PartitionedGSet. Contributed by Xing Lin. (#3201)

2021-07-31 Thread shv

This is an automated email from the ASF dual-hosted git repository.

shv pushed a commit to branch fgl
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit b135d26f1e53443162b1106ae56e96a766632171
Author: Xing Lin 
AuthorDate: Sat Jul 31 12:56:05 2021 -0700

HDFS-16128. [FGL] Added support for saving/loading an FS Image for 
PartitionedGSet. Contributed by Xing Lin. (#3201)
---
 .../org/apache/hadoop/util/PartitionedGSet.java|  24 +++--
 .../hadoop/hdfs/server/namenode/FSDirMkdirOp.java  |   4 +-
 .../hadoop/hdfs/server/namenode/FSDirectory.java   |  70 ++
 .../hadoop/hdfs/server/namenode/FSImage.java   |  12 +++
 .../hdfs/server/namenode/FSImageFormatPBINode.java |  11 ++-
 .../hadoop/hdfs/server/namenode/INodeMap.java  | 105 -
 6 files changed, 168 insertions(+), 58 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
index f3569cc..f493402 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
@@ -68,7 +68,7 @@ public class PartitionedGSet implements 
GSet {
* Consists of a hash table {@link LightWeightGSet} and a lock, which
* controls access to this partition independently on the other ones.
*/
-  private class PartitionEntry extends LightWeightGSet {
+  public class PartitionEntry extends LightWeightGSet {
 private final LatchLock partLock;
 
 PartitionEntry(int defaultPartitionCapacity) {
@@ -121,7 +121,7 @@ public class PartitionedGSet implements 
GSet {
 return size;
   }
 
-  protected PartitionEntry getPartition(final K key) {
+  public PartitionEntry getPartition(final K key) {
 Entry partEntry = partitions.floorEntry(key);
 if(partEntry == null) {
   return null;
@@ -174,6 +174,10 @@ public class PartitionedGSet implements 
GSet {
 E result = part.put(element);
 if(result == null) {  // new element
   size++;
+  LOG.debug("partitionPGSet.put: added key {}, size is now {} ", key, 
size);
+} else {
+  LOG.debug("partitionPGSet.put: replaced key {}, size is now {}",
+  key, size);
 }
 return result;
   }
@@ -230,19 +234,25 @@ public class PartitionedGSet implements 
GSet {
   try {
 long[] key = (long[]) inodeClass.
 getMethod("getNamespaceKey", int.class).invoke(e.getKey(), 2);
-long[] firstKey = new long[0];
+long[] firstKey = new long[key.length];
 if(part.iterator().hasNext()) {
   Object first = part.iterator().next();
-  firstKey = (long[]) inodeClass.getMethod(
+  long[] firstKeyRef = (long[]) inodeClass.getMethod(
 "getNamespaceKey", int.class).invoke(first, 2);
   Object parent = inodeClass.
   getMethod("getParent").invoke(first);
   long parentId = (parent == null ? 0L :
 (long) inodeClass.getMethod("getId").invoke(parent));
+  for (int j=0; j < key.length; j++) {
+firstKey[j] = firstKeyRef[j];
+  }
   firstKey[0] = parentId;
 }
 LOG.error("Partition #{}\t key: {}\t size: {}\t first: {}",
 i++, key, s, firstKey);  // SHV should be info
+  } catch (NoSuchElementException ex) {
+LOG.error("iterator.next() throws NoSuchElementException.");
+throw ex;
   } catch (Exception ex) {
 LOG.error("Cannot find Method getNamespaceKey() in {}", inodeClass);
   }
@@ -250,8 +260,8 @@ public class PartitionedGSet implements 
GSet {
 partSizeAvg = (int) (totalSize / parts.size());
 LOG.error("Partition sizes: min = {}, avg = {}, max = {}, sum = {}",
 partSizeMin, partSizeAvg, partSizeMax, totalSize);
-LOG.error("Number of partitions: empty = {}, full = {}",
-numEmptyPartitions, numFullPartitions);
+LOG.error("Number of partitions: empty = {}, in-use = {}, full = {}",
+numEmptyPartitions, parts.size()-numEmptyPartitions, 
numFullPartitions);
   }
 
   @Override
@@ -277,6 +287,8 @@ public class PartitionedGSet implements 
GSet {
 private Iterator keyIterator;
 private Iterator partitionIterator;
 
+// Set partitionIterator to point to the first partition, or set it to null
+// when there is no partitions created for this PartitionedGSet.
 public EntryIterator() {
   keyIterator = partitions.keySet().iterator();
  
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index 5a40906..ef08e9e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java

[hadoop] 04/05: HDFS-16128. [FGL] Added support for saving/loading an FS Image for PartitionedGSet. Contributed by Xing Lin. (#3201)

2021-07-23 Thread shv

This is an automated email from the ASF dual-hosted git repository.

shv pushed a commit to branch fgl
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 0a73bde9a0051060943fdd2d7461008f471ce30d
Author: Xing Lin 
AuthorDate: Fri Jul 23 12:58:31 2021 -0700

HDFS-16128. [FGL] Added support for saving/loading an FS Image for 
PartitionedGSet. Contributed by Xing Lin. (#3201)
---
 .../org/apache/hadoop/util/PartitionedGSet.java| 21 --
 .../hadoop/hdfs/server/namenode/FSDirMkdirOp.java  |  4 +-
 .../hadoop/hdfs/server/namenode/FSDirectory.java   | 83 ++
 .../hadoop/hdfs/server/namenode/FSImage.java   | 12 
 .../hdfs/server/namenode/FSImageFormatPBINode.java | 11 +--
 .../hadoop/hdfs/server/namenode/INodeMap.java  | 77 
 6 files changed, 182 insertions(+), 26 deletions(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
index f3569cc..5fe50ab 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PartitionedGSet.java
@@ -68,7 +68,7 @@ public class PartitionedGSet implements 
GSet {
* Consists of a hash table {@link LightWeightGSet} and a lock, which
* controls access to this partition independently on the other ones.
*/
-  private class PartitionEntry extends LightWeightGSet {
+  public class PartitionEntry extends LightWeightGSet {
 private final LatchLock partLock;
 
 PartitionEntry(int defaultPartitionCapacity) {
@@ -121,7 +121,7 @@ public class PartitionedGSet implements 
GSet {
 return size;
   }
 
-  protected PartitionEntry getPartition(final K key) {
+  public PartitionEntry getPartition(final K key) {
 Entry partEntry = partitions.floorEntry(key);
 if(partEntry == null) {
   return null;
@@ -174,6 +174,10 @@ public class PartitionedGSet implements 
GSet {
 E result = part.put(element);
 if(result == null) {  // new element
   size++;
+  LOG.debug("partitionPGSet.put: added key {}, size is now {} ", key, 
size);
+} else {
+  LOG.debug("partitionPGSet.put: replaced key {}, size is now {}",
+  key, size);
 }
 return result;
   }
@@ -230,15 +234,18 @@ public class PartitionedGSet implements 
GSet {
   try {
 long[] key = (long[]) inodeClass.
 getMethod("getNamespaceKey", int.class).invoke(e.getKey(), 2);
-long[] firstKey = new long[0];
+long[] firstKey = new long[key.length];
 if(part.iterator().hasNext()) {
   Object first = part.iterator().next();
-  firstKey = (long[]) inodeClass.getMethod(
+  long[] firstKeyRef = (long[]) inodeClass.getMethod(
 "getNamespaceKey", int.class).invoke(first, 2);
   Object parent = inodeClass.
   getMethod("getParent").invoke(first);
   long parentId = (parent == null ? 0L :
 (long) inodeClass.getMethod("getId").invoke(parent));
+  for (int j=0; j < key.length; j++) {
+firstKey[j] = firstKeyRef[j];
+  }
   firstKey[0] = parentId;
 }
 LOG.error("Partition #{}\t key: {}\t size: {}\t first: {}",
@@ -250,8 +257,8 @@ public class PartitionedGSet implements 
GSet {
 partSizeAvg = (int) (totalSize / parts.size());
 LOG.error("Partition sizes: min = {}, avg = {}, max = {}, sum = {}",
 partSizeMin, partSizeAvg, partSizeMax, totalSize);
-LOG.error("Number of partitions: empty = {}, full = {}",
-numEmptyPartitions, numFullPartitions);
+LOG.error("Number of partitions: empty = {}, in-use = {}, full = {}",
+numEmptyPartitions, parts.size()-numEmptyPartitions, 
numFullPartitions);
   }
 
   @Override
@@ -277,6 +284,8 @@ public class PartitionedGSet implements 
GSet {
 private Iterator keyIterator;
 private Iterator partitionIterator;
 
+// Set partitionIterator to point to the first partition, or set it to null
+// when there is no partitions created for this PartitionedGSet.
 public EntryIterator() {
   keyIterator = partitions.keySet().iterator();
  
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index 5a40906..1c979e5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -35,6 +35,7 @@ import 
org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.security.AccessControlException;
 
 import java.io.IOException;
+import java.util.Arrays;

[hadoop] 04/05: HDFS-16128. [FGL] Added support for saving/loading an FS Image for PartitionedGSet. Contributed by Xing Lin. (#3201)

[hadoop] 04/05: HDFS-16128. [FGL] Added support for saving/loading an FS Image for PartitionedGSet. Contributed by Xing Lin. (#3201)

2 matches

Site Navigation

Mail list logo

Footer information