[25/50] [abbrv] hbase git commit: HBASE-15669 HFile size is not considered correctly in a replication request

2016-05-10 Thread syuanjiang 
HBASE-15669 HFile size is not considered correctly in a replication request


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/34e9a6ff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/34e9a6ff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/34e9a6ff

Branch: refs/heads/hbase-12439
Commit: 34e9a6ff301f40aa3f6ce33ac1b86f9e50fa6694
Parents: bec81b1
Author: Ashish Singhi 
Authored: Fri May 6 17:26:17 2016 +0530
Committer: Ashish Singhi 
Committed: Fri May 6 17:26:17 2016 +0530

--
 .../hadoop/hbase/protobuf/ProtobufUtil.java |  12 +-
 .../hbase/protobuf/generated/WALProtos.java | 159 ---
 hbase-protocol/src/main/protobuf/WAL.proto  |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  18 ++-
 .../regionserver/ReplicationSource.java |  44 -
 .../regionserver/TestReplicationSink.java   |   4 +-
 .../TestReplicationSourceManager.java   |  25 ++-
 7 files changed, 229 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 50a4920..62dfd45 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -3063,13 +3063,16 @@ public final class ProtobufUtil {
* @param tableName The tableName into which the bulk load is being 
imported into.
* @param encodedRegionName Encoded region name of the region which is being 
bulk loaded.
* @param storeFilesA set of store files of a column family are bulk 
loaded.
+   * @param storeFilesSize  Map of store files and their lengths
* @param bulkloadSeqId sequence ID (by a force flush) used to create 
bulk load hfile
*  name
* @return The WAL log marker for bulk loads.
*/
   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName 
tableName,
-  ByteString encodedRegionName, Map> storeFiles, long 
bulkloadSeqId) {
-BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
+  ByteString encodedRegionName, Map> storeFiles,
+  Map storeFilesSize, long bulkloadSeqId) {
+BulkLoadDescriptor.Builder desc =
+BulkLoadDescriptor.newBuilder()
 .setTableName(ProtobufUtil.toProtoTableName(tableName))
 
.setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
 
@@ -3078,7 +3081,10 @@ public final class ProtobufUtil {
   .setFamilyName(ByteStringer.wrap(entry.getKey()))
   .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to 
region
   for (Path path : entry.getValue()) {
-builder.addStoreFile(path.getName());
+String name = path.getName();
+builder.addStoreFile(name);
+Long size = storeFilesSize.get(name) == null ? (Long) 0L : 
storeFilesSize.get(name);
+builder.setStoreFileSize(size);
   }
   desc.addStores(builder);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
index d74688e..6252d51 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
@@ -7821,6 +7821,24 @@ public final class WALProtos {
  */
 com.google.protobuf.ByteString
 getStoreFileBytes(int index);
+
+// optional uint64 store_file_size = 4;
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+boolean hasStoreFileSize();
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+long getStoreFileSize();
   }
   /**
* Protobuf type {@code hbase.pb.StoreDescriptor}
@@ -7891,6 +7909,11 @@ public final class WALProtos {
   storeFile_.add(input.readBytes());
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  storeFileSize_ = input.readUInt64();
+  break;
+}
   }
 }
   } catch (com.goog

hbase git commit: HBASE-15669 HFile size is not considered correctly in a replication request

2016-05-06 Thread ashishsinghi 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 f5b57cd04 -> 5ef9d4752


HBASE-15669 HFile size is not considered correctly in a replication request


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ef9d475
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ef9d475
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ef9d475

Branch: refs/heads/branch-1.3
Commit: 5ef9d475281b498a3c97b3842aa15699965109a7
Parents: f5b57cd
Author: Ashish Singhi 
Authored: Fri May 6 17:28:06 2016 +0530
Committer: Ashish Singhi 
Committed: Fri May 6 17:30:22 2016 +0530

--
 .../hadoop/hbase/protobuf/ProtobufUtil.java |  12 +-
 .../hbase/protobuf/generated/WALProtos.java | 159 ---
 hbase-protocol/src/main/protobuf/WAL.proto  |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  18 ++-
 .../regionserver/ReplicationSource.java |  44 -
 .../regionserver/TestReplicationSink.java   |   4 +-
 .../TestReplicationSourceManager.java   |  25 ++-
 7 files changed, 229 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5ef9d475/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 82f5f0d..08cf6fa 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -3128,13 +3128,16 @@ public final class ProtobufUtil {
* @param tableName The tableName into which the bulk load is being 
imported into.
* @param encodedRegionName Encoded region name of the region which is being 
bulk loaded.
* @param storeFilesA set of store files of a column family are bulk 
loaded.
+   * @param storeFilesSize  Map of store files and their lengths
* @param bulkloadSeqId sequence ID (by a force flush) used to create 
bulk load hfile
*  name
* @return The WAL log marker for bulk loads.
*/
   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName 
tableName,
-  ByteString encodedRegionName, Map> storeFiles, long 
bulkloadSeqId) {
-BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
+  ByteString encodedRegionName, Map> storeFiles,
+  Map storeFilesSize, long bulkloadSeqId) {
+BulkLoadDescriptor.Builder desc =
+BulkLoadDescriptor.newBuilder()
 .setTableName(ProtobufUtil.toProtoTableName(tableName))
 
.setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
 
@@ -3143,7 +3146,10 @@ public final class ProtobufUtil {
   .setFamilyName(ByteStringer.wrap(entry.getKey()))
   .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to 
region
   for (Path path : entry.getValue()) {
-builder.addStoreFile(path.getName());
+String name = path.getName();
+builder.addStoreFile(name);
+Long size = storeFilesSize.get(name) == null ? (Long) 0L : 
storeFilesSize.get(name);
+builder.setStoreFileSize(size);
   }
   desc.addStores(builder);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/5ef9d475/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
index d74688e..6252d51 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
@@ -7821,6 +7821,24 @@ public final class WALProtos {
  */
 com.google.protobuf.ByteString
 getStoreFileBytes(int index);
+
+// optional uint64 store_file_size = 4;
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+boolean hasStoreFileSize();
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+long getStoreFileSize();
   }
   /**
* Protobuf type {@code hbase.pb.StoreDescriptor}
@@ -7891,6 +7909,11 @@ public final class WALProtos {
   storeFile_.add(input.readBytes());
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  storeFileSize_ = input.readUInt64();
+

hbase git commit: HBASE-15669 HFile size is not considered correctly in a replication request

2016-05-06 Thread ashishsinghi 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 b9df7978f -> 0964884b9


HBASE-15669 HFile size is not considered correctly in a replication request


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0964884b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0964884b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0964884b

Branch: refs/heads/branch-1
Commit: 0964884b925f251725bcd101f23f77a5d3d829e1
Parents: b9df797
Author: Ashish Singhi 
Authored: Fri May 6 17:28:06 2016 +0530
Committer: Ashish Singhi 
Committed: Fri May 6 17:28:06 2016 +0530

--
 .../hadoop/hbase/protobuf/ProtobufUtil.java |  12 +-
 .../hbase/protobuf/generated/WALProtos.java | 159 ---
 hbase-protocol/src/main/protobuf/WAL.proto  |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  18 ++-
 .../regionserver/ReplicationSource.java |  44 -
 .../regionserver/TestReplicationSink.java   |   4 +-
 .../TestReplicationSourceManager.java   |  25 ++-
 7 files changed, 229 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0964884b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 82f5f0d..08cf6fa 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -3128,13 +3128,16 @@ public final class ProtobufUtil {
* @param tableName The tableName into which the bulk load is being 
imported into.
* @param encodedRegionName Encoded region name of the region which is being 
bulk loaded.
* @param storeFilesA set of store files of a column family are bulk 
loaded.
+   * @param storeFilesSize  Map of store files and their lengths
* @param bulkloadSeqId sequence ID (by a force flush) used to create 
bulk load hfile
*  name
* @return The WAL log marker for bulk loads.
*/
   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName 
tableName,
-  ByteString encodedRegionName, Map> storeFiles, long 
bulkloadSeqId) {
-BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
+  ByteString encodedRegionName, Map> storeFiles,
+  Map storeFilesSize, long bulkloadSeqId) {
+BulkLoadDescriptor.Builder desc =
+BulkLoadDescriptor.newBuilder()
 .setTableName(ProtobufUtil.toProtoTableName(tableName))
 
.setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
 
@@ -3143,7 +3146,10 @@ public final class ProtobufUtil {
   .setFamilyName(ByteStringer.wrap(entry.getKey()))
   .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to 
region
   for (Path path : entry.getValue()) {
-builder.addStoreFile(path.getName());
+String name = path.getName();
+builder.addStoreFile(name);
+Long size = storeFilesSize.get(name) == null ? (Long) 0L : 
storeFilesSize.get(name);
+builder.setStoreFileSize(size);
   }
   desc.addStores(builder);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/0964884b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
index d74688e..6252d51 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
@@ -7821,6 +7821,24 @@ public final class WALProtos {
  */
 com.google.protobuf.ByteString
 getStoreFileBytes(int index);
+
+// optional uint64 store_file_size = 4;
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+boolean hasStoreFileSize();
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+long getStoreFileSize();
   }
   /**
* Protobuf type {@code hbase.pb.StoreDescriptor}
@@ -7891,6 +7909,11 @@ public final class WALProtos {
   storeFile_.add(input.readBytes());
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  storeFileSize_ = input.readUInt64();
+

hbase git commit: HBASE-15669 HFile size is not considered correctly in a replication request

2016-05-06 Thread ashishsinghi 
Repository: hbase
Updated Branches:
  refs/heads/master bec81b197 -> 34e9a6ff3


HBASE-15669 HFile size is not considered correctly in a replication request


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/34e9a6ff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/34e9a6ff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/34e9a6ff

Branch: refs/heads/master
Commit: 34e9a6ff301f40aa3f6ce33ac1b86f9e50fa6694
Parents: bec81b1
Author: Ashish Singhi 
Authored: Fri May 6 17:26:17 2016 +0530
Committer: Ashish Singhi 
Committed: Fri May 6 17:26:17 2016 +0530

--
 .../hadoop/hbase/protobuf/ProtobufUtil.java |  12 +-
 .../hbase/protobuf/generated/WALProtos.java | 159 ---
 hbase-protocol/src/main/protobuf/WAL.proto  |   1 +
 .../hadoop/hbase/regionserver/HRegion.java  |  18 ++-
 .../regionserver/ReplicationSource.java |  44 -
 .../regionserver/TestReplicationSink.java   |   4 +-
 .../TestReplicationSourceManager.java   |  25 ++-
 7 files changed, 229 insertions(+), 34 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 50a4920..62dfd45 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -3063,13 +3063,16 @@ public final class ProtobufUtil {
* @param tableName The tableName into which the bulk load is being 
imported into.
* @param encodedRegionName Encoded region name of the region which is being 
bulk loaded.
* @param storeFilesA set of store files of a column family are bulk 
loaded.
+   * @param storeFilesSize  Map of store files and their lengths
* @param bulkloadSeqId sequence ID (by a force flush) used to create 
bulk load hfile
*  name
* @return The WAL log marker for bulk loads.
*/
   public static WALProtos.BulkLoadDescriptor toBulkLoadDescriptor(TableName 
tableName,
-  ByteString encodedRegionName, Map> storeFiles, long 
bulkloadSeqId) {
-BulkLoadDescriptor.Builder desc = BulkLoadDescriptor.newBuilder()
+  ByteString encodedRegionName, Map> storeFiles,
+  Map storeFilesSize, long bulkloadSeqId) {
+BulkLoadDescriptor.Builder desc =
+BulkLoadDescriptor.newBuilder()
 .setTableName(ProtobufUtil.toProtoTableName(tableName))
 
.setEncodedRegionName(encodedRegionName).setBulkloadSeqNum(bulkloadSeqId);
 
@@ -3078,7 +3081,10 @@ public final class ProtobufUtil {
   .setFamilyName(ByteStringer.wrap(entry.getKey()))
   .setStoreHomeDir(Bytes.toString(entry.getKey())); // relative to 
region
   for (Path path : entry.getValue()) {
-builder.addStoreFile(path.getName());
+String name = path.getName();
+builder.addStoreFile(name);
+Long size = storeFilesSize.get(name) == null ? (Long) 0L : 
storeFilesSize.get(name);
+builder.setStoreFileSize(size);
   }
   desc.addStores(builder);
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/34e9a6ff/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
--
diff --git 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
index d74688e..6252d51 100644
--- 
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
+++ 
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/WALProtos.java
@@ -7821,6 +7821,24 @@ public final class WALProtos {
  */
 com.google.protobuf.ByteString
 getStoreFileBytes(int index);
+
+// optional uint64 store_file_size = 4;
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+boolean hasStoreFileSize();
+/**
+ * optional uint64 store_file_size = 4;
+ *
+ * 
+ * size of store file
+ * 
+ */
+long getStoreFileSize();
   }
   /**
* Protobuf type {@code hbase.pb.StoreDescriptor}
@@ -7891,6 +7909,11 @@ public final class WALProtos {
   storeFile_.add(input.readBytes());
   break;
 }
+case 32: {
+  bitField0_ |= 0x0004;
+  storeFileSize_ = input.readUInt64();
+