[11/16] hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f631b96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f631b96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f631b96
Branch: refs/heads/hbase-12439
Commit: 5f631b9653a4bf86a2bebed58abed747c04b704f
Parents: 629b04f
Author: Ashish Singhi
Authored: Fri Jan 6 16:15:49 2017 +0530
Committer: Ashish Singhi
Committed: Fri Jan 6 16:18:20 2017 +0530
--
.../hbase/replication/ReplicationQueues.java| 6 +-
.../replication/ReplicationQueuesZKImpl.java| 11 ++--
.../TableBasedReplicationQueuesImpl.java| 4 +-
.../hbase/regionserver/HRegionServer.java | 4 ++
.../regionserver/HFileReplicator.java | 2 +-
.../replication/regionserver/Replication.java | 55 +++--
.../regionserver/ReplicationObserver.java | 62
.../regionserver/ReplicationSource.java | 11 ++--
.../ReplicationSourceInterface.java | 6 +-
.../regionserver/ReplicationSourceManager.java | 4 +-
.../cleaner/TestReplicationHFileCleaner.java| 9 +--
.../replication/ReplicationSourceDummy.java | 3 +-
.../replication/TestReplicationStateBasic.java | 33 ++-
13 files changed, 140 insertions(+), 70 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0ae27d0..be5a590 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.replication;
import java.util.List;
import java.util.SortedSet;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
@@ -144,10 +145,11 @@ public interface ReplicationQueues {
/**
* Add new hfile references to the queue.
* @param peerId peer cluster id to which the hfiles need to be replicated
- * @param files list of hfile references to be added
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path
in region dir which
+ * will be added in the queue }
* @throws ReplicationException if fails to add a hfile reference
*/
- void addHFileRefs(String peerId, List files) throws
ReplicationException;
+ void addHFileRefs(String peerId, List> pairs) throws
ReplicationException;
/**
* Remove hfile references from the queue.
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index 7c548d9..1de1315 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -319,16 +320,18 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
}
@Override
- public void addHFileRefs(String peerId, List files) throws
ReplicationException {
+ public void addHFileRefs(String peerId, List> pairs)
+ throws ReplicationException {
String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
boolean debugEnabled = LOG.isDebugEnabled();
if (debugEnabled) {
- LOG.debug("Adding hfile references " + files + " in queue " + peerZnode);
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
}
List listOfOps = new ArrayList();
-int size = files.size();
+int size = pairs.size();
for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.createAndFailSilent(ZKUtil.joinZNode(peerZ
hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Repository: hbase
Updated Branches:
refs/heads/branch-1 667c5eb3a -> e8e40d862
HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8e40d86
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8e40d86
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8e40d86
Branch: refs/heads/branch-1
Commit: e8e40d86258a2f19838f3aea55dafbbc1b860942
Parents: 667c5eb
Author: Ashish Singhi
Authored: Fri Jan 6 16:57:52 2017 +0530
Committer: Ashish Singhi
Committed: Fri Jan 6 16:57:52 2017 +0530
--
.../hbase/replication/ReplicationQueues.java| 6 +-
.../replication/ReplicationQueuesZKImpl.java| 11 ++--
.../hbase/regionserver/HRegionServer.java | 4 ++
.../regionserver/HFileReplicator.java | 2 +-
.../replication/regionserver/Replication.java | 54 +++--
.../regionserver/ReplicationObserver.java | 63
.../regionserver/ReplicationSource.java | 11 ++--
.../ReplicationSourceInterface.java | 6 +-
.../regionserver/ReplicationSourceManager.java | 4 +-
.../cleaner/TestReplicationHFileCleaner.java| 9 +--
.../replication/ReplicationSourceDummy.java | 3 +-
.../replication/TestReplicationStateBasic.java | 33 +-
12 files changed, 138 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8e40d86/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 1b1c770..2409111 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.SortedMap;
import java.util.SortedSet;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
@@ -151,10 +152,11 @@ public interface ReplicationQueues {
/**
* Add new hfile references to the queue.
* @param peerId peer cluster id to which the hfiles need to be replicated
- * @param files list of hfile references to be added
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path
in region dir which
+ * will be added in the queue }
* @throws ReplicationException if fails to add a hfile reference
*/
- void addHFileRefs(String peerId, List files) throws
ReplicationException;
+ void addHFileRefs(String peerId, List> pairs) throws
ReplicationException;
/**
* Remove hfile references from the queue.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8e40d86/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index a903159..a1bd829 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -508,16 +509,18 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
}
@Override
- public void addHFileRefs(String peerId, List files) throws
ReplicationException {
+ public void addHFileRefs(String peerId, List> pairs)
+ throws ReplicationException {
String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
boolean debugEnabled = LOG.isDebugEnabled();
if (debugEnabled) {
- LOG.debug("Adding hfile references " + files + " in queue " + peerZnode);
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
}
List listOfOps = new ArrayList();
-int size = files.size();
+int size = pairs.size();
for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.createAndFailSilent(ZKUtil.joinZNode(
hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Repository: hbase
Updated Branches:
refs/heads/master 629b04f44 -> 5f631b965
HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f631b96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f631b96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f631b96
Branch: refs/heads/master
Commit: 5f631b9653a4bf86a2bebed58abed747c04b704f
Parents: 629b04f
Author: Ashish Singhi
Authored: Fri Jan 6 16:15:49 2017 +0530
Committer: Ashish Singhi
Committed: Fri Jan 6 16:18:20 2017 +0530
--
.../hbase/replication/ReplicationQueues.java| 6 +-
.../replication/ReplicationQueuesZKImpl.java| 11 ++--
.../TableBasedReplicationQueuesImpl.java| 4 +-
.../hbase/regionserver/HRegionServer.java | 4 ++
.../regionserver/HFileReplicator.java | 2 +-
.../replication/regionserver/Replication.java | 55 +++--
.../regionserver/ReplicationObserver.java | 62
.../regionserver/ReplicationSource.java | 11 ++--
.../ReplicationSourceInterface.java | 6 +-
.../regionserver/ReplicationSourceManager.java | 4 +-
.../cleaner/TestReplicationHFileCleaner.java| 9 +--
.../replication/ReplicationSourceDummy.java | 3 +-
.../replication/TestReplicationStateBasic.java | 33 ++-
13 files changed, 140 insertions(+), 70 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0ae27d0..be5a590 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.replication;
import java.util.List;
import java.util.SortedSet;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
@@ -144,10 +145,11 @@ public interface ReplicationQueues {
/**
* Add new hfile references to the queue.
* @param peerId peer cluster id to which the hfiles need to be replicated
- * @param files list of hfile references to be added
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path
in region dir which
+ * will be added in the queue }
* @throws ReplicationException if fails to add a hfile reference
*/
- void addHFileRefs(String peerId, List files) throws
ReplicationException;
+ void addHFileRefs(String peerId, List> pairs) throws
ReplicationException;
/**
* Remove hfile references from the queue.
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index 7c548d9..1de1315 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -319,16 +320,18 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
}
@Override
- public void addHFileRefs(String peerId, List files) throws
ReplicationException {
+ public void addHFileRefs(String peerId, List> pairs)
+ throws ReplicationException {
String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
boolean debugEnabled = LOG.isDebugEnabled();
if (debugEnabled) {
- LOG.debug("Adding hfile references " + files + " in queue " + peerZnode);
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
}
List listOfOps = new ArrayList();
-int size = files.size();
+int size = pairs.size();
for (int i = 0; i < size; i++
