hbase git commit: HBASE-15172 Support setting storage policy in bulkload
Repository: hbase
Updated Branches:
refs/heads/master e02ae7724 -> 629b04f44
HBASE-15172 Support setting storage policy in bulkload
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/629b04f4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/629b04f4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/629b04f4
Branch: refs/heads/master
Commit: 629b04f44f19b9589c9bcfb84da0cf5e0d4d1f18
Parents: e02ae77
Author: Yu Li
Authored: Fri Jan 6 18:35:38 2017 +0800
Committer: Yu Li
Committed: Fri Jan 6 18:35:38 2017 +0800
--
.../hbase/mapreduce/HFileOutputFormat2.java | 31 -
.../hbase/mapreduce/TestHFileOutputFormat2.java | 70
2 files changed, 100 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/629b04f4/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 22a73c9..6987bf7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
@@ -125,6 +126,9 @@ public class HFileOutputFormat2
private static final String OUTPUT_TABLE_NAME_CONF_KEY =
"hbase.mapreduce.hfileoutputformat.table.name";
+ public static final String STORAGE_POLICY_PROPERTY =
"hbase.hstore.storagepolicy";
+ public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX =
STORAGE_POLICY_PROPERTY + ".";
+
@Override
public RecordWriter getRecordWriter(
final TaskAttemptContext context) throws IOException,
InterruptedException {
@@ -230,7 +234,9 @@ public class HFileOutputFormat2
// If this is a new column family, verify that the directory exists
if (wl == null) {
-fs.mkdirs(new Path(outputDir, Bytes.toString(family)));
+Path cfPath = new Path(outputDir, Bytes.toString(family));
+fs.mkdirs(cfPath);
+configureStoragePolicy(conf, fs, family, cfPath);
}
// If any of the HFiles for the column families has reached
@@ -382,6 +388,29 @@ public class HFileOutputFormat2
}
}
+ /**
+ * Configure block storage policy for CF after the directory is created.
+ */
+ static void configureStoragePolicy(final Configuration conf, final
FileSystem fs,
+ byte[] family, Path cfPath) {
+if (null == conf || null == fs || null == family || null == cfPath) {
+ return;
+}
+
+String policy =
+conf.get(STORAGE_POLICY_PROPERTY_CF_PREFIX + Bytes.toString(family),
+ conf.get(STORAGE_POLICY_PROPERTY));
+if (null != policy && !policy.trim().isEmpty()) {
+ try {
+if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).setStoragePolicy(cfPath, policy.trim());
+}
+ } catch (Throwable e) {
+LOG.warn("failed to set block storage policy of [" + cfPath + "] to ["
+ policy + "]", e);
+ }
+}
+ }
+
/*
* Data structure to hold a Writer and amount of data written on it.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/629b04f4/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index 486c961..21a39d4 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -92,6 +93,10 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbas
hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Repository: hbase
Updated Branches:
refs/heads/master 629b04f44 -> 5f631b965
HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f631b96
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f631b96
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f631b96
Branch: refs/heads/master
Commit: 5f631b9653a4bf86a2bebed58abed747c04b704f
Parents: 629b04f
Author: Ashish Singhi
Authored: Fri Jan 6 16:15:49 2017 +0530
Committer: Ashish Singhi
Committed: Fri Jan 6 16:18:20 2017 +0530
--
.../hbase/replication/ReplicationQueues.java| 6 +-
.../replication/ReplicationQueuesZKImpl.java| 11 ++--
.../TableBasedReplicationQueuesImpl.java| 4 +-
.../hbase/regionserver/HRegionServer.java | 4 ++
.../regionserver/HFileReplicator.java | 2 +-
.../replication/regionserver/Replication.java | 55 +++--
.../regionserver/ReplicationObserver.java | 62
.../regionserver/ReplicationSource.java | 11 ++--
.../ReplicationSourceInterface.java | 6 +-
.../regionserver/ReplicationSourceManager.java | 4 +-
.../cleaner/TestReplicationHFileCleaner.java| 9 +--
.../replication/ReplicationSourceDummy.java | 3 +-
.../replication/TestReplicationStateBasic.java | 33 ++-
13 files changed, 140 insertions(+), 70 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 0ae27d0..be5a590 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.replication;
import java.util.List;
import java.util.SortedSet;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
@@ -144,10 +145,11 @@ public interface ReplicationQueues {
/**
* Add new hfile references to the queue.
* @param peerId peer cluster id to which the hfiles need to be replicated
- * @param files list of hfile references to be added
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path
in region dir which
+ * will be added in the queue }
* @throws ReplicationException if fails to add a hfile reference
*/
- void addHFileRefs(String peerId, List files) throws
ReplicationException;
+ void addHFileRefs(String peerId, List> pairs) throws
ReplicationException;
/**
* Remove hfile references from the queue.
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index 7c548d9..1de1315 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -319,16 +320,18 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
}
@Override
- public void addHFileRefs(String peerId, List files) throws
ReplicationException {
+ public void addHFileRefs(String peerId, List> pairs)
+ throws ReplicationException {
String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
boolean debugEnabled = LOG.isDebugEnabled();
if (debugEnabled) {
- LOG.debug("Adding hfile references " + files + " in queue " + peerZnode);
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
}
List listOfOps = new ArrayList();
-int size = files.size();
+int size = pairs.size();
for (int i = 0; i < size; i++
hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Repository: hbase
Updated Branches:
refs/heads/branch-1 667c5eb3a -> e8e40d862
HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8e40d86
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8e40d86
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8e40d86
Branch: refs/heads/branch-1
Commit: e8e40d86258a2f19838f3aea55dafbbc1b860942
Parents: 667c5eb
Author: Ashish Singhi
Authored: Fri Jan 6 16:57:52 2017 +0530
Committer: Ashish Singhi
Committed: Fri Jan 6 16:57:52 2017 +0530
--
.../hbase/replication/ReplicationQueues.java| 6 +-
.../replication/ReplicationQueuesZKImpl.java| 11 ++--
.../hbase/regionserver/HRegionServer.java | 4 ++
.../regionserver/HFileReplicator.java | 2 +-
.../replication/regionserver/Replication.java | 54 +++--
.../regionserver/ReplicationObserver.java | 63
.../regionserver/ReplicationSource.java | 11 ++--
.../ReplicationSourceInterface.java | 6 +-
.../regionserver/ReplicationSourceManager.java | 4 +-
.../cleaner/TestReplicationHFileCleaner.java| 9 +--
.../replication/ReplicationSourceDummy.java | 3 +-
.../replication/TestReplicationStateBasic.java | 33 +-
12 files changed, 138 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8e40d86/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
index 1b1c770..2409111 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.SortedMap;
import java.util.SortedSet;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.Pair;
@@ -151,10 +152,11 @@ public interface ReplicationQueues {
/**
* Add new hfile references to the queue.
* @param peerId peer cluster id to which the hfiles need to be replicated
- * @param files list of hfile references to be added
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path
in region dir which
+ * will be added in the queue }
* @throws ReplicationException if fails to add a hfile reference
*/
- void addHFileRefs(String peerId, List files) throws
ReplicationException;
+ void addHFileRefs(String peerId, List> pairs) throws
ReplicationException;
/**
* Remove hfile references from the queue.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e8e40d86/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index a903159..a1bd829 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
@@ -508,16 +509,18 @@ public class ReplicationQueuesZKImpl extends
ReplicationStateZKBase implements R
}
@Override
- public void addHFileRefs(String peerId, List files) throws
ReplicationException {
+ public void addHFileRefs(String peerId, List> pairs)
+ throws ReplicationException {
String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
boolean debugEnabled = LOG.isDebugEnabled();
if (debugEnabled) {
- LOG.debug("Adding hfile references " + files + " in queue " + peerZnode);
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
}
List listOfOps = new ArrayList();
-int size = files.size();
+int size = pairs.size();
for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.createAndFailSilent(ZKUtil.joinZNode(
hbase git commit: HBASE-17351 Unable to generate tar ball for master branch
Repository: hbase Updated Branches: refs/heads/master 5f631b965 -> 910e885a7 HBASE-17351 Unable to generate tar ball for master branch Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/910e885a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/910e885a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/910e885a Branch: refs/heads/master Commit: 910e885a75e2092e49ae577d1481d58845fffae5 Parents: 5f631b9 Author: Esteban Gutierrez Authored: Tue Dec 20 15:18:28 2016 -0800 Committer: Esteban Gutierrez Committed: Fri Jan 6 08:42:20 2017 -0800 -- pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/910e885a/pom.xml -- diff --git a/pom.xml b/pom.xml index 02320dc..285e358 100644 --- a/pom.xml +++ b/pom.xml @@ -876,6 +876,7 @@ org.apache.maven.plugins maven-enforcer-plugin +1.4 org.codehaus.mojo
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel)
Repository: hbase
Updated Branches:
refs/heads/master 910e885a7 -> 6d48eb06c
HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d48eb06
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d48eb06
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d48eb06
Branch: refs/heads/master
Commit: 6d48eb06c9cd50259e3137eea8e3554d190f678d
Parents: 910e885
Author: tedyu
Authored: Fri Jan 6 09:07:02 2017 -0800
Committer: tedyu
Committed: Fri Jan 6 09:07:02 2017 -0800
--
.../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6d48eb06/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
index b683fcc..f89215b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
@@ -360,7 +360,7 @@ public class PoolMap implements Map {
@Override
public R get() {
- if (super.size() < maxSize) {
+ if (super.size() <= 0) {
return null;
}
nextResource %= super.size();
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel)
Repository: hbase
Updated Branches:
refs/heads/branch-1 e8e40d862 -> d0393bdf2
HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d0393bdf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d0393bdf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d0393bdf
Branch: refs/heads/branch-1
Commit: d0393bdf28860bb1ede57420e54baa748f69b3b6
Parents: e8e40d8
Author: tedyu
Authored: Fri Jan 6 09:08:55 2017 -0800
Committer: tedyu
Committed: Fri Jan 6 09:08:55 2017 -0800
--
.../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d0393bdf/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
index b683fcc..f89215b 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
@@ -360,7 +360,7 @@ public class PoolMap implements Map {
@Override
public R get() {
- if (super.size() < maxSize) {
+ if (super.size() <= 0) {
return null;
}
nextResource %= super.size();
svn commit: r17689 - /release/hbase/KEYS
Author: antonov Date: Fri Jan 6 19:27:40 2017 New Revision: 17689 Log: added Mikhail Antonov to HBase KEYS file Modified: release/hbase/KEYS Modified: release/hbase/KEYS == --- release/hbase/KEYS (original) +++ release/hbase/KEYS Fri Jan 6 19:27:40 2017 @@ -863,3 +863,61 @@ Ow2NCnnFduV9eptWof4mhM+zGMelUHWYl9bpr3po zsw2TSlqnlvmH87wmIyZKQtrDGSJp8qqLx+cfFL3wp7I =3+z7 -END PGP PUBLIC KEY BLOCK- +pub 4096R/35A4ABE2 2016-10-26 +uid Mikhail Antonov (CODE SIGNING KEY) +sig 335A4ABE2 2016-10-26 Mikhail Antonov (CODE SIGNING KEY) +sub 4096R/DCB945A3 2016-10-26 +sig 35A4ABE2 2016-10-26 Mikhail Antonov (CODE SIGNING KEY) + +-BEGIN PGP PUBLIC KEY BLOCK- +Version: GnuPG v1 + +mQINBFgQcI8BEADRuGffsqJi/DciKOF075Ex4xRLkbUe1Ac0pI9NWQnU/eGaRZI8 +r3Np6KiA4bbaKwYbbEfF+4onStnAHHEkOBPUpUDN/oICYZ7BBOMD3jct92lhS2ZS +tSR7KQsM4c1bjJeQE1ZkWyw6j5kNExqQJTA2llt1bIpaIWkU77qn1MDrMZ2aswY9 +GbrtPTNyYE/41xOSVgtSoitLvdbqh/NvdwUfdDxPIE3AkmoRnY5y+fkOKMZHqpcn +/Q3CoDMFyl9Fg2DTID8TocLylcGyCFSR5/LuA2SMYTfVCsnLOUXum/C8nZfyijT3 +TqHpECcViVtkqZvArLZ6vfmdw+tuuEoESc5f5GD/5qlpVthNU6Sx4/K4JWzIa96h +fJp4XejrCN5o/5tSLJ5rBX5GqNTSzKM5fwNSGMpJoOAmScNnwRuA2gcxUOrWaBcq +qXOiMiGXdbszHfIrHy+r/M7GVSqQr0nnBKA4mADeY5NeBEAX/ENcVVj4yZv5xvFx +ASP8mpluQfWGbPlHNfRuFEPj40i+Mci9GlgXQda7O2GNKlS9oTic7jXRjsDw345x +wroM5ekyQqWG50BP1PRbxrnSNgQ2SQxAZyEMmwAwAtmR1a7Jm6GgMpvv83DxUluz +DuG0htNhHAKuwIkSzR1WQvv9AY+XwbiuLYMlYroGcn5ELMAA9+vxBC6KkQARAQAB +tDdNaWtoYWlsIEFudG9ub3YgKENPREUgU0lHTklORyBLRVkpIDxhbnRvbm92QGFw +YWNoZS5vcmc+iQI3BBMBCgAhBQJYEHCPAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4B +AheAAAoJEEpUY3A1pKvisZYP/juGnosfaICeWrMk54kfdLOvzZ1zIMLAr06blIjR +V/bjW2PmebjO6vNFRb+Sl+MWYEu/ZEbWi1aQR2LPYYMzt0oCyJIZTjIB2+fC+AZ4 +uAX0q3squWVsr/n8Edm44KMtcRHsUxkJHcte+iihafT71MPjvZO73Lt0BUHO52mY +sUtoDnvNhalmiJqtwhRZ33Lje0eTL5olmzCbsvJEfLBjs8ThUgVQHzcUiZddpuBh +uWxBcrwI+VH+DhCQlEOSnH0tAju2kNHIczXoggTqTtrQc44h5/j49gn2jyNokNjD +4plo7xv3gUU/aj3+d41EvWRmrbGMYMe7ND+1yFV4ADbqpdHEkU431PE0svFYlr2I +3ndvExBgRhaq/j1BpNoi0+N1BsarBJ9PnCcZNIx2v0A26I2mjkGl2yBOIV68muXI +Rl1JDVHy7P4naDm4niJFnQ3coyL9ybrklzz/Cxvnth2R1YegH8jIix9Rv1/vnjKT +pmb0N0TiNLUYcUsCkw8RSmtnSs1sae7Sz+JfCRYJ9VKGT5UGp7tdSPYaxGIqSLhN +7PSJk1wRW7YFxZr6amorP/zU8iI7ryeqkFpED78ETqBTh3KyELP7EFToN185oOGY +kx9YC7zErpi8wWXUp5WoYGM5w6AXvTCf5RcmRN/cRP5VplMsYnHznCcUTyBXNLqn +AO1muQINBFgQcI8BEACemoaZheD8NPm5JbA8zXfXpsootks+wqxFLLlhnZyXls61 +43ZiTj/72U3kz/XVzIYQu5SmKvrOgDmzgGQTy2cAM9BY92n2F/5UF/yNRgpuGLgh +Sz0/2r+EMA1dRxbPst79M8y6LOIBPncIdb8ESJiDT9JbwHgvTPKyKMky8NDg1RYU +3ghK1gsaJMLo1XRd3gnELajzzzvzS9Dlwwo0g3as1WLRUsWDHua0WT86Ci3+WKKT +iHO7U63AwPsp+aCTX0M0VwIxDovIAMJPCT2UnGcMxM29D+rD4cHTDPY5HqRcWBqv +Qj1nYNsjbmKliOcrz6P1i5or6WjjMg4oufdm/jufcu17CVXrgekwEQarCrIGxnfM +d3QteaKbtE2JuHbU4wWyDIsrezNFXZbLBjuGaH9PdyaPFXs5bOD31iI3DqdFqOd9 +o92Uo1NAKd1snUAtR/QzsNOFgPvgn8Werhesg0Z9SulROcIVwGfA+ayiWYbLVhMC +trtnqpTvP6jfur8fDfZBesKENaPd3YuxpPwt0A49+pr/Ll89ai8q4+HNX8U4d71N +mNfklPS5bv8HbqdzGXXIjVzQYHiIT5IxN1MncJqcUaOjdgQiCiZY8H9AK8KLZFP/ +phNL3fSbUMqtW3YXTRPyrGleWbU3EcX1mqhV8Yhb/gr/+8jUuWFKZ2hbcZvdmQAR +AQABiQIfBBgBCgAJBQJYEHCPAhsMAAoJEEpUY3A1pKvifBUP/jh5RJemGqIujmb7 +QnbIYjmmX960yD2P762DRTQPaCCRpduYdSs3fCUmzCO6TSYE3+/Kqi/vo07mvpJT +InJgL/jiF8lr8Cn0vMw1XTHX9sCxMTxN816UEx82AOJbSZ7cPad5bhRWii5GhVUY +AxDS2e25FGLyiIb3p1jozeS4IUsdtm79KkZo//Jvrk/fcVH7RpgjDLW3O35vsKZi +VaA2oEqxRgyz95Vyf1EkgIGtvgurWZo1Io11E9iUwVlubFJp4FAdD7XauNP745nh +Lh6shebM+etVUV3IWchq71kNNkBoYq2IuISIHLKnz9pO/VGgTfTGxHdocoCJ2eRU +g1n5Cf/AtQpm6Zue4DqSjQkBAQ0PiQ9jNOSF03SOASarDyjbvByImOrE/VHd4Xaf +hq6h6f/5Xff0+ue49MT4NNiOYt+ufu10B9WlwFTu+4fuP1/GkbPDdo8Pr7eELymX +gX2wno+ndB703tymjpYf3aeM+pWv/kQNKLU8JbLdTqU37ygR6nue33I3njELmYpR +osKfzhnmvIJMITUC2i7YBvLkUQQlbmcSoDf1m7D9uxSmzDKnMCggZJYcuXcn+yj5 +rgcTW+5Xy8s6BVuUFgBznn0XrWdqSZ77kp+qPHbYVwJXHJ23PvZxugpEdlxj/ONp +UghJ82SATKRAnx3OcQZQyOV1eB+P +=HL9r +-END PGP PUBLIC KEY BLOCK-
svn commit: r17692 - /dev/hbase/1.3.0RC0/
Author: antonov Date: Fri Jan 6 19:57:41 2017 New Revision: 17692 Log: HBase 1.3 RC0 Added: dev/hbase/1.3.0RC0/ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz (with props) dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz (with props) dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.md5 dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.mds dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.sha Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz == Binary file - no diff available. Propchange: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc Fri Jan 6 19:57:41 2017 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v1 + +iQIcBAABCgAGBQJYb/SuAAoJEEpUY3A1pKviEgMP/1EV9k7eL9KpjwqwmucyYto0 +cLwZObiNlcGS8jIoCvbLVZJRDn2I6pIpJKBFkB3GXPdSvU/nxsVUax3fLjo3tUu8 +ZP2wyURYcVW239rikDQsW0e+Tr+z9DTWz+I5khSeCh5c+h1Ib8Ex01cR7hL++GCG +LTQEal0jtqouSwWXgef0qc486uMiftRyBHEnMn9JY5anD0YgHySXepJmMUODE56y +0AFy3vjww0oqyrJ4DAqpO4gVylocVOq1tYfUZsEu3gQJwttV28FknpTzB5fUmmE2 +loBDLwV+fAcWY/dt5UxlD87Sve8rjnZjRbIn4F3gWEXW6l24Sb8hi9m5nJJPB5+/ +fSH/5Bf/toIv6dnKSG9zYakGqy1dSxkz1YaZN5XyEZ8lhoUOzCWyjm9ard378h2n +9EbdXUbBOmrNx/qtdfLl3raY4cVLPTKmWZGWfeSPY20gn2oFk2NCwKsYfFIVytID +Sp4IQgw50xtD9QvDhQWJ4r8jAdb3jxEWbcUdTN2pJryRMqDKtea9YlU05QRRwM4m ++W6gArkFuzKbFn+3RK8dZt/v3ex+R9zDxMu6rWHRY8n7DRvCQpYITnlfL5TBghrV +oDJog1KX6vaqJdc0TGdczvjMamibVo68mnnlDwQp3jgn6AEjQxFF4FofxTp8fCld +Um09gdo7Vqw+HHeOmtAN +=3bcF +-END PGP SIGNATURE- Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 Fri Jan 6 19:57:41 2017 @@ -0,0 +1 @@ +hbase-1.3.0-bin.tar.gz: DE 9E 68 3C 41 A2 8F D8 DC 9D AD A0 AB 71 CE 2A Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds Fri Jan 6 19:57:41 2017 @@ -0,0 +1,17 @@ +hbase-1.3.0-bin.tar.gz:MD5 = DE 9E 68 3C 41 A2 8F D8 DC 9D AD A0 AB 71 CE + 2A +hbase-1.3.0-bin.tar.gz: SHA1 = 8FED 2108 DB23 A6C5 19BD 6454 CA8C A517 162E + 29B1 +hbase-1.3.0-bin.tar.gz: RMD160 = 03E9 A503 C19D 16F3 74B3 1EE3 F816 68B1 F05A + 2F31 +hbase-1.3.0-bin.tar.gz: SHA224 = B83B77B0 DA220232 92CC0E79 754C4455 D9AEE4E7 + 3277930E EA7F39F9 +hbase-1.3.0-bin.tar.gz: SHA256 = 453CD243 D4DFA1F4 A9D17932 6651F543 47DE1D2A + 761BA4E4 694BFCD4 5A6794EC +hbase-1.3.0-bin.tar.gz: SHA384 = 334EF1B5 14FEE299 6A4E2885 1983C551 CBB30570 + 150DBD95 1F1F77B9 BB9CFFCB 05451717 15D3BA10 + E3794ACB 3F8649C1 +hbase-1.3.0-bin.tar.gz: SHA512 = 1BCB6A76 52C56CE7 D9A6C6BC 3F5A9401 07EDD99D + 3A9568EF B8D24000 7086D81D 6744EB0C F0E8575A + 02ED910B 4BC1D4E9 5A6F4DC0 F9129797 C3DB451C + FA0BC7FD Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha Fri Jan 6 19:57:41 2017 @@ -0,0 +1,3 @@ +hbase-1.3.0-bin.tar.gz: 1BCB6A76 52C56CE7 D9A6C6BC 3F5A9401 07EDD99D 3A9568EF +B8D24000 7086D81D 6744EB0C F0E8575A 02ED910B 4BC1D4E9 +5A6F4DC0 F9129797 C3DB451C FA0BC7FD Added: dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz == Binary file - no diff available. Propchange: dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc == --- dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc Fri Jan 6 19:57:41 2017 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGN
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due to test failure
Repository: hbase
Updated Branches:
refs/heads/master 6d48eb06c -> 4c98f97c3
HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due
to test failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c98f97c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c98f97c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c98f97c
Branch: refs/heads/master
Commit: 4c98f97c318995593bbe8b67ba856003cb5ed463
Parents: 6d48eb0
Author: tedyu
Authored: Fri Jan 6 12:12:49 2017 -0800
Committer: tedyu
Committed: Fri Jan 6 12:12:49 2017 -0800
--
.../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c98f97c/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
index f89215b..b683fcc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
@@ -360,7 +360,7 @@ public class PoolMap implements Map {
@Override
public R get() {
- if (super.size() <= 0) {
+ if (super.size() < maxSize) {
return null;
}
nextResource %= super.size();
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due to test failure
Repository: hbase
Updated Branches:
refs/heads/branch-1 d0393bdf2 -> e371b1e72
HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due
to test failure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e371b1e7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e371b1e7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e371b1e7
Branch: refs/heads/branch-1
Commit: e371b1e72377f1a9cf481cc64294c202d86036ef
Parents: d0393bd
Author: tedyu
Authored: Fri Jan 6 12:13:18 2017 -0800
Committer: tedyu
Committed: Fri Jan 6 12:13:18 2017 -0800
--
.../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e371b1e7/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
index f89215b..b683fcc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java
@@ -360,7 +360,7 @@ public class PoolMap implements Map {
@Override
public R get() {
- if (super.size() <= 0) {
+ if (super.size() < maxSize) {
return null;
}
nextResource %= super.size();
hbase git commit: HBASE-17408 Introduce per request limit by number of mutations (ChiaPing Tsai)
Repository: hbase
Updated Branches:
refs/heads/master 4c98f97c3 -> b2a9be02a
HBASE-17408 Introduce per request limit by number of mutations (ChiaPing Tsai)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b2a9be02
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b2a9be02
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b2a9be02
Branch: refs/heads/master
Commit: b2a9be02ac10908438dc84e9d483bc8785d7ce19
Parents: 4c98f97
Author: tedyu
Authored: Fri Jan 6 13:07:15 2017 -0800
Committer: tedyu
Committed: Fri Jan 6 13:07:15 2017 -0800
--
.../hbase/client/SimpleRequestController.java | 164 +--
.../client/TestSimpleRequestController.java | 91 +++---
.../hbase/coprocessor/RegionServerObserver.java | 2 +-
3 files changed, 182 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b2a9be02/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java
index 6343af6..de2cbe1 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java
@@ -49,30 +49,39 @@ import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
/**
- * Holds back the request if the submitted size or number has reached the
- * threshold.
+ * Holds back the requests if they reach any thresholds.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
class SimpleRequestController implements RequestController {
private static final Log LOG =
LogFactory.getLog(SimpleRequestController.class);
/**
- * The maximum size of single RegionServer.
+ * The maximum heap size for each request.
*/
public static final String HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE =
"hbase.client.max.perrequest.heapsize";
/**
- * Default value of #HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE
+ * Default value of {@link #HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE}.
*/
@VisibleForTesting
static final long DEFAULT_HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE = 4194304;
/**
+ * The maximum number of rows for each request.
+ */
+ public static final String HBASE_CLIENT_MAX_PERREQUEST_ROWS =
"hbase.client.max.perrequest.rows";
+ /**
+ * Default value of {@link #HBASE_CLIENT_MAX_PERREQUEST_ROWS}.
+ */
+ @VisibleForTesting
+ static final long DEFAULT_HBASE_CLIENT_MAX_PERREQUEST_ROWS = 2048;
+
+ /**
* The maximum size of submit.
*/
public static final String HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE =
"hbase.client.max.submit.heapsize";
/**
- * Default value of #HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE
+ * Default value of {@link #HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE}.
*/
@VisibleForTesting
static final long DEFAULT_HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE =
DEFAULT_HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE;
@@ -89,9 +98,13 @@ class SimpleRequestController implements RequestController {
private final int maxTotalConcurrentTasks;
/**
- * The max heap size of all tasks simultaneously executed on a server.
+ * The maximum heap size for each request.
*/
private final long maxHeapSizePerRequest;
+ /**
+ * The maximum number of rows for each request.
+ */
+ private final long maxRowsPerRequest;
private final long maxHeapSizeSubmit;
/**
* The number of tasks we run in parallel on a single region. With 1 (the
@@ -116,41 +129,46 @@ class SimpleRequestController implements
RequestController {
private static final int DEFAULT_THRESHOLD_TO_LOG_REGION_DETAILS = 2;
private final int thresholdToLogRegionDetails;
SimpleRequestController(final Configuration conf) {
-this.maxTotalConcurrentTasks =
conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
+this.maxTotalConcurrentTasks = checkAndGet(conf,
+HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS);
-this.maxConcurrentTasksPerServer =
conf.getInt(HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS,
+this.maxConcurrentTasksPerServer = checkAndGet(conf,
+HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS,
HConstants.DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS);
-this.maxConcurrentTasksPerRegion =
conf.getInt(HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS,
+this.maxConcurrentTasksPerRegion = checkAndGet(conf,
+HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS,
HConstants.DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS);
[2/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb3fe8a2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb3fe8a2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb3fe8a2
Branch: refs/heads/branch-1
Commit: bb3fe8a2fb5ec129f1e30d4c15f011b8a79fed19
Parents: e371b1e
Author: Josh Elser
Authored: Tue Jan 3 19:02:08 2017 -0500
Committer: Josh Elser
Committed: Fri Jan 6 16:46:25 2017 -0500
--
.../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++-
.../hbase/rest/client/TestXmlParsing.java | 76
2 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb3fe8a2/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
--
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index e8845eb..de1e23f 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -26,6 +26,9 @@ import java.io.InterruptedIOException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -209,12 +212,12 @@ public class RemoteAdmin {
try {
return (StorageClusterVersionModel) getUnmarsheller().unmarshal(
- new ByteArrayInputStream(response.getBody()));
+ getInputStream(response));
} catch (JAXBException jaxbe) {
throw new IOException(
"Issue parsing StorageClusterVersionModel object in XML form: "
- + jaxbe.getLocalizedMessage());
+ + jaxbe.getLocalizedMessage(), jaxbe);
}
case 404:
throw new IOException("Cluster version not found");
@@ -398,4 +401,23 @@ public class RemoteAdmin {
throw new IOException("get request to " + path.toString()
+ " request timed out");
}
+
+ /**
+ * Convert the REST server's response to an XML reader.
+ *
+ * @param response The REST server's response.
+ * @return A reader over the parsed XML document.
+ * @throws IOException If the document fails to parse
+ */
+ private XMLStreamReader getInputStream(Response response) throws IOException
{
+try {
+ // Prevent the parser from reading XMl with external entities defined
+ XMLInputFactory xif = XMLInputFactory.newFactory();
+ xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+ xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ return xif.createXMLStreamReader(new
ByteArrayInputStream(response.getBody()));
+} catch (XMLStreamException e) {
+ throw new IOException("Failed to parse XML", e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb3fe8a2/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
--
diff --git
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
new file mode 100644
index 000..56dc05e
--- /dev/null
+++
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import sta
[4/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4f008bb0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4f008bb0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4f008bb0
Branch: refs/heads/branch-1.2
Commit: 4f008bb01d5c7ef745333d37ddbf7a1b04ba1852
Parents: 2e344a7
Author: Josh Elser
Authored: Tue Jan 3 19:02:08 2017 -0500
Committer: Josh Elser
Committed: Fri Jan 6 18:18:24 2017 -0500
--
.../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++-
.../hbase/rest/client/TestXmlParsing.java | 76
2 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4f008bb0/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
--
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index e8845eb..de1e23f 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -26,6 +26,9 @@ import java.io.InterruptedIOException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -209,12 +212,12 @@ public class RemoteAdmin {
try {
return (StorageClusterVersionModel) getUnmarsheller().unmarshal(
- new ByteArrayInputStream(response.getBody()));
+ getInputStream(response));
} catch (JAXBException jaxbe) {
throw new IOException(
"Issue parsing StorageClusterVersionModel object in XML form: "
- + jaxbe.getLocalizedMessage());
+ + jaxbe.getLocalizedMessage(), jaxbe);
}
case 404:
throw new IOException("Cluster version not found");
@@ -398,4 +401,23 @@ public class RemoteAdmin {
throw new IOException("get request to " + path.toString()
+ " request timed out");
}
+
+ /**
+ * Convert the REST server's response to an XML reader.
+ *
+ * @param response The REST server's response.
+ * @return A reader over the parsed XML document.
+ * @throws IOException If the document fails to parse
+ */
+ private XMLStreamReader getInputStream(Response response) throws IOException
{
+try {
+ // Prevent the parser from reading XMl with external entities defined
+ XMLInputFactory xif = XMLInputFactory.newFactory();
+ xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+ xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ return xif.createXMLStreamReader(new
ByteArrayInputStream(response.getBody()));
+} catch (XMLStreamException e) {
+ throw new IOException("Failed to parse XML", e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4f008bb0/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
--
diff --git
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
new file mode 100644
index 000..56dc05e
--- /dev/null
+++
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import s
[5/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ca72bb28
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ca72bb28
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ca72bb28
Branch: refs/heads/branch-1.1
Commit: ca72bb2860bcfe8264e91924a2a3e07fe72238aa
Parents: 3cbc5cc
Author: Josh Elser
Authored: Tue Jan 3 19:02:08 2017 -0500
Committer: Josh Elser
Committed: Fri Jan 6 18:33:04 2017 -0500
--
.../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++-
.../hbase/rest/client/TestXmlParsing.java | 76
2 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ca72bb28/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
--
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index 2809ca9..fc3c9fd 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -26,6 +26,9 @@ import java.io.InterruptedIOException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -209,12 +212,12 @@ public class RemoteAdmin {
try {
return (StorageClusterVersionModel) getUnmarsheller().unmarshal(
- new ByteArrayInputStream(response.getBody()));
+ getInputStream(response));
} catch (JAXBException jaxbe) {
throw new IOException(
"Issue parsing StorageClusterVersionModel object in XML form: "
- + jaxbe.getLocalizedMessage());
+ + jaxbe.getLocalizedMessage(), jaxbe);
}
case 404:
throw new IOException("Cluster version not found");
@@ -398,4 +401,23 @@ public class RemoteAdmin {
throw new IOException("get request to " + path.toString()
+ " request timed out");
}
+
+ /**
+ * Convert the REST server's response to an XML reader.
+ *
+ * @param response The REST server's response.
+ * @return A reader over the parsed XML document.
+ * @throws IOException If the document fails to parse
+ */
+ private XMLStreamReader getInputStream(Response response) throws IOException
{
+try {
+ // Prevent the parser from reading XMl with external entities defined
+ XMLInputFactory xif = XMLInputFactory.newFactory();
+ xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+ xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ return xif.createXMLStreamReader(new
ByteArrayInputStream(response.getBody()));
+} catch (XMLStreamException e) {
+ throw new IOException("Failed to parse XML", e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ca72bb28/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
--
diff --git
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
new file mode 100644
index 000..56dc05e
--- /dev/null
+++
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import s
[3/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77cf73b7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77cf73b7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77cf73b7
Branch: refs/heads/branch-1.3
Commit: 77cf73b758a41accc54af31e1c73e6ad80d0e7c7
Parents: 8f0d0e7
Author: Josh Elser
Authored: Tue Jan 3 19:02:08 2017 -0500
Committer: Josh Elser
Committed: Fri Jan 6 17:45:01 2017 -0500
--
.../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++-
.../hbase/rest/client/TestXmlParsing.java | 76
2 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/77cf73b7/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
--
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index e8845eb..de1e23f 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -26,6 +26,9 @@ import java.io.InterruptedIOException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -209,12 +212,12 @@ public class RemoteAdmin {
try {
return (StorageClusterVersionModel) getUnmarsheller().unmarshal(
- new ByteArrayInputStream(response.getBody()));
+ getInputStream(response));
} catch (JAXBException jaxbe) {
throw new IOException(
"Issue parsing StorageClusterVersionModel object in XML form: "
- + jaxbe.getLocalizedMessage());
+ + jaxbe.getLocalizedMessage(), jaxbe);
}
case 404:
throw new IOException("Cluster version not found");
@@ -398,4 +401,23 @@ public class RemoteAdmin {
throw new IOException("get request to " + path.toString()
+ " request timed out");
}
+
+ /**
+ * Convert the REST server's response to an XML reader.
+ *
+ * @param response The REST server's response.
+ * @return A reader over the parsed XML document.
+ * @throws IOException If the document fails to parse
+ */
+ private XMLStreamReader getInputStream(Response response) throws IOException
{
+try {
+ // Prevent the parser from reading XMl with external entities defined
+ XMLInputFactory xif = XMLInputFactory.newFactory();
+ xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+ xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ return xif.createXMLStreamReader(new
ByteArrayInputStream(response.getBody()));
+} catch (XMLStreamException e) {
+ throw new IOException("Failed to parse XML", e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77cf73b7/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
--
diff --git
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
new file mode 100644
index 000..56dc05e
--- /dev/null
+++
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rest.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import s
[1/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
Repository: hbase
Updated Branches:
refs/heads/branch-1 e371b1e72 -> bb3fe8a2f
refs/heads/branch-1.1 3cbc5cc9e -> ca72bb286
refs/heads/branch-1.2 2e344a7c9 -> 4f008bb01
refs/heads/branch-1.3 8f0d0e78c -> 77cf73b75
refs/heads/master b2a9be02a -> 6fecf55a7
HBASE-17424 Disable external entity parsing in RemoteAdmin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6fecf55a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6fecf55a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6fecf55a
Branch: refs/heads/master
Commit: 6fecf55a7eb05d4f85a17ad9f56d9728f1c5d3e7
Parents: b2a9be0
Author: Josh Elser
Authored: Tue Jan 3 19:02:08 2017 -0500
Committer: Josh Elser
Committed: Fri Jan 6 16:38:50 2017 -0500
--
.../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++-
.../hbase/rest/client/TestXmlParsing.java | 76
2 files changed, 100 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6fecf55a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
--
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
index e8845eb..de1e23f 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java
@@ -26,6 +26,9 @@ import java.io.InterruptedIOException;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -209,12 +212,12 @@ public class RemoteAdmin {
try {
return (StorageClusterVersionModel) getUnmarsheller().unmarshal(
- new ByteArrayInputStream(response.getBody()));
+ getInputStream(response));
} catch (JAXBException jaxbe) {
throw new IOException(
"Issue parsing StorageClusterVersionModel object in XML form: "
- + jaxbe.getLocalizedMessage());
+ + jaxbe.getLocalizedMessage(), jaxbe);
}
case 404:
throw new IOException("Cluster version not found");
@@ -398,4 +401,23 @@ public class RemoteAdmin {
throw new IOException("get request to " + path.toString()
+ " request timed out");
}
+
+ /**
+ * Convert the REST server's response to an XML reader.
+ *
+ * @param response The REST server's response.
+ * @return A reader over the parsed XML document.
+ * @throws IOException If the document fails to parse
+ */
+ private XMLStreamReader getInputStream(Response response) throws IOException
{
+try {
+ // Prevent the parser from reading XMl with external entities defined
+ XMLInputFactory xif = XMLInputFactory.newFactory();
+ xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+ xif.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ return xif.createXMLStreamReader(new
ByteArrayInputStream(response.getBody()));
+} catch (XMLStreamException e) {
+ throw new IOException("Failed to parse XML", e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6fecf55a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
--
diff --git
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
new file mode 100644
index 000..56dc05e
--- /dev/null
+++
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specif
