hbase git commit: HBASE-15172 Support setting storage policy in bulkload
Repository: hbase Updated Branches: refs/heads/master e02ae7724 -> 629b04f44 HBASE-15172 Support setting storage policy in bulkload Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/629b04f4 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/629b04f4 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/629b04f4 Branch: refs/heads/master Commit: 629b04f44f19b9589c9bcfb84da0cf5e0d4d1f18 Parents: e02ae77 Author: Yu Li Authored: Fri Jan 6 18:35:38 2017 +0800 Committer: Yu Li Committed: Fri Jan 6 18:35:38 2017 +0800 -- .../hbase/mapreduce/HFileOutputFormat2.java | 31 - .../hbase/mapreduce/TestHFileOutputFormat2.java | 70 2 files changed, 100 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/629b04f4/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 22a73c9..6987bf7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; @@ -125,6 +126,9 @@ public class HFileOutputFormat2 private static final String OUTPUT_TABLE_NAME_CONF_KEY = "hbase.mapreduce.hfileoutputformat.table.name"; + public static final String STORAGE_POLICY_PROPERTY = "hbase.hstore.storagepolicy"; + public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + "."; + @Override public RecordWriter getRecordWriter( final TaskAttemptContext context) throws IOException, InterruptedException { @@ -230,7 +234,9 @@ public class HFileOutputFormat2 // If this is a new column family, verify that the directory exists if (wl == null) { -fs.mkdirs(new Path(outputDir, Bytes.toString(family))); +Path cfPath = new Path(outputDir, Bytes.toString(family)); +fs.mkdirs(cfPath); +configureStoragePolicy(conf, fs, family, cfPath); } // If any of the HFiles for the column families has reached @@ -382,6 +388,29 @@ public class HFileOutputFormat2 } } + /** + * Configure block storage policy for CF after the directory is created. + */ + static void configureStoragePolicy(final Configuration conf, final FileSystem fs, + byte[] family, Path cfPath) { +if (null == conf || null == fs || null == family || null == cfPath) { + return; +} + +String policy = +conf.get(STORAGE_POLICY_PROPERTY_CF_PREFIX + Bytes.toString(family), + conf.get(STORAGE_POLICY_PROPERTY)); +if (null != policy && !policy.trim().isEmpty()) { + try { +if (fs instanceof DistributedFileSystem) { + ((DistributedFileSystem) fs).setStoragePolicy(cfPath, policy.trim()); +} + } catch (Throwable e) { +LOG.warn("failed to set block storage policy of [" + cfPath + "] to [" + policy + "]", e); + } +} + } + /* * Data structure to hold a Writer and amount of data written on it. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/629b04f4/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java -- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 486c961..21a39d4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -92,6 +93,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbas
hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Repository: hbase Updated Branches: refs/heads/master 629b04f44 -> 5f631b965 HBASE-17290 Potential loss of data for replication of bulk loaded hfiles Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f631b96 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f631b96 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f631b96 Branch: refs/heads/master Commit: 5f631b9653a4bf86a2bebed58abed747c04b704f Parents: 629b04f Author: Ashish Singhi Authored: Fri Jan 6 16:15:49 2017 +0530 Committer: Ashish Singhi Committed: Fri Jan 6 16:18:20 2017 +0530 -- .../hbase/replication/ReplicationQueues.java| 6 +- .../replication/ReplicationQueuesZKImpl.java| 11 ++-- .../TableBasedReplicationQueuesImpl.java| 4 +- .../hbase/regionserver/HRegionServer.java | 4 ++ .../regionserver/HFileReplicator.java | 2 +- .../replication/regionserver/Replication.java | 55 +++-- .../regionserver/ReplicationObserver.java | 62 .../regionserver/ReplicationSource.java | 11 ++-- .../ReplicationSourceInterface.java | 6 +- .../regionserver/ReplicationSourceManager.java | 4 +- .../cleaner/TestReplicationHFileCleaner.java| 9 +-- .../replication/ReplicationSourceDummy.java | 3 +- .../replication/TestReplicationStateBasic.java | 33 ++- 13 files changed, 140 insertions(+), 70 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java index 0ae27d0..be5a590 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.replication; import java.util.List; import java.util.SortedSet; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Pair; @@ -144,10 +145,11 @@ public interface ReplicationQueues { /** * Add new hfile references to the queue. * @param peerId peer cluster id to which the hfiles need to be replicated - * @param files list of hfile references to be added + * @param pairs list of pairs of { HFile location in staging dir, HFile path in region dir which + * will be added in the queue } * @throws ReplicationException if fails to add a hfile reference */ - void addHFileRefs(String peerId, List files) throws ReplicationException; + void addHFileRefs(String peerId, List> pairs) throws ReplicationException; /** * Remove hfile references from the queue. http://git-wip-us.apache.org/repos/asf/hbase/blob/5f631b96/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java index 7c548d9..1de1315 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.exceptions.DeserializationException; @@ -319,16 +320,18 @@ public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements R } @Override - public void addHFileRefs(String peerId, List files) throws ReplicationException { + public void addHFileRefs(String peerId, List> pairs) + throws ReplicationException { String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId); boolean debugEnabled = LOG.isDebugEnabled(); if (debugEnabled) { - LOG.debug("Adding hfile references " + files + " in queue " + peerZnode); + LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode); } List listOfOps = new ArrayList(); -int size = files.size(); +int size = pairs.size(); for (int i = 0; i < size; i++
hbase git commit: HBASE-17290 Potential loss of data for replication of bulk loaded hfiles
Repository: hbase Updated Branches: refs/heads/branch-1 667c5eb3a -> e8e40d862 HBASE-17290 Potential loss of data for replication of bulk loaded hfiles Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e8e40d86 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e8e40d86 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e8e40d86 Branch: refs/heads/branch-1 Commit: e8e40d86258a2f19838f3aea55dafbbc1b860942 Parents: 667c5eb Author: Ashish Singhi Authored: Fri Jan 6 16:57:52 2017 +0530 Committer: Ashish Singhi Committed: Fri Jan 6 16:57:52 2017 +0530 -- .../hbase/replication/ReplicationQueues.java| 6 +- .../replication/ReplicationQueuesZKImpl.java| 11 ++-- .../hbase/regionserver/HRegionServer.java | 4 ++ .../regionserver/HFileReplicator.java | 2 +- .../replication/regionserver/Replication.java | 54 +++-- .../regionserver/ReplicationObserver.java | 63 .../regionserver/ReplicationSource.java | 11 ++-- .../ReplicationSourceInterface.java | 6 +- .../regionserver/ReplicationSourceManager.java | 4 +- .../cleaner/TestReplicationHFileCleaner.java| 9 +-- .../replication/ReplicationSourceDummy.java | 3 +- .../replication/TestReplicationStateBasic.java | 33 +- 12 files changed, 138 insertions(+), 68 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e8e40d86/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java index 1b1c770..2409111 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.SortedMap; import java.util.SortedSet; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Pair; @@ -151,10 +152,11 @@ public interface ReplicationQueues { /** * Add new hfile references to the queue. * @param peerId peer cluster id to which the hfiles need to be replicated - * @param files list of hfile references to be added + * @param pairs list of pairs of { HFile location in staging dir, HFile path in region dir which + * will be added in the queue } * @throws ReplicationException if fails to add a hfile reference */ - void addHFileRefs(String peerId, List files) throws ReplicationException; + void addHFileRefs(String peerId, List> pairs) throws ReplicationException; /** * Remove hfile references from the queue. http://git-wip-us.apache.org/repos/asf/hbase/blob/e8e40d86/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java index a903159..a1bd829 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java @@ -30,6 +30,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.exceptions.DeserializationException; @@ -508,16 +509,18 @@ public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements R } @Override - public void addHFileRefs(String peerId, List files) throws ReplicationException { + public void addHFileRefs(String peerId, List> pairs) + throws ReplicationException { String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId); boolean debugEnabled = LOG.isDebugEnabled(); if (debugEnabled) { - LOG.debug("Adding hfile references " + files + " in queue " + peerZnode); + LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode); } List listOfOps = new ArrayList(); -int size = files.size(); +int size = pairs.size(); for (int i = 0; i < size; i++) { - listOfOps.add(ZKUtilOp.createAndFailSilent(ZKUtil.joinZNode(
hbase git commit: HBASE-17351 Unable to generate tar ball for master branch
Repository: hbase Updated Branches: refs/heads/master 5f631b965 -> 910e885a7 HBASE-17351 Unable to generate tar ball for master branch Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/910e885a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/910e885a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/910e885a Branch: refs/heads/master Commit: 910e885a75e2092e49ae577d1481d58845fffae5 Parents: 5f631b9 Author: Esteban Gutierrez Authored: Tue Dec 20 15:18:28 2016 -0800 Committer: Esteban Gutierrez Committed: Fri Jan 6 08:42:20 2017 -0800 -- pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/910e885a/pom.xml -- diff --git a/pom.xml b/pom.xml index 02320dc..285e358 100644 --- a/pom.xml +++ b/pom.xml @@ -876,6 +876,7 @@ org.apache.maven.plugins maven-enforcer-plugin +1.4 org.codehaus.mojo
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel)
Repository: hbase Updated Branches: refs/heads/master 910e885a7 -> 6d48eb06c HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6d48eb06 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6d48eb06 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6d48eb06 Branch: refs/heads/master Commit: 6d48eb06c9cd50259e3137eea8e3554d190f678d Parents: 910e885 Author: tedyu Authored: Fri Jan 6 09:07:02 2017 -0800 Committer: tedyu Committed: Fri Jan 6 09:07:02 2017 -0800 -- .../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/6d48eb06/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java index b683fcc..f89215b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java @@ -360,7 +360,7 @@ public class PoolMap implements Map { @Override public R get() { - if (super.size() < maxSize) { + if (super.size() <= 0) { return null; } nextResource %= super.size();
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel)
Repository: hbase Updated Branches: refs/heads/branch-1 e8e40d862 -> d0393bdf2 HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() (Jan Hentschel) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d0393bdf Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d0393bdf Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d0393bdf Branch: refs/heads/branch-1 Commit: d0393bdf28860bb1ede57420e54baa748f69b3b6 Parents: e8e40d8 Author: tedyu Authored: Fri Jan 6 09:08:55 2017 -0800 Committer: tedyu Committed: Fri Jan 6 09:08:55 2017 -0800 -- .../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/d0393bdf/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java index b683fcc..f89215b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java @@ -360,7 +360,7 @@ public class PoolMap implements Map { @Override public R get() { - if (super.size() < maxSize) { + if (super.size() <= 0) { return null; } nextResource %= super.size();
svn commit: r17689 - /release/hbase/KEYS
Author: antonov Date: Fri Jan 6 19:27:40 2017 New Revision: 17689 Log: added Mikhail Antonov to HBase KEYS file Modified: release/hbase/KEYS Modified: release/hbase/KEYS == --- release/hbase/KEYS (original) +++ release/hbase/KEYS Fri Jan 6 19:27:40 2017 @@ -863,3 +863,61 @@ Ow2NCnnFduV9eptWof4mhM+zGMelUHWYl9bpr3po zsw2TSlqnlvmH87wmIyZKQtrDGSJp8qqLx+cfFL3wp7I =3+z7 -END PGP PUBLIC KEY BLOCK- +pub 4096R/35A4ABE2 2016-10-26 +uid Mikhail Antonov (CODE SIGNING KEY) +sig 335A4ABE2 2016-10-26 Mikhail Antonov (CODE SIGNING KEY) +sub 4096R/DCB945A3 2016-10-26 +sig 35A4ABE2 2016-10-26 Mikhail Antonov (CODE SIGNING KEY) + +-BEGIN PGP PUBLIC KEY BLOCK- +Version: GnuPG v1 + +mQINBFgQcI8BEADRuGffsqJi/DciKOF075Ex4xRLkbUe1Ac0pI9NWQnU/eGaRZI8 +r3Np6KiA4bbaKwYbbEfF+4onStnAHHEkOBPUpUDN/oICYZ7BBOMD3jct92lhS2ZS +tSR7KQsM4c1bjJeQE1ZkWyw6j5kNExqQJTA2llt1bIpaIWkU77qn1MDrMZ2aswY9 +GbrtPTNyYE/41xOSVgtSoitLvdbqh/NvdwUfdDxPIE3AkmoRnY5y+fkOKMZHqpcn +/Q3CoDMFyl9Fg2DTID8TocLylcGyCFSR5/LuA2SMYTfVCsnLOUXum/C8nZfyijT3 +TqHpECcViVtkqZvArLZ6vfmdw+tuuEoESc5f5GD/5qlpVthNU6Sx4/K4JWzIa96h +fJp4XejrCN5o/5tSLJ5rBX5GqNTSzKM5fwNSGMpJoOAmScNnwRuA2gcxUOrWaBcq +qXOiMiGXdbszHfIrHy+r/M7GVSqQr0nnBKA4mADeY5NeBEAX/ENcVVj4yZv5xvFx +ASP8mpluQfWGbPlHNfRuFEPj40i+Mci9GlgXQda7O2GNKlS9oTic7jXRjsDw345x +wroM5ekyQqWG50BP1PRbxrnSNgQ2SQxAZyEMmwAwAtmR1a7Jm6GgMpvv83DxUluz +DuG0htNhHAKuwIkSzR1WQvv9AY+XwbiuLYMlYroGcn5ELMAA9+vxBC6KkQARAQAB +tDdNaWtoYWlsIEFudG9ub3YgKENPREUgU0lHTklORyBLRVkpIDxhbnRvbm92QGFw +YWNoZS5vcmc+iQI3BBMBCgAhBQJYEHCPAhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4B +AheAAAoJEEpUY3A1pKvisZYP/juGnosfaICeWrMk54kfdLOvzZ1zIMLAr06blIjR +V/bjW2PmebjO6vNFRb+Sl+MWYEu/ZEbWi1aQR2LPYYMzt0oCyJIZTjIB2+fC+AZ4 +uAX0q3squWVsr/n8Edm44KMtcRHsUxkJHcte+iihafT71MPjvZO73Lt0BUHO52mY +sUtoDnvNhalmiJqtwhRZ33Lje0eTL5olmzCbsvJEfLBjs8ThUgVQHzcUiZddpuBh +uWxBcrwI+VH+DhCQlEOSnH0tAju2kNHIczXoggTqTtrQc44h5/j49gn2jyNokNjD +4plo7xv3gUU/aj3+d41EvWRmrbGMYMe7ND+1yFV4ADbqpdHEkU431PE0svFYlr2I +3ndvExBgRhaq/j1BpNoi0+N1BsarBJ9PnCcZNIx2v0A26I2mjkGl2yBOIV68muXI +Rl1JDVHy7P4naDm4niJFnQ3coyL9ybrklzz/Cxvnth2R1YegH8jIix9Rv1/vnjKT +pmb0N0TiNLUYcUsCkw8RSmtnSs1sae7Sz+JfCRYJ9VKGT5UGp7tdSPYaxGIqSLhN +7PSJk1wRW7YFxZr6amorP/zU8iI7ryeqkFpED78ETqBTh3KyELP7EFToN185oOGY +kx9YC7zErpi8wWXUp5WoYGM5w6AXvTCf5RcmRN/cRP5VplMsYnHznCcUTyBXNLqn +AO1muQINBFgQcI8BEACemoaZheD8NPm5JbA8zXfXpsootks+wqxFLLlhnZyXls61 +43ZiTj/72U3kz/XVzIYQu5SmKvrOgDmzgGQTy2cAM9BY92n2F/5UF/yNRgpuGLgh +Sz0/2r+EMA1dRxbPst79M8y6LOIBPncIdb8ESJiDT9JbwHgvTPKyKMky8NDg1RYU +3ghK1gsaJMLo1XRd3gnELajzzzvzS9Dlwwo0g3as1WLRUsWDHua0WT86Ci3+WKKT +iHO7U63AwPsp+aCTX0M0VwIxDovIAMJPCT2UnGcMxM29D+rD4cHTDPY5HqRcWBqv +Qj1nYNsjbmKliOcrz6P1i5or6WjjMg4oufdm/jufcu17CVXrgekwEQarCrIGxnfM +d3QteaKbtE2JuHbU4wWyDIsrezNFXZbLBjuGaH9PdyaPFXs5bOD31iI3DqdFqOd9 +o92Uo1NAKd1snUAtR/QzsNOFgPvgn8Werhesg0Z9SulROcIVwGfA+ayiWYbLVhMC +trtnqpTvP6jfur8fDfZBesKENaPd3YuxpPwt0A49+pr/Ll89ai8q4+HNX8U4d71N +mNfklPS5bv8HbqdzGXXIjVzQYHiIT5IxN1MncJqcUaOjdgQiCiZY8H9AK8KLZFP/ +phNL3fSbUMqtW3YXTRPyrGleWbU3EcX1mqhV8Yhb/gr/+8jUuWFKZ2hbcZvdmQAR +AQABiQIfBBgBCgAJBQJYEHCPAhsMAAoJEEpUY3A1pKvifBUP/jh5RJemGqIujmb7 +QnbIYjmmX960yD2P762DRTQPaCCRpduYdSs3fCUmzCO6TSYE3+/Kqi/vo07mvpJT +InJgL/jiF8lr8Cn0vMw1XTHX9sCxMTxN816UEx82AOJbSZ7cPad5bhRWii5GhVUY +AxDS2e25FGLyiIb3p1jozeS4IUsdtm79KkZo//Jvrk/fcVH7RpgjDLW3O35vsKZi +VaA2oEqxRgyz95Vyf1EkgIGtvgurWZo1Io11E9iUwVlubFJp4FAdD7XauNP745nh +Lh6shebM+etVUV3IWchq71kNNkBoYq2IuISIHLKnz9pO/VGgTfTGxHdocoCJ2eRU +g1n5Cf/AtQpm6Zue4DqSjQkBAQ0PiQ9jNOSF03SOASarDyjbvByImOrE/VHd4Xaf +hq6h6f/5Xff0+ue49MT4NNiOYt+ufu10B9WlwFTu+4fuP1/GkbPDdo8Pr7eELymX +gX2wno+ndB703tymjpYf3aeM+pWv/kQNKLU8JbLdTqU37ygR6nue33I3njELmYpR +osKfzhnmvIJMITUC2i7YBvLkUQQlbmcSoDf1m7D9uxSmzDKnMCggZJYcuXcn+yj5 +rgcTW+5Xy8s6BVuUFgBznn0XrWdqSZ77kp+qPHbYVwJXHJ23PvZxugpEdlxj/ONp +UghJ82SATKRAnx3OcQZQyOV1eB+P +=HL9r +-END PGP PUBLIC KEY BLOCK-
svn commit: r17692 - /dev/hbase/1.3.0RC0/
Author: antonov Date: Fri Jan 6 19:57:41 2017 New Revision: 17692 Log: HBase 1.3 RC0 Added: dev/hbase/1.3.0RC0/ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz (with props) dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz (with props) dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.md5 dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.mds dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.sha Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz == Binary file - no diff available. Propchange: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.asc Fri Jan 6 19:57:41 2017 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v1 + +iQIcBAABCgAGBQJYb/SuAAoJEEpUY3A1pKviEgMP/1EV9k7eL9KpjwqwmucyYto0 +cLwZObiNlcGS8jIoCvbLVZJRDn2I6pIpJKBFkB3GXPdSvU/nxsVUax3fLjo3tUu8 +ZP2wyURYcVW239rikDQsW0e+Tr+z9DTWz+I5khSeCh5c+h1Ib8Ex01cR7hL++GCG +LTQEal0jtqouSwWXgef0qc486uMiftRyBHEnMn9JY5anD0YgHySXepJmMUODE56y +0AFy3vjww0oqyrJ4DAqpO4gVylocVOq1tYfUZsEu3gQJwttV28FknpTzB5fUmmE2 +loBDLwV+fAcWY/dt5UxlD87Sve8rjnZjRbIn4F3gWEXW6l24Sb8hi9m5nJJPB5+/ +fSH/5Bf/toIv6dnKSG9zYakGqy1dSxkz1YaZN5XyEZ8lhoUOzCWyjm9ard378h2n +9EbdXUbBOmrNx/qtdfLl3raY4cVLPTKmWZGWfeSPY20gn2oFk2NCwKsYfFIVytID +Sp4IQgw50xtD9QvDhQWJ4r8jAdb3jxEWbcUdTN2pJryRMqDKtea9YlU05QRRwM4m ++W6gArkFuzKbFn+3RK8dZt/v3ex+R9zDxMu6rWHRY8n7DRvCQpYITnlfL5TBghrV +oDJog1KX6vaqJdc0TGdczvjMamibVo68mnnlDwQp3jgn6AEjQxFF4FofxTp8fCld +Um09gdo7Vqw+HHeOmtAN +=3bcF +-END PGP SIGNATURE- Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.md5 Fri Jan 6 19:57:41 2017 @@ -0,0 +1 @@ +hbase-1.3.0-bin.tar.gz: DE 9E 68 3C 41 A2 8F D8 DC 9D AD A0 AB 71 CE 2A Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.mds Fri Jan 6 19:57:41 2017 @@ -0,0 +1,17 @@ +hbase-1.3.0-bin.tar.gz:MD5 = DE 9E 68 3C 41 A2 8F D8 DC 9D AD A0 AB 71 CE + 2A +hbase-1.3.0-bin.tar.gz: SHA1 = 8FED 2108 DB23 A6C5 19BD 6454 CA8C A517 162E + 29B1 +hbase-1.3.0-bin.tar.gz: RMD160 = 03E9 A503 C19D 16F3 74B3 1EE3 F816 68B1 F05A + 2F31 +hbase-1.3.0-bin.tar.gz: SHA224 = B83B77B0 DA220232 92CC0E79 754C4455 D9AEE4E7 + 3277930E EA7F39F9 +hbase-1.3.0-bin.tar.gz: SHA256 = 453CD243 D4DFA1F4 A9D17932 6651F543 47DE1D2A + 761BA4E4 694BFCD4 5A6794EC +hbase-1.3.0-bin.tar.gz: SHA384 = 334EF1B5 14FEE299 6A4E2885 1983C551 CBB30570 + 150DBD95 1F1F77B9 BB9CFFCB 05451717 15D3BA10 + E3794ACB 3F8649C1 +hbase-1.3.0-bin.tar.gz: SHA512 = 1BCB6A76 52C56CE7 D9A6C6BC 3F5A9401 07EDD99D + 3A9568EF B8D24000 7086D81D 6744EB0C F0E8575A + 02ED910B 4BC1D4E9 5A6F4DC0 F9129797 C3DB451C + FA0BC7FD Added: dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha == --- dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-bin.tar.gz.sha Fri Jan 6 19:57:41 2017 @@ -0,0 +1,3 @@ +hbase-1.3.0-bin.tar.gz: 1BCB6A76 52C56CE7 D9A6C6BC 3F5A9401 07EDD99D 3A9568EF +B8D24000 7086D81D 6744EB0C F0E8575A 02ED910B 4BC1D4E9 +5A6F4DC0 F9129797 C3DB451C FA0BC7FD Added: dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz == Binary file - no diff available. Propchange: dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc == --- dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc (added) +++ dev/hbase/1.3.0RC0/hbase-1.3.0-src.tar.gz.asc Fri Jan 6 19:57:41 2017 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGN
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due to test failure
Repository: hbase Updated Branches: refs/heads/master 6d48eb06c -> 4c98f97c3 HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due to test failure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c98f97c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c98f97c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c98f97c Branch: refs/heads/master Commit: 4c98f97c318995593bbe8b67ba856003cb5ed463 Parents: 6d48eb0 Author: tedyu Authored: Fri Jan 6 12:12:49 2017 -0800 Committer: tedyu Committed: Fri Jan 6 12:12:49 2017 -0800 -- .../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4c98f97c/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java index f89215b..b683fcc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java @@ -360,7 +360,7 @@ public class PoolMap implements Map { @Override public R get() { - if (super.size() <= 0) { + if (super.size() < maxSize) { return null; } nextResource %= super.size();
hbase git commit: HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due to test failure
Repository: hbase Updated Branches: refs/heads/branch-1 d0393bdf2 -> e371b1e72 HBASE-17431 Incorrect precheck condition in RoundRobinPool#get() - revert due to test failure Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e371b1e7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e371b1e7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e371b1e7 Branch: refs/heads/branch-1 Commit: e371b1e72377f1a9cf481cc64294c202d86036ef Parents: d0393bd Author: tedyu Authored: Fri Jan 6 12:13:18 2017 -0800 Committer: tedyu Committed: Fri Jan 6 12:13:18 2017 -0800 -- .../src/main/java/org/apache/hadoop/hbase/util/PoolMap.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/e371b1e7/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java index f89215b..b683fcc 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/PoolMap.java @@ -360,7 +360,7 @@ public class PoolMap implements Map { @Override public R get() { - if (super.size() <= 0) { + if (super.size() < maxSize) { return null; } nextResource %= super.size();
hbase git commit: HBASE-17408 Introduce per request limit by number of mutations (ChiaPing Tsai)
Repository: hbase Updated Branches: refs/heads/master 4c98f97c3 -> b2a9be02a HBASE-17408 Introduce per request limit by number of mutations (ChiaPing Tsai) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b2a9be02 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b2a9be02 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b2a9be02 Branch: refs/heads/master Commit: b2a9be02ac10908438dc84e9d483bc8785d7ce19 Parents: 4c98f97 Author: tedyu Authored: Fri Jan 6 13:07:15 2017 -0800 Committer: tedyu Committed: Fri Jan 6 13:07:15 2017 -0800 -- .../hbase/client/SimpleRequestController.java | 164 +-- .../client/TestSimpleRequestController.java | 91 +++--- .../hbase/coprocessor/RegionServerObserver.java | 2 +- 3 files changed, 182 insertions(+), 75 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/b2a9be02/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java -- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java index 6343af6..de2cbe1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/SimpleRequestController.java @@ -49,30 +49,39 @@ import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; /** - * Holds back the request if the submitted size or number has reached the - * threshold. + * Holds back the requests if they reach any thresholds. */ @InterfaceAudience.Private @InterfaceStability.Evolving class SimpleRequestController implements RequestController { private static final Log LOG = LogFactory.getLog(SimpleRequestController.class); /** - * The maximum size of single RegionServer. + * The maximum heap size for each request. */ public static final String HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE = "hbase.client.max.perrequest.heapsize"; /** - * Default value of #HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE + * Default value of {@link #HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE}. */ @VisibleForTesting static final long DEFAULT_HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE = 4194304; /** + * The maximum number of rows for each request. + */ + public static final String HBASE_CLIENT_MAX_PERREQUEST_ROWS = "hbase.client.max.perrequest.rows"; + /** + * Default value of {@link #HBASE_CLIENT_MAX_PERREQUEST_ROWS}. + */ + @VisibleForTesting + static final long DEFAULT_HBASE_CLIENT_MAX_PERREQUEST_ROWS = 2048; + + /** * The maximum size of submit. */ public static final String HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE = "hbase.client.max.submit.heapsize"; /** - * Default value of #HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE + * Default value of {@link #HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE}. */ @VisibleForTesting static final long DEFAULT_HBASE_CLIENT_MAX_SUBMIT_HEAPSIZE = DEFAULT_HBASE_CLIENT_MAX_PERREQUEST_HEAPSIZE; @@ -89,9 +98,13 @@ class SimpleRequestController implements RequestController { private final int maxTotalConcurrentTasks; /** - * The max heap size of all tasks simultaneously executed on a server. + * The maximum heap size for each request. */ private final long maxHeapSizePerRequest; + /** + * The maximum number of rows for each request. + */ + private final long maxRowsPerRequest; private final long maxHeapSizeSubmit; /** * The number of tasks we run in parallel on a single region. With 1 (the @@ -116,41 +129,46 @@ class SimpleRequestController implements RequestController { private static final int DEFAULT_THRESHOLD_TO_LOG_REGION_DETAILS = 2; private final int thresholdToLogRegionDetails; SimpleRequestController(final Configuration conf) { -this.maxTotalConcurrentTasks = conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, +this.maxTotalConcurrentTasks = checkAndGet(conf, +HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS); -this.maxConcurrentTasksPerServer = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS, +this.maxConcurrentTasksPerServer = checkAndGet(conf, +HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS); -this.maxConcurrentTasksPerRegion = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS, +this.maxConcurrentTasksPerRegion = checkAndGet(conf, +HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS, HConstants.DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS);
[2/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb3fe8a2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb3fe8a2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb3fe8a2 Branch: refs/heads/branch-1 Commit: bb3fe8a2fb5ec129f1e30d4c15f011b8a79fed19 Parents: e371b1e Author: Josh Elser Authored: Tue Jan 3 19:02:08 2017 -0500 Committer: Josh Elser Committed: Fri Jan 6 16:46:25 2017 -0500 -- .../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++- .../hbase/rest/client/TestXmlParsing.java | 76 2 files changed, 100 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/bb3fe8a2/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java index e8845eb..de1e23f 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java @@ -26,6 +26,9 @@ import java.io.InterruptedIOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -209,12 +212,12 @@ public class RemoteAdmin { try { return (StorageClusterVersionModel) getUnmarsheller().unmarshal( - new ByteArrayInputStream(response.getBody())); + getInputStream(response)); } catch (JAXBException jaxbe) { throw new IOException( "Issue parsing StorageClusterVersionModel object in XML form: " - + jaxbe.getLocalizedMessage()); + + jaxbe.getLocalizedMessage(), jaxbe); } case 404: throw new IOException("Cluster version not found"); @@ -398,4 +401,23 @@ public class RemoteAdmin { throw new IOException("get request to " + path.toString() + " request timed out"); } + + /** + * Convert the REST server's response to an XML reader. + * + * @param response The REST server's response. + * @return A reader over the parsed XML document. + * @throws IOException If the document fails to parse + */ + private XMLStreamReader getInputStream(Response response) throws IOException { +try { + // Prevent the parser from reading XMl with external entities defined + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLStreamReader(new ByteArrayInputStream(response.getBody())); +} catch (XMLStreamException e) { + throw new IOException("Failed to parse XML", e); +} + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/bb3fe8a2/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java -- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java new file mode 100644 index 000..56dc05e --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import sta
[4/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4f008bb0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4f008bb0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4f008bb0 Branch: refs/heads/branch-1.2 Commit: 4f008bb01d5c7ef745333d37ddbf7a1b04ba1852 Parents: 2e344a7 Author: Josh Elser Authored: Tue Jan 3 19:02:08 2017 -0500 Committer: Josh Elser Committed: Fri Jan 6 18:18:24 2017 -0500 -- .../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++- .../hbase/rest/client/TestXmlParsing.java | 76 2 files changed, 100 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/4f008bb0/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java index e8845eb..de1e23f 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java @@ -26,6 +26,9 @@ import java.io.InterruptedIOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -209,12 +212,12 @@ public class RemoteAdmin { try { return (StorageClusterVersionModel) getUnmarsheller().unmarshal( - new ByteArrayInputStream(response.getBody())); + getInputStream(response)); } catch (JAXBException jaxbe) { throw new IOException( "Issue parsing StorageClusterVersionModel object in XML form: " - + jaxbe.getLocalizedMessage()); + + jaxbe.getLocalizedMessage(), jaxbe); } case 404: throw new IOException("Cluster version not found"); @@ -398,4 +401,23 @@ public class RemoteAdmin { throw new IOException("get request to " + path.toString() + " request timed out"); } + + /** + * Convert the REST server's response to an XML reader. + * + * @param response The REST server's response. + * @return A reader over the parsed XML document. + * @throws IOException If the document fails to parse + */ + private XMLStreamReader getInputStream(Response response) throws IOException { +try { + // Prevent the parser from reading XMl with external entities defined + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLStreamReader(new ByteArrayInputStream(response.getBody())); +} catch (XMLStreamException e) { + throw new IOException("Failed to parse XML", e); +} + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/4f008bb0/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java -- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java new file mode 100644 index 000..56dc05e --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import s
[5/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ca72bb28 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ca72bb28 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ca72bb28 Branch: refs/heads/branch-1.1 Commit: ca72bb2860bcfe8264e91924a2a3e07fe72238aa Parents: 3cbc5cc Author: Josh Elser Authored: Tue Jan 3 19:02:08 2017 -0500 Committer: Josh Elser Committed: Fri Jan 6 18:33:04 2017 -0500 -- .../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++- .../hbase/rest/client/TestXmlParsing.java | 76 2 files changed, 100 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/ca72bb28/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java index 2809ca9..fc3c9fd 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java @@ -26,6 +26,9 @@ import java.io.InterruptedIOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -209,12 +212,12 @@ public class RemoteAdmin { try { return (StorageClusterVersionModel) getUnmarsheller().unmarshal( - new ByteArrayInputStream(response.getBody())); + getInputStream(response)); } catch (JAXBException jaxbe) { throw new IOException( "Issue parsing StorageClusterVersionModel object in XML form: " - + jaxbe.getLocalizedMessage()); + + jaxbe.getLocalizedMessage(), jaxbe); } case 404: throw new IOException("Cluster version not found"); @@ -398,4 +401,23 @@ public class RemoteAdmin { throw new IOException("get request to " + path.toString() + " request timed out"); } + + /** + * Convert the REST server's response to an XML reader. + * + * @param response The REST server's response. + * @return A reader over the parsed XML document. + * @throws IOException If the document fails to parse + */ + private XMLStreamReader getInputStream(Response response) throws IOException { +try { + // Prevent the parser from reading XMl with external entities defined + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLStreamReader(new ByteArrayInputStream(response.getBody())); +} catch (XMLStreamException e) { + throw new IOException("Failed to parse XML", e); +} + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/ca72bb28/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java -- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java new file mode 100644 index 000..56dc05e --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import s
[3/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
HBASE-17424 Disable external entity parsing in RemoteAdmin Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77cf73b7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77cf73b7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77cf73b7 Branch: refs/heads/branch-1.3 Commit: 77cf73b758a41accc54af31e1c73e6ad80d0e7c7 Parents: 8f0d0e7 Author: Josh Elser Authored: Tue Jan 3 19:02:08 2017 -0500 Committer: Josh Elser Committed: Fri Jan 6 17:45:01 2017 -0500 -- .../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++- .../hbase/rest/client/TestXmlParsing.java | 76 2 files changed, 100 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/77cf73b7/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java index e8845eb..de1e23f 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java @@ -26,6 +26,9 @@ import java.io.InterruptedIOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -209,12 +212,12 @@ public class RemoteAdmin { try { return (StorageClusterVersionModel) getUnmarsheller().unmarshal( - new ByteArrayInputStream(response.getBody())); + getInputStream(response)); } catch (JAXBException jaxbe) { throw new IOException( "Issue parsing StorageClusterVersionModel object in XML form: " - + jaxbe.getLocalizedMessage()); + + jaxbe.getLocalizedMessage(), jaxbe); } case 404: throw new IOException("Cluster version not found"); @@ -398,4 +401,23 @@ public class RemoteAdmin { throw new IOException("get request to " + path.toString() + " request timed out"); } + + /** + * Convert the REST server's response to an XML reader. + * + * @param response The REST server's response. + * @return A reader over the parsed XML document. + * @throws IOException If the document fails to parse + */ + private XMLStreamReader getInputStream(Response response) throws IOException { +try { + // Prevent the parser from reading XMl with external entities defined + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLStreamReader(new ByteArrayInputStream(response.getBody())); +} catch (XMLStreamException e) { + throw new IOException("Failed to parse XML", e); +} + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/77cf73b7/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java -- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java new file mode 100644 index 000..56dc05e --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rest.client; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import s
[1/5] hbase git commit: HBASE-17424 Disable external entity parsing in RemoteAdmin
Repository: hbase Updated Branches: refs/heads/branch-1 e371b1e72 -> bb3fe8a2f refs/heads/branch-1.1 3cbc5cc9e -> ca72bb286 refs/heads/branch-1.2 2e344a7c9 -> 4f008bb01 refs/heads/branch-1.3 8f0d0e78c -> 77cf73b75 refs/heads/master b2a9be02a -> 6fecf55a7 HBASE-17424 Disable external entity parsing in RemoteAdmin Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6fecf55a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6fecf55a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6fecf55a Branch: refs/heads/master Commit: 6fecf55a7eb05d4f85a17ad9f56d9728f1c5d3e7 Parents: b2a9be0 Author: Josh Elser Authored: Tue Jan 3 19:02:08 2017 -0500 Committer: Josh Elser Committed: Fri Jan 6 16:38:50 2017 -0500 -- .../hadoop/hbase/rest/client/RemoteAdmin.java | 26 ++- .../hbase/rest/client/TestXmlParsing.java | 76 2 files changed, 100 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/6fecf55a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java -- diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java index e8845eb..de1e23f 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteAdmin.java @@ -26,6 +26,9 @@ import java.io.InterruptedIOException; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -209,12 +212,12 @@ public class RemoteAdmin { try { return (StorageClusterVersionModel) getUnmarsheller().unmarshal( - new ByteArrayInputStream(response.getBody())); + getInputStream(response)); } catch (JAXBException jaxbe) { throw new IOException( "Issue parsing StorageClusterVersionModel object in XML form: " - + jaxbe.getLocalizedMessage()); + + jaxbe.getLocalizedMessage(), jaxbe); } case 404: throw new IOException("Cluster version not found"); @@ -398,4 +401,23 @@ public class RemoteAdmin { throw new IOException("get request to " + path.toString() + " request timed out"); } + + /** + * Convert the REST server's response to an XML reader. + * + * @param response The REST server's response. + * @return A reader over the parsed XML document. + * @throws IOException If the document fails to parse + */ + private XMLStreamReader getInputStream(Response response) throws IOException { +try { + // Prevent the parser from reading XMl with external entities defined + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLStreamReader(new ByteArrayInputStream(response.getBody())); +} catch (XMLStreamException e) { + throw new IOException("Failed to parse XML", e); +} + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/6fecf55a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java -- diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java new file mode 100644 index 000..56dc05e --- /dev/null +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestXmlParsing.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specif