Author: daijy Date: Wed May 13 22:43:18 2015 New Revision: 1679288 URL: http://svn.apache.org/r1679288 Log: PIG-4544: Upgrade Hbase to 0.98.12
Modified: pig/branches/branch-0.15/CHANGES.txt pig/branches/branch-0.15/ivy.xml pig/branches/branch-0.15/ivy/libraries.properties pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java pig/branches/branch-0.15/test/org/apache/pig/test/TestHBaseStorage.java Modified: pig/branches/branch-0.15/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/CHANGES.txt?rev=1679288&r1=1679287&r2=1679288&view=diff ============================================================================== --- pig/branches/branch-0.15/CHANGES.txt (original) +++ pig/branches/branch-0.15/CHANGES.txt Wed May 13 22:43:18 2015 @@ -66,6 +66,8 @@ PIG-4333: Split BigData tests into multi BUG FIXES +PIG-4544: Upgrade Hbase to 0.98.12 (daijy) + PIG-4481: e2e tests ComputeSpec_1, ComputeSpec_2 and StreamingPerformance_3 produce different result on Windows (daijy) PIG-4496: Fix CBZip2InputStream to close underlying stream (petersla via daijy) Modified: pig/branches/branch-0.15/ivy.xml URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/ivy.xml?rev=1679288&r1=1679287&r2=1679288&view=diff ============================================================================== --- pig/branches/branch-0.15/ivy.xml (original) +++ pig/branches/branch-0.15/ivy.xml Wed May 13 22:43:18 2015 @@ -187,12 +187,14 @@ conf="hadoop20->default;checkstyle->master"> <exclude org="org.codehaus.jackson" module="jackson-core-asl"/> <exclude org="org.codehaus.jackson" module="jackson-mapper-asl"/> + <exclude org="io.netty" module="netty"/> </dependency> <dependency org="org.apache.avro" name="avro-mapred" rev="${avro.version}" conf="hadoop23->default;checkstyle->master"> <artifact name="avro-mapred" type="jar" m:classifier="hadoop2"/> <exclude org="org.codehaus.jackson" module="jackson-core-asl"/> <exclude org="org.codehaus.jackson" module="jackson-mapper-asl"/> + <exclude org="io.netty" module="netty"/> </dependency> <dependency org="org.apache.avro" name="trevni-core" rev="${avro.version}" conf="compile->default;checkstyle->master"> @@ -247,7 +249,7 @@ <dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/> <dependency org="org.antlr" name="ST4" rev="${stringtemplate.version}" conf="compile->default"/> <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master"/> - <dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="test->master"/> + <dependency org="io.netty" name="netty" rev="${netty.version}" conf="test->master"/> <dependency org="dk.brics.automaton" name="automaton" rev="1.11-8" conf="compile->default"/> <dependency org="org.jruby" name="jruby-complete" rev="${jruby.version}" conf="compile->master"/> @@ -352,12 +354,14 @@ <exclude org="asm" module="asm"/> </dependency> - <dependency org="org.htrace" name="htrace-core" rev="${htrace.version}" conf="hadoop23->master"/> + <dependency org="org.htrace" name="htrace-core" rev="3.0.4" conf="hadoop23->master"/> + <dependency org="org.apache.htrace" name="htrace-core" rev="${htrace.version}" conf="hadoop23->master"/> <dependency org="org.fusesource.leveldbjni" name="leveldbjni-all" rev="${leveldbjni.version}" conf="hadoop23->master"/> <dependency org="org.cloudera.htrace" name="htrace-core" rev="2.00" conf="hbase95->master"> <artifact name="htrace-core" type="jar"/> </dependency> + <dependency org="com.lmax" name="disruptor" rev="3.3.0" conf="hbase95->master"/> <!-- for TestHBaseStorage --> <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}" Modified: pig/branches/branch-0.15/ivy/libraries.properties URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/ivy/libraries.properties?rev=1679288&r1=1679287&r2=1679288&view=diff ============================================================================== --- pig/branches/branch-0.15/ivy/libraries.properties (original) +++ pig/branches/branch-0.15/ivy/libraries.properties Wed May 13 22:43:18 2015 @@ -45,7 +45,7 @@ hadoop-common.version=2.6.0 hadoop-hdfs.version=2.6.0 hadoop-mapreduce.version=2.6.0 hbase94.version=0.94.1 -hbase95.version=0.96.0-${hbase.hadoop.version} +hbase95.version=0.98.12-${hbase.hadoop.version} hsqldb.version=1.8.0.10 hive.version=0.14.0 httpcomponents.version=4.1 @@ -71,7 +71,7 @@ rhino.version=1.7R2 antlr.version=3.4 stringtemplate.version=4.0.4 log4j.version=1.2.16 -netty.version=3.2.2 +netty.version=3.6.6.Final rats-lib.version=0.5.1 slf4j-api.version=1.6.1 slf4j-log4j12.version=1.6.1 @@ -97,4 +97,4 @@ parquet-pig-bundle.version=1.2.3 snappy.version=0.2 leveldbjni.version=1.8 curator.version=2.6.0 -htrace.version=3.0.4 +htrace.version=3.1.0-incubating Modified: pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java?rev=1679288&r1=1679287&r2=1679288&view=diff ============================================================================== --- pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java (original) +++ pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java Wed May 13 22:43:18 2015 @@ -1046,7 +1046,8 @@ public class HBaseStorage extends LoadFu * @throws IOException */ public Delete createDelete(Object key, byte type, long timestamp) throws IOException { - Delete delete = new Delete(objToBytes(key, type), timestamp); + Delete delete = new Delete(objToBytes(key, type)); + delete.setTimestamp(timestamp); if(noWAL_) { delete.setWriteToWAL(false); Modified: pig/branches/branch-0.15/test/org/apache/pig/test/TestHBaseStorage.java URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/test/org/apache/pig/test/TestHBaseStorage.java?rev=1679288&r1=1679287&r2=1679288&view=diff ============================================================================== --- pig/branches/branch-0.15/test/org/apache/pig/test/TestHBaseStorage.java (original) +++ pig/branches/branch-0.15/test/org/apache/pig/test/TestHBaseStorage.java Wed May 13 22:43:18 2015 @@ -1272,13 +1272,27 @@ public class TestHBaseStorage { * @throws ParseException */ @Test - public void testNoWAL() throws IOException, ParseException { + public void testNoWAL() throws Exception { HBaseStorage hbaseStorage = new HBaseStorage(TESTCOLUMN_A, "-noWAL"); Object key = "somekey"; byte type = DataType.CHARARRAY; - Assert.assertFalse(hbaseStorage.createPut(key, type).getWriteToWAL()); - Assert.assertFalse(hbaseStorage.createDelete(key, type, System.currentTimeMillis()).getWriteToWAL()); + Put put = hbaseStorage.createPut(key, type); + Delete delete = hbaseStorage.createDelete(key, type, System.currentTimeMillis()); + boolean hasDurabilityMethod = false; + try { + put.getClass().getMethod("getDurability"); + hasDurabilityMethod = true; + } catch (NoSuchMethodException e) { + } + if (hasDurabilityMethod) { // Hbase version 0.95+ + Object skipWal = Class.forName("org.apache.hadoop.hbase.client.Durability").getField("SKIP_WAL").get(put); + Assert.assertEquals(put.getClass().getMethod("getDurability").invoke(put), skipWal); + Assert.assertEquals(delete.getClass().getMethod("getDurability").invoke(delete), skipWal); + } else { + Assert.assertFalse(put.getWriteToWAL()); + Assert.assertFalse(delete.getWriteToWAL()); + } } /** @@ -1287,13 +1301,27 @@ public class TestHBaseStorage { * @throws ParseException */ @Test - public void testWIthWAL() throws IOException, ParseException { + public void testWIthWAL() throws Exception { HBaseStorage hbaseStorage = new HBaseStorage(TESTCOLUMN_A); Object key = "somekey"; byte type = DataType.CHARARRAY; - Assert.assertTrue(hbaseStorage.createPut(key, type).getWriteToWAL()); - Assert.assertTrue(hbaseStorage.createDelete(key, type, System.currentTimeMillis()).getWriteToWAL()); + Put put = hbaseStorage.createPut(key, type); + Delete delete = hbaseStorage.createDelete(key, type, System.currentTimeMillis()); + boolean hasDurabilityMethod = false; + try { + put.getClass().getMethod("getDurability"); + hasDurabilityMethod = true; + } catch (NoSuchMethodException e) { + } + if (hasDurabilityMethod) { // Hbase version 0.95+ + Object skipWal = Class.forName("org.apache.hadoop.hbase.client.Durability").getField("SKIP_WAL").get(put); + Assert.assertNotEquals(put.getClass().getMethod("getDurability").invoke(put), skipWal); + Assert.assertNotEquals(delete.getClass().getMethod("getDurability").invoke(delete), skipWal); + } else { + Assert.assertTrue(put.getWriteToWAL()); + Assert.assertTrue(delete.getWriteToWAL()); + } } /** @@ -1552,7 +1580,7 @@ public class TestHBaseStorage { */ private static long getColTimestamp(Result result, String colName) { byte[][] colArray = Bytes.toByteArrays(colName.split(":")); - return result.getColumnLatestCell(colArray[0], colArray[1]).getTimestamp(); + return result.getColumnLatest(colArray[0], colArray[1]).getTimestamp(); } }