Author: knoguchi
Date: Tue Mar 12 04:27:48 2019
New Revision: 1855282
URL: http://svn.apache.org/viewvc?rev=1855282&view=rev
Log:
PIG-5383: OrcStorage fails when "bytearray" represents unknown type (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java
pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1855282&r1=1855281&r2=1855282&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 12 04:27:48 2019
@@ -87,6 +87,9 @@ PIG-5251: Bump joda-time to 2.9.9 (dbist
OPTIMIZATIONS
BUG FIXES
+
+PIG-5383: OrcStorage fails when "bytearray" represents unknown type (knoguchi)
+
PIG-5372: SAMPLE/RANDOM(udf) before skewed join failing with NPE (knoguchi)
PIG-5374: Use CircularFifoBuffer in InterRecordReader (szita)
Modified: pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java?rev=1855282&r1=1855281&r2=1855282&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java Tue Mar 12
04:27:48 2019
@@ -658,9 +658,13 @@ public class HiveUtils {
@Override
public BytesWritable getPrimitiveWritableObject(Object o) {
- return o == null ? null : (o instanceof DataByteArray
+ try {
+ return o == null ? null : (o instanceof DataByteArray
? new BytesWritable(((DataByteArray) o).get())
- : new BytesWritable((byte[]) o));
+ : new BytesWritable((byte[]) DataType.toBytes(o)));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
@Override
Modified: pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1855282&r1=1855281&r2=1855282&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java Tue Mar 12
04:27:48 2019
@@ -20,6 +20,8 @@ package org.apache.pig.builtin;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.tuple;
import java.io.File;
import java.io.FileNotFoundException;
@@ -57,6 +59,7 @@ import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.Text;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.builtin.mock.Storage.Data;
import org.apache.pig.data.BinSedesTuple;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
@@ -283,6 +286,28 @@ public class TestOrcStorage {
assertTrue(t.toString().endsWith(",12345678.6547456)"));
}
+ @Test
+ // See PIG-5383
+ public void testByteArrayStore() throws Exception {
+ Data data = resetData(pigServer);
+ data.set("foo", "intButTypeByteArray:bytearray",
+ tuple(1),
+ tuple(2),
+ tuple(3)
+ );
+
+ // Emunlating the case when "bytearray" represents an unknown type
+
+ pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
+ pigServer.registerQuery("store A into '" + OUTPUT1 + "' using
OrcStorage();");
+ pigServer.registerQuery("B = load '" + OUTPUT1 + "' using
OrcStorage();");
+ Iterator <Tuple> iter = pigServer.openIterator("B");
+ assertEquals(Integer.valueOf(1),
DataType.toInteger(iter.next().get(0), DataType.BYTEARRAY));
+ assertEquals(Integer.valueOf(2),
DataType.toInteger(iter.next().get(0), DataType.BYTEARRAY));
+ assertEquals(Integer.valueOf(3),
DataType.toInteger(iter.next().get(0), DataType.BYTEARRAY));
+ assertFalse(iter.hasNext());
+ }
+
private void verifyData(Path orcFile, Iterator<Tuple> iter, FileSystem fs,
int expectedTotalRows) throws Exception {
int expectedRows = 0;
