HIVE-16691: Add test for more datatypes for LlapInputFormat (Jason Dere, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/04246db0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/04246db0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/04246db0 Branch: refs/heads/hive-14535 Commit: 04246db02177dc93286645d172ea226be7a36572 Parents: 4045ca3 Author: Jason Dere <[email protected]> Authored: Fri May 19 10:33:40 2017 -0700 Committer: Jason Dere <[email protected]> Committed: Fri May 19 10:33:40 2017 -0700 ---------------------------------------------------------------------- .../apache/hive/jdbc/TestJdbcWithMiniLlap.java | 312 ++++++++++++------- 1 file changed, 194 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/04246db0/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java index 606a263..b5f1f13 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java @@ -27,12 +27,15 @@ import static org.junit.Assert.fail; import java.io.File; import java.lang.reflect.Field; +import java.math.BigDecimal; import java.net.URL; import java.sql.Connection; +import java.sql.Date; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -95,6 +98,7 @@ public class TestJdbcWithMiniLlap { private static MiniHS2 miniHS2 = null; private static String dataFileDir; private static Path kvDataFilePath; + private static Path dataTypesFilePath; private static final String tmpDir = System.getProperty("test.tmp.dir"); private static HiveConf conf = null; @@ -120,6 +124,7 @@ public class TestJdbcWithMiniLlap { dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", ""); kvDataFilePath = new Path(dataFileDir, "kv1.txt"); + dataTypesFilePath = new Path(dataFileDir, "datatypes.txt"); Map<String, String> confOverlay = new HashMap<String, String>(); miniHS2.start(confOverlay); miniHS2.getDFS().getFileSystem().mkdirs(new Path("/apps_staging_dir/anonymous")); @@ -167,37 +172,31 @@ public class TestJdbcWithMiniLlap { stmt.close(); } - private void createTableWithComplexTypes(String tableName) throws Exception { + private void createDataTypesTable(String tableName) throws Exception { Statement stmt = hs2Conn.createStatement(); // create table stmt.execute("DROP TABLE IF EXISTS " + tableName); - stmt.execute("CREATE TABLE " + tableName - + " (c0 int, c1 array<int>, c2 map<int, string>, c3 struct<f1:int, f2:string, f3:array<int>>, c4 array<struct<f1:int, f2:string, f3:array<int>>>)"); - - // load data - stmt.execute("insert into " + tableName - + " select 1" - + ", array(1, 2, 3)" - + ", map(1, 'one', 2, 'two')" - + ", named_struct('f1', 1, 'f2', 'two', 'f3', array(1,2,3))" - + ", array(named_struct('f1', 11, 'f2', 'two', 'f3', array(2,3,4)))"); - - // Inserting nulls into complex columns doesn't work without this CASE workaround - what a hack. - stmt.execute("insert into " + tableName - + " select 2" - + ", case when 2 = 2 then null else array(1, 2, 3) end" - + ", case when 2 = 2 then null else map(1, 'one', 2, 'two') end" - + ", case when 2 = 2 then null else named_struct('f1', 1, 'f2', 'two', 'f3', array(1,2,3)) end" - + ", case when 2 = 2 then null else array(named_struct('f1', 11, 'f2', 'two', 'f3', array(2,3,4))) end"); - - // TODO: test nested nulls in complex types. Currently blocked by HIVE-16587. - //stmt.execute("insert into " + tableName - // + " select 3" - // + ", array(1, 2, null)" - // + ", map(1, 'one', 2, null)" - // + ", named_struct('f1', cast(null as int), 'f2', cast(null as string), 'f3', array(1,2,null))" - // + ", array(named_struct('f1', 11, 'f2', 'two', 'f3', array(2,3,4)))"); + // tables with various types + stmt.execute("create table " + tableName + + " (c1 int, c2 boolean, c3 double, c4 string," + + " c5 array<int>, c6 map<int,string>, c7 map<string,string>," + + " c8 struct<r:string,s:int,t:double>," + + " c9 tinyint, c10 smallint, c11 float, c12 bigint," + + " c13 array<array<string>>," + + " c14 map<int, map<int,int>>," + + " c15 struct<r:int,s:struct<a:int,b:string>>," + + " c16 array<struct<m:map<string,string>,n:int>>," + + " c17 timestamp, " + + " c18 decimal(16,7), " + + " c19 binary, " + + " c20 date," + + " c21 varchar(20)," + + " c22 char(15)," + + " c23 binary" + + ")"); + stmt.execute("load data local inpath '" + + dataTypesFilePath.toString() + "' into table " + tableName); stmt.close(); } @@ -254,102 +253,178 @@ public class TestJdbcWithMiniLlap { } @Test(timeout = 60000) - public void testComplexTypes() throws Exception { - createTableWithComplexTypes("complex1"); + public void testDataTypes() throws Exception { + createDataTypesTable("datatypes"); RowCollector2 rowCollector = new RowCollector2(); - String query = "select * from complex1"; + String query = "select * from datatypes"; int rowCount = processQuery(query, 1, rowCollector); - assertEquals(2, rowCount); + assertEquals(3, rowCount); // Verify schema - FieldDesc c0Desc = rowCollector.schema.getColumns().get(0); - assertEquals("complex1.c0", c0Desc.getName()); - assertEquals("int", c0Desc.getTypeInfo().getTypeName()); - - FieldDesc c1Desc = rowCollector.schema.getColumns().get(1); - assertEquals("complex1.c1", c1Desc.getName()); - assertEquals("array<int>", c1Desc.getTypeInfo().getTypeName()); - - FieldDesc c2Desc = rowCollector.schema.getColumns().get(2); - assertEquals("complex1.c2", c2Desc.getName()); - assertEquals("map<int,string>", c2Desc.getTypeInfo().getTypeName()); - - FieldDesc c3Desc = rowCollector.schema.getColumns().get(3); - assertEquals("complex1.c3", c3Desc.getName()); - assertEquals(Category.STRUCT, c3Desc.getTypeInfo().getCategory()); - verifyStructFieldSchema((StructTypeInfo) c3Desc.getTypeInfo()); - - FieldDesc c4Desc = rowCollector.schema.getColumns().get(4); - assertEquals("complex1.c4", c4Desc.getName()); - assertEquals(Category.LIST, c4Desc.getTypeInfo().getCategory()); - TypeInfo c4ElementType = ((ListTypeInfo) c4Desc.getTypeInfo()).getListElementTypeInfo(); - assertEquals(Category.STRUCT, c4ElementType.getCategory()); - verifyStructFieldSchema((StructTypeInfo) c4ElementType); - - // First row + String[][] colNameTypes = new String[][] { + {"datatypes.c1", "int"}, + {"datatypes.c2", "boolean"}, + {"datatypes.c3", "double"}, + {"datatypes.c4", "string"}, + {"datatypes.c5", "array<int>"}, + {"datatypes.c6", "map<int,string>"}, + {"datatypes.c7", "map<string,string>"}, + {"datatypes.c8", "struct<r:string,s:int,t:double>"}, + {"datatypes.c9", "tinyint"}, + {"datatypes.c10", "smallint"}, + {"datatypes.c11", "float"}, + {"datatypes.c12", "bigint"}, + {"datatypes.c13", "array<array<string>>"}, + {"datatypes.c14", "map<int,map<int,int>>"}, + {"datatypes.c15", "struct<r:int,s:struct<a:int,b:string>>"}, + {"datatypes.c16", "array<struct<m:map<string,string>,n:int>>"}, + {"datatypes.c17", "timestamp"}, + {"datatypes.c18", "decimal(16,7)"}, + {"datatypes.c19", "binary"}, + {"datatypes.c20", "date"}, + {"datatypes.c21", "varchar(20)"}, + {"datatypes.c22", "char(15)"}, + {"datatypes.c23", "binary"}, + }; + FieldDesc fieldDesc; + assertEquals(23, rowCollector.numColumns); + for (int idx = 0; idx < rowCollector.numColumns; ++idx) { + fieldDesc = rowCollector.schema.getColumns().get(idx); + assertEquals("ColName idx=" + idx, colNameTypes[idx][0], fieldDesc.getName()); + assertEquals("ColType idx=" + idx, colNameTypes[idx][1], fieldDesc.getTypeInfo().getTypeName()); + } + + // First row is all nulls Object[] rowValues = rowCollector.rows.get(0); - assertEquals(Integer.valueOf(1), ((Integer) rowValues[0])); - - // assertEquals("[1, 2, 3]", rowValues[1]); - List<?> c1Value = (List<?>) rowValues[1]; - assertEquals(3, c1Value.size()); - assertEquals(Integer.valueOf(1), c1Value.get(0)); - assertEquals(Integer.valueOf(2), c1Value.get(1)); - assertEquals(Integer.valueOf(3), c1Value.get(2)); - - // assertEquals("{1=one, 2=two}", rowValues[2]); - Map<?,?> c2Value = (Map<?,?>) rowValues[2]; - assertEquals(2, c2Value.size()); - assertEquals("one", c2Value.get(Integer.valueOf(1))); - assertEquals("two", c2Value.get(Integer.valueOf(2))); - - // assertEquals("[1, two, [1, 2, 3]]", rowValues[3]); - List<?> c3Value = (List<?>) rowValues[3]; - assertEquals(Integer.valueOf(1), c3Value.get(0)); - assertEquals("two", c3Value.get(1)); - List<?> f3Value = (List<?>) c3Value.get(2); - assertEquals(Integer.valueOf(1), f3Value.get(0)); - assertEquals(Integer.valueOf(2), f3Value.get(1)); - assertEquals(Integer.valueOf(3), f3Value.get(2)); - - // assertEquals("[[11, two, [2, 3, 4]]]", rowValues[4]); - List<?> c4Value = (List<?>) rowValues[4]; - assertEquals(1, c4Value.size()); - List<?> c4Element = (List<?>) c4Value.get(0); - assertEquals(Integer.valueOf(11), c4Element.get(0)); - assertEquals("two", c4Element.get(1)); - f3Value = (List<?>) c4Element.get(2); - assertEquals(3, f3Value.size()); - assertEquals(Integer.valueOf(2), f3Value.get(0)); - assertEquals(Integer.valueOf(3), f3Value.get(1)); - assertEquals(Integer.valueOf(4), f3Value.get(2)); - - // Second row - rowValues = rowCollector.rows.get(1); - assertEquals(Integer.valueOf(2), ((Integer) rowValues[0])); - assertEquals(null, rowValues[1]); - assertEquals(null, rowValues[2]); - assertEquals(null, rowValues[3]); - assertEquals(null, rowValues[4]); - } + for (int idx = 0; idx < rowCollector.numColumns; ++idx) { + assertEquals("idx=" + idx, null, rowValues[idx]); + } - private void verifyStructFieldSchema(StructTypeInfo structType) { - assertEquals("f1", structType.getAllStructFieldNames().get(0)); - TypeInfo f1Type = structType.getStructFieldTypeInfo("f1"); - assertEquals(Category.PRIMITIVE, f1Type.getCategory()); - assertEquals(PrimitiveCategory.INT, ((PrimitiveTypeInfo) f1Type).getPrimitiveCategory()); - - assertEquals("f2", structType.getAllStructFieldNames().get(1)); - TypeInfo f2Type = structType.getStructFieldTypeInfo("f2"); - assertEquals(Category.PRIMITIVE, f2Type.getCategory()); - assertEquals(PrimitiveCategory.STRING, ((PrimitiveTypeInfo) f2Type).getPrimitiveCategory()); - - assertEquals("f3", structType.getAllStructFieldNames().get(2)); - TypeInfo f3Type = structType.getStructFieldTypeInfo("f3"); - assertEquals(Category.LIST, f3Type.getCategory()); - assertEquals( - PrimitiveCategory.INT, - ((PrimitiveTypeInfo) ((ListTypeInfo) f3Type).getListElementTypeInfo()).getPrimitiveCategory()); + // Second Row + rowValues = rowCollector.rows.get(1); + assertEquals(Integer.valueOf(-1), rowValues[0]); + assertEquals(Boolean.FALSE, rowValues[1]); + assertEquals(Double.valueOf(-1.1d), rowValues[2]); + assertEquals("", rowValues[3]); + + List<?> c5Value = (List<?>) rowValues[4]; + assertEquals(0, c5Value.size()); + + Map<?,?> c6Value = (Map<?,?>) rowValues[5]; + assertEquals(0, c6Value.size()); + + Map<?,?> c7Value = (Map<?,?>) rowValues[6]; + assertEquals(0, c7Value.size()); + + List<?> c8Value = (List<?>) rowValues[7]; + assertEquals(null, c8Value.get(0)); + assertEquals(null, c8Value.get(1)); + assertEquals(null, c8Value.get(2)); + + assertEquals(Byte.valueOf((byte) -1), rowValues[8]); + assertEquals(Short.valueOf((short) -1), rowValues[9]); + assertEquals(Float.valueOf(-1.0f), rowValues[10]); + assertEquals(Long.valueOf(-1l), rowValues[11]); + + List<?> c13Value = (List<?>) rowValues[12]; + assertEquals(0, c13Value.size()); + + Map<?,?> c14Value = (Map<?,?>) rowValues[13]; + assertEquals(0, c14Value.size()); + + List<?> c15Value = (List<?>) rowValues[14]; + assertEquals(null, c15Value.get(0)); + assertEquals(null, c15Value.get(1)); + + List<?> c16Value = (List<?>) rowValues[15]; + assertEquals(0, c16Value.size()); + + assertEquals(null, rowValues[16]); + assertEquals(null, rowValues[17]); + assertEquals(null, rowValues[18]); + assertEquals(null, rowValues[19]); + assertEquals(null, rowValues[20]); + assertEquals(null, rowValues[21]); + assertEquals(null, rowValues[22]); + + // Third row + rowValues = rowCollector.rows.get(2); + assertEquals(Integer.valueOf(1), rowValues[0]); + assertEquals(Boolean.TRUE, rowValues[1]); + assertEquals(Double.valueOf(1.1d), rowValues[2]); + assertEquals("1", rowValues[3]); + + c5Value = (List<?>) rowValues[4]; + assertEquals(2, c5Value.size()); + assertEquals(Integer.valueOf(1), c5Value.get(0)); + assertEquals(Integer.valueOf(2), c5Value.get(1)); + + c6Value = (Map<?,?>) rowValues[5]; + assertEquals(2, c6Value.size()); + assertEquals("x", c6Value.get(Integer.valueOf(1))); + assertEquals("y", c6Value.get(Integer.valueOf(2))); + + c7Value = (Map<?,?>) rowValues[6]; + assertEquals(1, c7Value.size()); + assertEquals("v", c7Value.get("k")); + + c8Value = (List<?>) rowValues[7]; + assertEquals("a", c8Value.get(0)); + assertEquals(Integer.valueOf(9), c8Value.get(1)); + assertEquals(Double.valueOf(2.2d), c8Value.get(2)); + + assertEquals(Byte.valueOf((byte) 1), rowValues[8]); + assertEquals(Short.valueOf((short) 1), rowValues[9]); + assertEquals(Float.valueOf(1.0f), rowValues[10]); + assertEquals(Long.valueOf(1l), rowValues[11]); + + c13Value = (List<?>) rowValues[12]; + assertEquals(2, c13Value.size()); + List<?> listVal = (List<?>) c13Value.get(0); + assertEquals("a", listVal.get(0)); + assertEquals("b", listVal.get(1)); + listVal = (List<?>) c13Value.get(1); + assertEquals("c", listVal.get(0)); + assertEquals("d", listVal.get(1)); + + c14Value = (Map<?,?>) rowValues[13]; + assertEquals(2, c14Value.size()); + Map<?,?> mapVal = (Map<?,?>) c14Value.get(Integer.valueOf(1)); + assertEquals(2, mapVal.size()); + assertEquals(Integer.valueOf(12), mapVal.get(Integer.valueOf(11))); + assertEquals(Integer.valueOf(14), mapVal.get(Integer.valueOf(13))); + mapVal = (Map<?,?>) c14Value.get(Integer.valueOf(2)); + assertEquals(1, mapVal.size()); + assertEquals(Integer.valueOf(22), mapVal.get(Integer.valueOf(21))); + + c15Value = (List<?>) rowValues[14]; + assertEquals(Integer.valueOf(1), c15Value.get(0)); + listVal = (List<?>) c15Value.get(1); + assertEquals(2, listVal.size()); + assertEquals(Integer.valueOf(2), listVal.get(0)); + assertEquals("x", listVal.get(1)); + + c16Value = (List<?>) rowValues[15]; + assertEquals(2, c16Value.size()); + listVal = (List<?>) c16Value.get(0); + assertEquals(2, listVal.size()); + mapVal = (Map<?,?>) listVal.get(0); + assertEquals(0, mapVal.size()); + assertEquals(Integer.valueOf(1), listVal.get(1)); + listVal = (List<?>) c16Value.get(1); + mapVal = (Map<?,?>) listVal.get(0); + assertEquals(2, mapVal.size()); + assertEquals("b", mapVal.get("a")); + assertEquals("d", mapVal.get("c")); + assertEquals(Integer.valueOf(2), listVal.get(1)); + + assertEquals(Timestamp.valueOf("2012-04-22 09:00:00.123456789"), rowValues[16]); + assertEquals(new BigDecimal("123456789.123456"), rowValues[17]); + assertArrayEquals("abcd".getBytes("UTF-8"), (byte[]) rowValues[18]); + assertEquals(Date.valueOf("2013-01-01"), rowValues[19]); + assertEquals("abc123", rowValues[20]); + assertEquals("abc123 ", rowValues[21]); + assertArrayEquals("X'01FF'".getBytes("UTF-8"), (byte[]) rowValues[22]); } private interface RowProcessor { @@ -369,7 +444,8 @@ public class TestJdbcWithMiniLlap { String[] arr = new String[numColumns]; for (int idx = 0; idx < numColumns; ++idx) { - arr[idx] = row.getValue(idx).toString(); + Object val = row.getValue(idx); + arr[idx] = (val == null ? null : val.toString()); } rows.add(arr); }
