This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 454415b4798 HIVE-28571: Basic UNIONTYPE support in CBO (Stamatis
Zampetakis reviewed by Soumyakanti Das, Attila Turoczy, Alessandro Solimando,
Shohei Okumiya)
454415b4798 is described below
commit 454415b4798dbb7accf5e5de23db3dc4801b7d68
Author: Stamatis Zampetakis <[email protected]>
AuthorDate: Tue Oct 8 12:46:07 2024 +0200
HIVE-28571: Basic UNIONTYPE support in CBO (Stamatis Zampetakis reviewed by
Soumyakanti Das, Attila Turoczy, Alessandro Solimando, Shohei Okumiya)
Support UNIONTYPE in the CBO path to take advantage of the powerful
optimizations that are performed in the CBO layer and avoid relying on the
fallback mechanism. The changes do not aim to cover new use-cases but just to
ensure that existing queries that involve UNIONTYPE can exploit the CBO.
1. Model union as struct at the CBO layer
2. Remove CalciteSemanticException from TypeConverter APIs (no longer
thrown)
3. Update q.out files since queries with UNIONTYPE can now use CBO
The plan changes in existing tests are trivial and expected.
* join_thrift: Extra Select Operator; minor perf impact and does not alter
result
* union21/unionDistinct_3: COUNT(1) becomes COUNT() but both are equivalent
in terms of semantics/performance
* udf_isnull_isnotnull: Select expressions simplify to true; valid
simplification based on the filter predicates
Close apache/hive#5497
---
.../ql/metadata/HiveMaterializedViewsRegistry.java | 9 +-
.../calcite/CalciteSemanticException.java | 2 +-
.../HiveAugmentSnapshotMaterializationRule.java | 10 +-
.../calcite/translator/TypeConverter.java | 44 +++---
.../hadoop/hive/ql/parse/CalcitePlanner.java | 9 +-
ql/src/test/queries/clientpositive/uniontype_cbo.q | 31 ++++
.../results/clientpositive/llap/join_thrift.q.out | 38 +++--
.../clientpositive/llap/udf_isnull_isnotnull.q.out | 2 +-
.../test/results/clientpositive/llap/union21.q.out | 10 +-
.../clientpositive/llap/unionDistinct_3.q.out | 2 +-
.../clientpositive/llap/uniontype_cbo.q.out | 171 +++++++++++++++++++++
11 files changed, 262 insertions(+), 66 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 93ce92faf89..9e1320e62ea 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -56,7 +56,6 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.log.PerfLogger;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
@@ -421,13 +420,7 @@ public final class HiveMaterializedViewsRegistry {
}
// 1.3 Build row type from field <type, name>
- RelDataType rowType;
- try {
- rowType = TypeConverter.getType(cluster, rr, null);
- } catch (CalciteSemanticException e) {
- // Bail out
- return null;
- }
+ RelDataType rowType = TypeConverter.getType(cluster, rr, null);
// 2. Build RelOptAbstractTable
List<String> fullyQualifiedTabName = new ArrayList<>();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
index a73aa3ba1e7..690899f85e0 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/CalciteSemanticException.java
@@ -34,7 +34,7 @@ public class CalciteSemanticException extends
SemanticException {
Having_clause_without_any_groupby, Invalid_column_reference,
Invalid_decimal,
Less_than_equal_greater_than, Others, Same_name_in_multiple_expressions,
Schema_less_table, Select_alias_in_having_clause, Select_transform,
Subquery,
- Table_sample_clauses, UDTF, Union_type, Unique_join,
+ Table_sample_clauses, UDTF, Unique_join,
HighPrecisionTimestamp // CALCITE-1690
};
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentSnapshotMaterializationRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentSnapshotMaterializationRule.java
index 003b7c86ef6..253d9b6c996 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentSnapshotMaterializationRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAugmentSnapshotMaterializationRule.java
@@ -33,7 +33,6 @@ import org.apache.calcite.util.ImmutableBeans;
import org.apache.hadoop.hive.common.type.SnapshotContext;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
@@ -95,13 +94,8 @@ public class HiveAugmentSnapshotMaterializationRule extends
RelRule<HiveAugmentS
@VisibleForTesting
static RelDataType snapshotIdType(RelDataTypeFactory typeFactory) {
if (snapshotIdType == null) {
- try {
- snapshotIdType = typeFactory.createSqlType(
- TypeConverter.convert(VirtualColumn.SNAPSHOT_ID.getTypeInfo(),
- typeFactory).getSqlTypeName());
- } catch (CalciteSemanticException e) {
- throw new RuntimeException(e);
- }
+ snapshotIdType = typeFactory.createSqlType(
+ TypeConverter.convert(VirtualColumn.SNAPSHOT_ID.getTypeInfo(),
typeFactory).getSqlTypeName());
}
return snapshotIdType;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
index 7d22797869f..d191ecef829 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
@@ -42,8 +42,6 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
-import
org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
import
org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter.HiveToken;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.RowResolver;
@@ -66,6 +64,11 @@ import com.google.common.collect.Lists;
public class TypeConverter {
private static final Map<String, HiveToken> calciteToHiveTypeNameMap;
+ /**
+ * Prefix for disambiguation fields inside a union. Since we are mapping a
union to struct the $ character is
+ * used to avoid conflicts with user-defined structs.
+ */
+ private static final String UNION_FIELD_PREFIX = "$tag_";
static {
Builder<String, HiveToken> b = ImmutableMap.<String, HiveToken> builder();
@@ -109,8 +112,7 @@ public class TypeConverter {
};
/*********************** Convert Hive Types To Calcite Types
***********************/
- public static RelDataType getType(RelOptCluster cluster,
- List<ColumnInfo> cInfoLst) throws CalciteSemanticException {
+ public static RelDataType getType(RelOptCluster cluster, List<ColumnInfo>
cInfoLst) {
RexBuilder rexBuilder = cluster.getRexBuilder();
RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
List<RelDataType> fieldTypes = new LinkedList<RelDataType>();
@@ -123,8 +125,7 @@ public class TypeConverter {
return dtFactory.createStructType(fieldTypes, fieldNames);
}
- public static RelDataType getType(RelOptCluster cluster, RowResolver rr,
- List<String> neededCols) throws CalciteSemanticException {
+ public static RelDataType getType(RelOptCluster cluster, RowResolver rr,
List<String> neededCols) {
RexBuilder rexBuilder = cluster.getRexBuilder();
RelDataTypeFactory dtFactory = rexBuilder.getTypeFactory();
RowSchema rs = rr.getRowSchema();
@@ -140,12 +141,11 @@ public class TypeConverter {
return dtFactory.createStructType(fieldTypes, fieldNames);
}
- public static RelDataType convert(TypeInfo type, RelDataTypeFactory
dtFactory) throws CalciteSemanticException {
+ public static RelDataType convert(TypeInfo type, RelDataTypeFactory
dtFactory) {
return convert(type, true, dtFactory);
}
- public static RelDataType convert(TypeInfo type, boolean nullable,
RelDataTypeFactory dtFactory)
- throws CalciteSemanticException {
+ public static RelDataType convert(TypeInfo type, boolean nullable,
RelDataTypeFactory dtFactory) {
RelDataType convertedType = null;
switch (type.getCategory()) {
@@ -253,21 +253,18 @@ public class TypeConverter {
return dtFactory.createTypeWithNullability(convertedType, nullable);
}
- public static RelDataType convert(ListTypeInfo lstType,
- RelDataTypeFactory dtFactory) throws CalciteSemanticException {
+ public static RelDataType convert(ListTypeInfo lstType, RelDataTypeFactory
dtFactory) {
RelDataType elemType = convert(lstType.getListElementTypeInfo(),
dtFactory);
return dtFactory.createArrayType(elemType, -1);
}
- public static RelDataType convert(MapTypeInfo mapType, RelDataTypeFactory
dtFactory)
- throws CalciteSemanticException {
+ public static RelDataType convert(MapTypeInfo mapType, RelDataTypeFactory
dtFactory) {
RelDataType keyType = convert(mapType.getMapKeyTypeInfo(), dtFactory);
RelDataType valueType = convert(mapType.getMapValueTypeInfo(), dtFactory);
return dtFactory.createMapType(keyType, valueType);
}
- public static RelDataType convert(StructTypeInfo structType,
- final RelDataTypeFactory dtFactory) throws CalciteSemanticException {
+ public static RelDataType convert(StructTypeInfo structType,
RelDataTypeFactory dtFactory) {
List<RelDataType> fTypes = new
ArrayList<RelDataType>(structType.getAllStructFieldTypeInfos().size());
for (TypeInfo ti : structType.getAllStructFieldTypeInfos()) {
fTypes.add(convert(ti,dtFactory));
@@ -275,10 +272,12 @@ public class TypeConverter {
return dtFactory.createStructType(fTypes,
structType.getAllStructFieldNames());
}
- public static RelDataType convert(UnionTypeInfo unionType,
RelDataTypeFactory dtFactory)
- throws CalciteSemanticException{
- // Union type is not supported in Calcite.
- throw new CalciteSemanticException("Union type is not supported",
UnsupportedFeature.Union_type);
+ public static RelDataType convert(UnionTypeInfo unionType,
RelDataTypeFactory dtFactory) {
+ RelDataTypeFactory.Builder b = dtFactory.builder();
+ for (TypeInfo ti : unionType.getAllUnionObjectTypeInfos()) {
+ b.add(UNION_FIELD_PREFIX + b.getFieldCount(), convert(ti, dtFactory));
+ }
+ return b.build();
}
public static TypeInfo convertLiteralType(RexLiteral literal) {
@@ -312,7 +311,12 @@ public class TypeConverter {
return f.getName();
}
});
- return TypeInfoFactory.getStructTypeInfo(fNames, fTypes);
+ boolean isUnionType = fNames.stream().allMatch(f ->
f.startsWith(UNION_FIELD_PREFIX));
+ if (isUnionType) {
+ return TypeInfoFactory.getUnionTypeInfo(fTypes);
+ } else {
+ return TypeInfoFactory.getStructTypeInfo(fNames, fTypes);
+ }
}
public static TypeInfo convertMapType(RelDataType rType) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index f8bf8722e92..deaa9b51d0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -3355,12 +3355,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
if(context == null) {
// we have correlated column, build data type from outer rr
- RelDataType rowType;
- try {
- rowType = TypeConverter.getType(cluster, outerRowResolver, null);
- } catch (CalciteSemanticException e) {
- throw new RuntimeException("Error converting type", e);
- }
+ RelDataType rowType = TypeConverter.getType(cluster,
outerRowResolver, null);
int index = col.getIndex() -
inputContext.inputRowType.getFieldList().size();
if (outerPositionToColumnName.get(index) == null) {
throw new RuntimeException(ErrorMsg.INVALID_COLUMN_NAME.getMsg());
@@ -3535,7 +3530,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
private AggregateCall convertGBAgg(AggregateInfo agg, List<RexNode>
gbChildProjLst,
- HashMap<String, Integer> rexNodeToPosMap, Integer childProjLstIndx)
throws SemanticException {
+ HashMap<String, Integer> rexNodeToPosMap, Integer childProjLstIndx) {
// 1. Get agg fn ret type in Calcite
RelDataType aggFnRetType = TypeConverter.convert(agg.getReturnType(),
this.cluster.getTypeFactory());
diff --git a/ql/src/test/queries/clientpositive/uniontype_cbo.q
b/ql/src/test/queries/clientpositive/uniontype_cbo.q
new file mode 100644
index 00000000000..40b4161af41
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/uniontype_cbo.q
@@ -0,0 +1,31 @@
+CREATE TABLE utable (cu UNIONTYPE<INTEGER, STRING>);
+DESCRIBE utable;
+
+EXPLAIN CBO
+INSERT INTO utable values
+(create_union(0, 10, 'ten')),
+(create_union(1, 10, 'ten'));
+INSERT INTO utable values
+(create_union(0, 10, 'ten')),
+(create_union(1, 10, 'ten'));
+
+EXPLAIN CBO
+SELECT cu FROM utable;
+SELECT cu FROM utable;
+
+EXPLAIN CBO
+SELECT extract_union(cu) FROM utable;
+SELECT extract_union(cu) FROM utable;
+
+CREATE TABLE author (id INT, fname STRING, age INT);
+INSERT INTO author VALUES (0, 'Victor' , '37'), (1, 'Alexander' , '44');
+
+EXPLAIN CBO
+CREATE TABLE uauthor AS SELECT create_union(id % 2, age, fname) as u_age_name
FROM author;
+CREATE TABLE uauthor AS SELECT create_union(id % 2, age, fname) as u_age_name
FROM author;
+
+DESCRIBE uauthor;
+
+EXPLAIN CBO
+SELECT u_age_name FROM uauthor;
+SELECT u_age_name FROM uauthor;
diff --git a/ql/src/test/results/clientpositive/llap/join_thrift.q.out
b/ql/src/test/results/clientpositive/llap/join_thrift.q.out
index 82c6879eb2b..0a50e523345 100644
--- a/ql/src/test/results/clientpositive/llap/join_thrift.q.out
+++ b/ql/src/test/results/clientpositive/llap/join_thrift.q.out
@@ -51,12 +51,16 @@ STAGE PLANS:
Filter Operator
predicate: aint is not null (type: boolean)
Statistics: Num rows: 11 Data size: 44 Basic stats:
COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: aint (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: aint (type: int)
+ Select Operator
+ expressions: aint (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 11 Data size: 44 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 44 Basic stats:
COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: no inputs
Map 3
@@ -68,13 +72,17 @@ STAGE PLANS:
Filter Operator
predicate: aint is not null (type: boolean)
Statistics: Num rows: 11 Data size: 28204 Basic stats:
COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: aint (type: int)
- null sort order: z
- sort order: +
- Map-reduce partition columns: aint (type: int)
+ Select Operator
+ expressions: aint (type: int), lintstring (type:
array<struct<myint:int,mystring:string,underscore_int:int>>)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 11 Data size: 28204 Basic stats:
COMPLETE Column stats: NONE
- value expressions: lintstring (type:
array<struct<myint:int,mystring:string,underscore_int:int>>)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 28204 Basic stats:
COMPLETE Column stats: NONE
+ value expressions: _col1 (type:
array<struct<myint:int,mystring:string,underscore_int:int>>)
Execution mode: vectorized, llap
LLAP IO: no inputs
Reducer 2
@@ -84,12 +92,12 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 aint (type: int)
- 1 aint (type: int)
- outputColumnNames: _col0, _col18
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE
Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col18 (type:
array<struct<myint:int,mystring:string,underscore_int:int>>)
+ expressions: _col0 (type: int), _col2 (type:
array<struct<myint:int,mystring:string,underscore_int:int>>)
outputColumnNames: _col0, _col1
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE
Column stats: NONE
File Output Operator
diff --git a/ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out
b/ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out
index 9571be9d004..5defb61580d 100644
--- a/ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_isnull_isnotnull.q.out
@@ -110,7 +110,7 @@ STAGE PLANS:
Limit
Number of rows: 1
Select Operator
- expressions: lint is not null (type: boolean), lintstring is
not null (type: boolean), mstringstring is not null (type: boolean)
+ expressions: true (type: boolean), lintstring is not null
(type: boolean), true (type: boolean)
outputColumnNames: _col0, _col1, _col2
ListSink
diff --git a/ql/src/test/results/clientpositive/llap/union21.q.out
b/ql/src/test/results/clientpositive/llap/union21.q.out
index 6ad3058fbf1..60ad7bb1698 100644
--- a/ql/src/test/results/clientpositive/llap/union21.q.out
+++ b/ql/src/test/results/clientpositive/llap/union21.q.out
@@ -61,7 +61,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 42500 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
- aggregations: count(1)
+ aggregations: count()
keys: _col0 (type: string)
minReductionHashAggr: 0.99
mode: hash
@@ -86,7 +86,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 92000 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
- aggregations: count(1)
+ aggregations: count()
keys: _col0 (type: string)
minReductionHashAggr: 0.99
mode: hash
@@ -111,7 +111,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 500 Data size: 43500 Basic stats:
COMPLETE Column stats: COMPLETE
Group By Operator
- aggregations: count(1)
+ aggregations: count()
keys: _col0 (type: string)
minReductionHashAggr: 0.99
mode: hash
@@ -136,7 +136,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 11 Data size: 2024 Basic stats:
COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
+ aggregations: count()
keys: _col0 (type: string)
minReductionHashAggr: 0.99
mode: hash
@@ -161,7 +161,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 11 Data size: 21120 Basic stats:
COMPLETE Column stats: NONE
Group By Operator
- aggregations: count(1)
+ aggregations: count()
keys: _col0 (type: string)
minReductionHashAggr: 0.99
mode: hash
diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out
b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out
index 57988074679..ae9e75a6d9b 100644
--- a/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/unionDistinct_3.q.out
@@ -2236,7 +2236,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 316 Data size: 58144 Basic stats:
COMPLETE Column stats: PARTIAL
Group By Operator
- aggregations: count(1)
+ aggregations: count()
keys: _col0 (type: string)
mode: complete
outputColumnNames: _col0, _col1
diff --git a/ql/src/test/results/clientpositive/llap/uniontype_cbo.q.out
b/ql/src/test/results/clientpositive/llap/uniontype_cbo.q.out
new file mode 100644
index 00000000000..2c6da3081c5
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/uniontype_cbo.q.out
@@ -0,0 +1,171 @@
+PREHOOK: query: CREATE TABLE utable (cu UNIONTYPE<INTEGER, STRING>)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@utable
+POSTHOOK: query: CREATE TABLE utable (cu UNIONTYPE<INTEGER, STRING>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@utable
+PREHOOK: query: DESCRIBE utable
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@utable
+POSTHOOK: query: DESCRIBE utable
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@utable
+cu uniontype<int,string>
+PREHOOK: query: EXPLAIN CBO
+INSERT INTO utable values
+(create_union(0, 10, 'ten')),
+(create_union(1, 10, 'ten'))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@utable
+POSTHOOK: query: EXPLAIN CBO
+INSERT INTO utable values
+(create_union(0, 10, 'ten')),
+(create_union(1, 10, 'ten'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@utable
+CBO PLAN:
+HiveTableFunctionScan(invocation=[inline(ARRAY(ROW(create_union(0, 10,
_UTF-16LE'ten':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")),
ROW(create_union(1, 10, _UTF-16LE'ten':VARCHAR(2147483647) CHARACTER SET
"UTF-16LE"))))], rowType=[RecordType(RecordType(INTEGER $tag_0,
VARCHAR(2147483647) $tag_1) col1)])
+ HiveTableScan(table=[[_dummy_database, _dummy_table]],
table:alias=[_dummy_table])
+
+PREHOOK: query: INSERT INTO utable values
+(create_union(0, 10, 'ten')),
+(create_union(1, 10, 'ten'))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@utable
+POSTHOOK: query: INSERT INTO utable values
+(create_union(0, 10, 'ten')),
+(create_union(1, 10, 'ten'))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@utable
+POSTHOOK: Lineage: utable.cu SCRIPT []
+PREHOOK: query: EXPLAIN CBO
+SELECT cu FROM utable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@utable
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO
+SELECT cu FROM utable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@utable
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(cu=[$0])
+ HiveTableScan(table=[[default, utable]], table:alias=[utable])
+
+PREHOOK: query: SELECT cu FROM utable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@utable
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT cu FROM utable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@utable
+#### A masked pattern was here ####
+{0:10}
+{1:"ten"}
+PREHOOK: query: EXPLAIN CBO
+SELECT extract_union(cu) FROM utable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@utable
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO
+SELECT extract_union(cu) FROM utable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@utable
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(_o__c0=[extract_union($0)])
+ HiveTableScan(table=[[default, utable]], table:alias=[utable])
+
+PREHOOK: query: SELECT extract_union(cu) FROM utable
+PREHOOK: type: QUERY
+PREHOOK: Input: default@utable
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT extract_union(cu) FROM utable
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@utable
+#### A masked pattern was here ####
+{"tag_0":10,"tag_1":null}
+{"tag_0":null,"tag_1":"ten"}
+PREHOOK: query: CREATE TABLE author (id INT, fname STRING, age INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@author
+POSTHOOK: query: CREATE TABLE author (id INT, fname STRING, age INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@author
+PREHOOK: query: INSERT INTO author VALUES (0, 'Victor' , '37'), (1,
'Alexander' , '44')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@author
+POSTHOOK: query: INSERT INTO author VALUES (0, 'Victor' , '37'), (1,
'Alexander' , '44')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@author
+POSTHOOK: Lineage: author.age SCRIPT []
+POSTHOOK: Lineage: author.fname SCRIPT []
+POSTHOOK: Lineage: author.id SCRIPT []
+PREHOOK: query: EXPLAIN CBO
+CREATE TABLE uauthor AS SELECT create_union(id % 2, age, fname) as u_age_name
FROM author
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@author
+PREHOOK: Output: database:default
+PREHOOK: Output: default@uauthor
+POSTHOOK: query: EXPLAIN CBO
+CREATE TABLE uauthor AS SELECT create_union(id % 2, age, fname) as u_age_name
FROM author
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@author
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@uauthor
+CBO PLAN:
+HiveProject(u_age_name=[create_union(MOD($0, 2), $2, $1)])
+ HiveTableScan(table=[[default, author]], table:alias=[author])
+
+PREHOOK: query: CREATE TABLE uauthor AS SELECT create_union(id % 2, age,
fname) as u_age_name FROM author
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@author
+PREHOOK: Output: database:default
+PREHOOK: Output: default@uauthor
+POSTHOOK: query: CREATE TABLE uauthor AS SELECT create_union(id % 2, age,
fname) as u_age_name FROM author
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@author
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@uauthor
+POSTHOOK: Lineage: uauthor.u_age_name EXPRESSION
[(author)author.FieldSchema(name:id, type:int, comment:null),
(author)author.FieldSchema(name:age, type:int, comment:null),
(author)author.FieldSchema(name:fname, type:string, comment:null), ]
+PREHOOK: query: DESCRIBE uauthor
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@uauthor
+POSTHOOK: query: DESCRIBE uauthor
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@uauthor
+u_age_name uniontype<int,string>
+PREHOOK: query: EXPLAIN CBO
+SELECT u_age_name FROM uauthor
+PREHOOK: type: QUERY
+PREHOOK: Input: default@uauthor
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN CBO
+SELECT u_age_name FROM uauthor
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@uauthor
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(u_age_name=[$0])
+ HiveTableScan(table=[[default, uauthor]], table:alias=[uauthor])
+
+PREHOOK: query: SELECT u_age_name FROM uauthor
+PREHOOK: type: QUERY
+PREHOOK: Input: default@uauthor
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT u_age_name FROM uauthor
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@uauthor
+#### A masked pattern was here ####
+{0:37}
+{1:"Alexander"}
