Repository: hive Updated Branches: refs/heads/master d0e868fcf -> 607f23d90
HIVE-11185: Fix compustat_avro.q/load_dyn_part14_win.q for Windows (Jason Dere, reviewed by Hari Subramaniyan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/607f23d9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/607f23d9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/607f23d9 Branch: refs/heads/master Commit: 607f23d90c8f94d13a64fae1e4f23e58dd3fb3a2 Parents: d0e868f Author: Jason Dere <jd...@hortonworks.com> Authored: Mon Aug 3 13:43:57 2015 -0700 Committer: Jason Dere <jd...@hortonworks.com> Committed: Mon Aug 3 13:43:57 2015 -0700 ---------------------------------------------------------------------- .../queries/clientpositive/compustat_avro.q | 8 +- .../clientpositive/load_dyn_part14_win.q | 18 +- .../results/clientpositive/compustat_avro.q.out | 8 +- .../clientpositive/load_dyn_part14_win.q.out | 167 ++++++++----------- 4 files changed, 95 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/queries/clientpositive/compustat_avro.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/compustat_avro.q b/ql/src/test/queries/clientpositive/compustat_avro.q index 4d1b604..4d781a3 100644 --- a/ql/src/test/queries/clientpositive/compustat_avro.q +++ b/ql/src/test/queries/clientpositive/compustat_avro.q @@ -1,4 +1,10 @@ drop table if exists testAvro; + +dfs -cp ${system:hive.root}data/files/grad.avsc ${system:test.tmp.dir}/; + +-- File URIs using system:hive.root (using file:/) don't seem to work properly in DDL statements on Windows, +-- so use dfs to copy them over to system:test.tmp.dir (which uses pfile:/), which does appear to work + create table testAvro ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' @@ -6,7 +12,7 @@ create table testAvro 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' - TBLPROPERTIES ('avro.schema.url'='file://${system:hive.root}data/files/grad.avsc'); + TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/grad.avsc'); describe formatted testAvro.col1; http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/queries/clientpositive/load_dyn_part14_win.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/load_dyn_part14_win.q b/ql/src/test/queries/clientpositive/load_dyn_part14_win.q old mode 100644 new mode 100755 index d32b539..8ff4772 --- a/ql/src/test/queries/clientpositive/load_dyn_part14_win.q +++ b/ql/src/test/queries/clientpositive/load_dyn_part14_win.q @@ -1,8 +1,9 @@ -- INCLUDE_OS_WINDOWS -- included only on windows because of difference in file name encoding logic +-- SORT_QUERY_RESULTS -create table if not exists nzhang_part14 (key string) +create table if not exists nzhang_part14 (key string) partitioned by (value string); describe extended nzhang_part14; @@ -13,26 +14,25 @@ set hive.exec.dynamic.partition.mode=nonstrict; explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T; insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T; show partitions nzhang_part14; -select * from nzhang_part14 where value <> 'a' -order by key, value; +select * from nzhang_part14 where value <> 'a'; http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/results/clientpositive/compustat_avro.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/compustat_avro.q.out b/ql/src/test/results/clientpositive/compustat_avro.q.out index 8422108..f4becf1 100644 --- a/ql/src/test/results/clientpositive/compustat_avro.q.out +++ b/ql/src/test/results/clientpositive/compustat_avro.q.out @@ -2,7 +2,9 @@ PREHOOK: query: drop table if exists testAvro PREHOOK: type: DROPTABLE POSTHOOK: query: drop table if exists testAvro POSTHOOK: type: DROPTABLE -PREHOOK: query: create table testAvro +#### A masked pattern was here #### + +create table testAvro ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' STORED AS INPUTFORMAT @@ -13,7 +15,9 @@ PREHOOK: query: create table testAvro PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@testAvro -POSTHOOK: query: create table testAvro +#### A masked pattern was here #### + +create table testAvro ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' STORED AS INPUTFORMAT http://git-wip-us.apache.org/repos/asf/hive/blob/607f23d9/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out b/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out index e499552..d793e39 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part14_win.q.out @@ -1,8 +1,9 @@ PREHOOK: query: -- INCLUDE_OS_WINDOWS -- included only on windows because of difference in file name encoding logic +-- SORT_QUERY_RESULTS -create table if not exists nzhang_part14 (key string) +create table if not exists nzhang_part14 (key string) partitioned by (value string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default @@ -10,8 +11,9 @@ PREHOOK: Output: default@nzhang_part14 POSTHOOK: query: -- INCLUDE_OS_WINDOWS -- included only on windows because of difference in file name encoding logic +-- SORT_QUERY_RESULTS -create table if not exists nzhang_part14 (key string) +create table if not exists nzhang_part14 (key string) partitioned by (value string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -34,21 +36,21 @@ value string PREHOOK: query: explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T PREHOOK: type: QUERY POSTHOOK: query: explain insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -72,24 +74,21 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'k2' (type: string), '' (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 85000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 'k1' (type: string), null (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false table: @@ -102,49 +101,37 @@ STAGE PLANS: Map Operator Tree: TableScan Union - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 + Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part14 TableScan Union - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 + Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part14 TableScan Union - Statistics: Num rows: 6 Data size: 1022 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: _col0 (type: string), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 1026 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.nzhang_part14 + Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.nzhang_part14 Stage: Stage-8 Conditional Operator @@ -207,24 +194,21 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'k3' (type: string), ' ' (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 85500 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 342 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 'k2' (type: string), '' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false table: @@ -239,24 +223,21 @@ STAGE PLANS: alias: src Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: 'k1' (type: string), UDFToString(null) (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 85000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Limit Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator sort order: - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: string), _col1 (type: string) + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 2 - Statistics: Num rows: 2 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 2 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 'k3' (type: string), ' ' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE File Output Operator compressed: false table: @@ -266,22 +247,22 @@ STAGE PLANS: PREHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@nzhang_part14 POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) select key, value from ( - select 'k1' as key, cast(null as string) as value from src limit 2 + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a union all - select 'k2' as key, '' as value from src limit 2 + select * from (select 'k2' as key, '' as value from src limit 2)b union all - select 'k3' as key, ' ' as value from src limit 2 + select * from (select 'k3' as key, ' ' as value from src limit 2)c ) T POSTHOOK: type: QUERY POSTHOOK: Input: default@src @@ -298,14 +279,12 @@ POSTHOOK: Input: default@nzhang_part14 value=%2520 value=__HIVE_DEFAULT_PARTITION__ PREHOOK: query: select * from nzhang_part14 where value <> 'a' -order by key, value PREHOOK: type: QUERY PREHOOK: Input: default@nzhang_part14 PREHOOK: Input: default@nzhang_part14@value=%2520 PREHOOK: Input: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ #### A masked pattern was here #### POSTHOOK: query: select * from nzhang_part14 where value <> 'a' -order by key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@nzhang_part14 POSTHOOK: Input: default@nzhang_part14@value=%2520