Repository: hive Updated Branches: refs/heads/branch-2 044d42942 -> 84483e7da
HIVE-16454: Add blobstore tests for inserting empty into dynamic partition/list bucket tables & inserting cross blobstore tables (Rentao Wu, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/84483e7d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/84483e7d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/84483e7d Branch: refs/heads/branch-2 Commit: 84483e7da10e5d38d9f1303f09a757a48669c40e Parents: 044d429 Author: Rentao Wu <r...@amazon.com> Authored: Thu Apr 20 13:07:51 2017 -0500 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Thu Apr 20 13:08:32 2017 -0500 ---------------------------------------------------------------------- .../insert_blobstore_to_blobstore.q | 29 ++++ .../insert_empty_into_blobstore.q | 53 +++++++ .../insert_blobstore_to_blobstore.q.out | 110 +++++++++++++ .../insert_empty_into_blobstore.q.out | 155 +++++++++++++++++++ 4 files changed, 347 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/84483e7d/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q new file mode 100644 index 0000000..8219ee2 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_blobstore_to_blobstore.q @@ -0,0 +1,29 @@ +-- Test inserting into a blobstore table from another blobstore table. + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source ( + a string, + b string, + c double) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_source'; + +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE blobstore_table; +CREATE TABLE blobstore_table LIKE blobstore_source +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_blobstore_to_blobstore/blobstore_table'; + +INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source; + +SELECT COUNT(*) FROM blobstore_table; + +-- INSERT INTO should append all records to existing ones. +INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source; + +SELECT COUNT(*) FROM blobstore_table; + +SELECT * FROM blobstore_table; http://git-wip-us.apache.org/repos/asf/hive/blob/84483e7d/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q new file mode 100644 index 0000000..d4f0c71 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/insert_empty_into_blobstore.q @@ -0,0 +1,53 @@ +-- Test inserting empty rows into dynamic partitioned and list bucketed blobstore tables + +SET hive.blobstore.optimizations.enabled=true; + +DROP TABLE empty; +DROP TABLE blobstore_dynamic_partitioning; +DROP TABLE blobstore_list_bucketing; + +CREATE TABLE empty ( + id int, + name string, + dept string, + pt string, + dt string, + hr string); + +CREATE TABLE blobstore_dynamic_partitioning ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_dynamic_partitioning'; + +INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty; + +SELECT COUNT(*) FROM blobstore_dynamic_partitioning; + +CREATE TABLE blobstore_list_bucketing ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +LOCATION '${hiveconf:test.blobstore.path.unique}/insert_empty_into_blobstore/blobstore_list_bucketing'; + +INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty; + +SELECT COUNT(*) FROM blobstore_list_bucketing; + +-- Now test empty inserts with blobstore optimizations turned off. This should give us same results. +SET hive.blobstore.optimizations.enabled=false; + +INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty; +SELECT COUNT(*) FROM blobstore_dynamic_partitioning; + +INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty; +SELECT COUNT(*) FROM blobstore_list_bucketing; http://git-wip-us.apache.org/repos/asf/hive/blob/84483e7d/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out new file mode 100644 index 0000000..4651899 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out @@ -0,0 +1,110 @@ +PREHOOK: query: DROP TABLE blobstore_source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_source +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE blobstore_source ( + a string, + b string, + c double) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_source +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: CREATE TABLE blobstore_source ( + a string, + b string, + c double) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: DROP TABLE blobstore_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE blobstore_table LIKE blobstore_source +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_table +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_table +POSTHOOK: query: CREATE TABLE blobstore_table LIKE blobstore_source +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/insert_blobstore_to_blobstore/blobstore_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_table +PREHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@blobstore_table +POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@blobstore_table +PREHOOK: query: SELECT COUNT(*) FROM blobstore_table +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM blobstore_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_table +#### A masked pattern was here #### +7 +PREHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@blobstore_table +POSTHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@blobstore_table +PREHOOK: query: SELECT COUNT(*) FROM blobstore_table +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM blobstore_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_table +#### A masked pattern was here #### +14 +PREHOOK: query: SELECT * FROM blobstore_table +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM blobstore_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_table +#### A masked pattern was here #### +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 http://git-wip-us.apache.org/repos/asf/hive/blob/84483e7d/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out new file mode 100644 index 0000000..8e5e096 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out @@ -0,0 +1,155 @@ +PREHOOK: query: DROP TABLE empty +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE empty +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE blobstore_dynamic_partitioning +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_dynamic_partitioning +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE blobstore_list_bucketing +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_list_bucketing +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE empty ( + id int, + name string, + dept string, + pt string, + dt string, + hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@empty +POSTHOOK: query: CREATE TABLE empty ( + id int, + name string, + dept string, + pt string, + dt string, + hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@empty +PREHOOK: query: CREATE TABLE blobstore_dynamic_partitioning ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_dynamic_partitioning +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_dynamic_partitioning +POSTHOOK: query: CREATE TABLE blobstore_dynamic_partitioning ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_dynamic_partitioning +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_dynamic_partitioning +PREHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty +PREHOOK: type: QUERY +PREHOOK: Input: default@empty +PREHOOK: Output: default@blobstore_dynamic_partitioning@pt=a +POSTHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='a', dt, hr) SELECT id, name, dept, dt, hr FROM empty +POSTHOOK: type: QUERY +POSTHOOK: Input: default@empty +PREHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_dynamic_partitioning +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_dynamic_partitioning +#### A masked pattern was here #### +0 +PREHOOK: query: CREATE TABLE blobstore_list_bucketing ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_list_bucketing +POSTHOOK: query: CREATE TABLE blobstore_list_bucketing ( + id int, + name string, + dept string) +PARTITIONED BY ( + pt string, + dt string, + hr string) +SKEWED BY (id) ON ('1', '2', '3') STORED AS DIRECTORIES +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/insert_empty_into_blobstore/blobstore_list_bucketing +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_list_bucketing +PREHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty +PREHOOK: type: QUERY +PREHOOK: Input: default@empty +PREHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a +POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', dt='a', hr='a') SELECT id, name, dept FROM empty +POSTHOOK: type: QUERY +POSTHOOK: Input: default@empty +POSTHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a +PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_list_bucketing +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_list_bucketing +#### A masked pattern was here #### +0 +PREHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty +PREHOOK: type: QUERY +PREHOOK: Input: default@empty +PREHOOK: Output: default@blobstore_dynamic_partitioning@pt=b +POSTHOOK: query: INSERT INTO TABLE blobstore_dynamic_partitioning PARTITION (pt='b', dt, hr) SELECT id, name, dept, dt, hr FROM empty +POSTHOOK: type: QUERY +POSTHOOK: Input: default@empty +PREHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_dynamic_partitioning +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM blobstore_dynamic_partitioning +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_dynamic_partitioning +#### A masked pattern was here #### +0 +PREHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty +PREHOOK: type: QUERY +PREHOOK: Input: default@empty +PREHOOK: Output: default@blobstore_list_bucketing@pt=b/dt=b/hr=b +POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='b', dt='b', hr='b') SELECT id, name, dept FROM empty +POSTHOOK: type: QUERY +POSTHOOK: Input: default@empty +POSTHOOK: Output: default@blobstore_list_bucketing@pt=b/dt=b/hr=b +POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).dept SIMPLE [(empty)empty.FieldSchema(name:dept, type:string, comment:null), ] +POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).id SIMPLE [(empty)empty.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=b,dt=b,hr=b).name SIMPLE [(empty)empty.FieldSchema(name:name, type:string, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_list_bucketing +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_list_bucketing +#### A masked pattern was here #### +0
