http://git-wip-us.apache.org/repos/asf/hive/blob/cb534ab0/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q new file mode 100644 index 0000000..6582035 --- /dev/null +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part.q @@ -0,0 +1,339 @@ +set hive.explain.user=true; +set hive.mapred.mode=nonstrict; +set hive.cli.print.header=true; +SET hive.exec.schema.evolution=true; +SET hive.vectorized.use.vectorized.input.format=false; +SET hive.vectorized.use.vector.serde.deserialize=true; +SET hive.vectorized.use.row.serde.deserialize=false; +set hive.fetch.task.conversion=none; +SET hive.vectorized.execution.enabled=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.metastore.disallow.incompatible.col.type.changes=true; +set hive.default.fileformat=textfile; +set hive.llap.io.enabled=false; + +-- SORT_QUERY_RESULTS +-- +-- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned +-- NOTE: the use of hive.vectorized.use.vector.serde.deserialize above which enables doing +-- vectorized reading of TEXTFILE format files using the vector SERDE methods. +-- + +CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) +row format delimited fields terminated by '|' stored as textfile; +load data local inpath '../../data/files/schema_evolution/schema_evolution_data.txt' overwrite into table schema_evolution_data; + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE ADD COLUMNS +-- +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS: INT PERMUTE SELECT +-- +-- +CREATE TABLE part_add_int_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT); + +insert into table part_add_int_permute_select partition(part=1) VALUES (1, 1111, 'new'); + +-- Table-Non-Cascade ADD COLUMNS ... +alter table part_add_int_permute_select add columns(c int); + +insert into table part_add_int_permute_select partition(part=1) VALUES (2, 2222, 'new', 3333); + +explain +select insert_num,part,a,b from part_add_int_permute_select; + +-- SELECT permutation columns to make sure NULL defaulting works right +select insert_num,part,a,b from part_add_int_permute_select; +select insert_num,part,a,b,c from part_add_int_permute_select; +select insert_num,part,c from part_add_int_permute_select; + +drop table part_add_int_permute_select; + + +-- SUBSECTION: ALTER TABLE ADD COLUMNS: INT, STRING, PERMUTE SELECT +-- +-- +CREATE TABLE part_add_int_string_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT); + +insert into table part_add_int_string_permute_select partition(part=1) VALUES (1, 1111, 'new'); + +-- Table-Non-Cascade ADD COLUMNS ... +alter table part_add_int_string_permute_select add columns(c int, d string); + +insert into table part_add_int_string_permute_select partition(part=1) VALUES (2, 2222, 'new', 3333, '4444'); + +explain +select insert_num,part,a,b from part_add_int_string_permute_select; + +-- SELECT permutation columns to make sure NULL defaulting works right +select insert_num,part,a,b from part_add_int_string_permute_select; +select insert_num,part,a,b,c from part_add_int_string_permute_select; +select insert_num,part,a,b,c,d from part_add_int_string_permute_select; +select insert_num,part,a,c,d from part_add_int_string_permute_select; +select insert_num,part,a,d from part_add_int_string_permute_select; +select insert_num,part,c from part_add_int_string_permute_select; +select insert_num,part,d from part_add_int_string_permute_select; + +drop table part_add_int_string_permute_select; + + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE: (STRING, CHAR, VARCHAR) +-- +CREATE TABLE part_change_string_group_double(insert_num int, c1 STRING, c2 CHAR(50), c3 VARCHAR(50), b STRING) PARTITIONED BY(part INT); + +insert into table part_change_string_group_double partition(part=1) SELECT insert_num, double_str, double_str, double_str, 'original' FROM schema_evolution_data; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_string_group_double replace columns (insert_num int, c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, b STRING); + +insert into table part_change_string_group_double partition(part=1) SELECT insert_num, double1, double1, double1, 'new' FROM schema_evolution_data WHERE insert_num = 111; + +explain +select insert_num,part,c1,c2,c3,b from part_change_string_group_double; + +select insert_num,part,c1,c2,c3,b from part_change_string_group_double; + +drop table part_change_string_group_double; + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP: DATE,TIMESTAMP, (STRING, CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) +-- +CREATE TABLE part_change_date_group_string_group_date_timestamp(insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, c5 DATE, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, b STRING) PARTITIONED BY(part INT); + +insert into table part_change_date_group_string_group_date_timestamp partition(part=1) SELECT insert_num, date1, date1, date1, date1, date1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, 'original' FROM schema_evolution_data; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_date_group_string_group_date_timestamp replace columns(insert_num int, c1 STRING, c2 CHAR(50), c3 CHAR(15), c4 VARCHAR(50), c5 VARCHAR(15), c6 STRING, c7 CHAR(50), c8 CHAR(15), c9 VARCHAR(50), c10 VARCHAR(15), b STRING); + +insert into table part_change_date_group_string_group_date_timestamp partition(part=1) VALUES (111, 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp; + +drop table part_change_date_group_string_group_date_timestamp; + + + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: +-- (TINYINT, SMALLINT, INT, BIGINT), STRING and +-- (TINYINT, SMALLINT, INT, BIGINT), CHAR and CHAR trunc and +-- (TINYINT, SMALLINT, INT, BIGINT), VARCHAR and VARCHAR trunc +-- +-- +CREATE TABLE part_change_numeric_group_string_group_multi_ints_string_group(insert_num int, + c1 tinyint, c2 smallint, c3 int, c4 bigint, + c5 tinyint, c6 smallint, c7 int, c8 bigint, c9 tinyint, c10 smallint, c11 int, c12 bigint, + c13 tinyint, c14 smallint, c15 int, c16 bigint, c17 tinyint, c18 smallint, c19 int, c20 bigint, + b STRING) PARTITIONED BY(part INT); + +insert into table part_change_numeric_group_string_group_multi_ints_string_group partition(part=1) SELECT insert_num, + tinyint1, smallint1, int1, bigint1, + tinyint1, smallint1, int1, bigint1, tinyint1, smallint1, int1, bigint1, + tinyint1, smallint1, int1, bigint1, tinyint1, smallint1, int1, bigint1, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_numeric_group_string_group_multi_ints_string_group replace columns (insert_num int, + c1 STRING, c2 STRING, c3 STRING, c4 STRING, + c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), c8 CHAR(50), c9 CHAR(5), c10 CHAR(5), c11 CHAR(5), c12 CHAR(5), + c13 VARCHAR(50), c14 VARCHAR(50), c15 VARCHAR(50), c16 VARCHAR(50), c17 VARCHAR(5), c18 VARCHAR(5), c19 VARCHAR(5), c20 VARCHAR(5), + b STRING) ; + +insert into table part_change_numeric_group_string_group_multi_ints_string_group partition(part=1) VALUES (111, + 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'new'); + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group; + +drop table part_change_numeric_group_string_group_multi_ints_string_group; + + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: +-- (DECIMAL, FLOAT, DOUBLE), STRING and +-- (DECIMAL, FLOAT, DOUBLE), CHAR and CHAR trunc and +-- (DECIMAL, FLOAT, DOUBLE), VARCHAR and VARCHAR trunc +-- +-- +CREATE TABLE part_change_numeric_group_string_group_floating_string_group(insert_num int, + c1 decimal(38,18), c2 float, c3 double, + c4 decimal(38,18), c5 float, c6 double, c7 decimal(38,18), c8 float, c9 double, + c10 decimal(38,18), c11 float, c12 double, c13 decimal(38,18), c14 float, c15 double, + b STRING) PARTITIONED BY(part INT); + +insert into table part_change_numeric_group_string_group_floating_string_group partition(part=1) SELECT insert_num, + decimal1, float1, double1, + decimal1, float1, double1, decimal1, float1, double1, + decimal1, float1, double1, decimal1, float1, double1, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_numeric_group_string_group_floating_string_group replace columns (insert_num int, + c1 STRING, c2 STRING, c3 STRING, + c4 CHAR(50), c5 CHAR(50), c6 CHAR(50), c7 CHAR(7), c8 CHAR(7), c9 CHAR(7), + c10 VARCHAR(50), c11 VARCHAR(50), c12 VARCHAR(50), c13 VARCHAR(7), c14 VARCHAR(7), c15 VARCHAR(7), + b STRING); + +insert into table part_change_numeric_group_string_group_floating_string_group partition(part=1) VALUES (111, + 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'new'); + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group; + +drop table part_change_numeric_group_string_group_floating_string_group; + + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP: STRING, (CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) and +-- CHAR, (VARCHAR, VARCHAR trunc, STRING) and VARCHAR, (CHAR, CHAR trunc, STRING) +-- +CREATE TABLE part_change_string_group_string_group_string(insert_num int, + c1 string, c2 string, c3 string, c4 string, + c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), + c8 VARCHAR(50), c9 VARCHAR(50), c10 VARCHAR(50), b STRING) PARTITIONED BY(part INT); + +insert into table part_change_string_group_string_group_string partition(part=1) SELECT insert_num, + string2, string2, string2, string2, + string2, string2, string2, + string2, string2, string2, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,b from part_change_string_group_string_group_string; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_string_group_string_group_string replace columns (insert_num int, + c1 CHAR(50), c2 CHAR(9), c3 VARCHAR(50), c4 CHAR(9), + c5 VARCHAR(50), c6 VARCHAR(9), c7 STRING, + c8 CHAR(50), c9 CHAR(9), c10 STRING, b STRING) ; + +insert into table part_change_string_group_string_group_string partition(part=1) VALUES (111, + 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', + 'new'); + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string; + +drop table part_change_string_group_string_group_string; + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: +-- TINYINT, (SMALLINT, INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and +-- SMALLINT, (INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and +-- INT, (BIGINT, DECIMAL, FLOAT, DOUBLE) and +-- BIGINT, (DECIMAL, FLOAT, DOUBLE) +-- +CREATE TABLE part_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_num int, + c1 tinyint, c2 tinyint, c3 tinyint, c4 tinyint, c5 tinyint, c6 tinyint, + c7 smallint, c8 smallint, c9 smallint, c10 smallint, c11 smallint, + c12 int, c13 int, c14 int, c15 int, + c16 bigint, c17 bigint, c18 bigint, + b STRING) PARTITIONED BY(part INT); + +insert into table part_change_lower_to_higher_numeric_group_tinyint_to_bigint partition(part=1) SELECT insert_num, + tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, + smallint1, smallint1, smallint1, smallint1, smallint1, + int1, int1, int1, int1, + bigint1, bigint1, bigint1, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_lower_to_higher_numeric_group_tinyint_to_bigint replace columns (insert_num int, + c1 SMALLINT, c2 INT, c3 BIGINT, c4 decimal(38,18), c5 FLOAT, c6 DOUBLE, + c7 INT, c8 BIGINT, c9 decimal(38,18), c10 FLOAT, c11 DOUBLE, + c12 BIGINT, c13 decimal(38,18), c14 FLOAT, c15 DOUBLE, + c16 decimal(38,18), c17 FLOAT, c18 DOUBLE, + b STRING) ; + +insert into table part_change_lower_to_higher_numeric_group_tinyint_to_bigint partition(part=1) VALUES (111, + 7000, 80000, 90000000, 1234.5678, 9876.543, 789.321, + 80000, 90000000, 1234.5678, 9876.543, 789.321, + 90000000, 1234.5678, 9876.543, 789.321, + 1234.5678, 9876.543, 789.321, + 'new'); + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from part_change_lower_to_higher_numeric_group_tinyint_to_bigint; + +drop table part_change_lower_to_higher_numeric_group_tinyint_to_bigint; + + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: +-- DECIMAL, (FLOAT, DOUBLE) and +-- FLOAT, (DOUBLE) +-- +CREATE TABLE part_change_lower_to_higher_numeric_group_decimal_to_float(insert_num int, + c1 decimal(38,18), c2 decimal(38,18), + c3 float, + b STRING) PARTITIONED BY(part INT); + +insert into table part_change_lower_to_higher_numeric_group_decimal_to_float partition(part=1) SELECT insert_num, + decimal1, decimal1, + float1, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace columns (insert_num int, c1 float, c2 double, c3 DOUBLE, b STRING) ; + +insert into table part_change_lower_to_higher_numeric_group_decimal_to_float partition(part=1) VALUES (111, 1234.5678, 9876.543, 1234.5678, 'new'); + +explain +select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; + +select insert_num,part,c1,c2,c3,b from part_change_lower_to_higher_numeric_group_decimal_to_float; + +drop table part_change_lower_to_higher_numeric_group_decimal_to_float; \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/cb534ab0/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q new file mode 100644 index 0000000..e38a01e --- /dev/null +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_complex.q @@ -0,0 +1,165 @@ +set hive.explain.user=true; +set hive.mapred.mode=nonstrict; +set hive.cli.print.header=true; +SET hive.exec.schema.evolution=true; +SET hive.vectorized.use.vectorized.input.format=false; +SET hive.vectorized.use.vector.serde.deserialize=true; +SET hive.vectorized.use.row.serde.deserialize=false; +set hive.fetch.task.conversion=none; +SET hive.vectorized.execution.enabled=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.metastore.disallow.incompatible.col.type.changes=false; +set hive.default.fileformat=textfile; +set hive.llap.io.enabled=false; + +-- SORT_QUERY_RESULTS +-- +-- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned --> all complex conversions +-- NOTE: the use of hive.vectorized.use.vector.serde.deserialize above which enables doing +-- vectorized reading of TEXTFILE format files using the vector SERDE methods. +-- +------------------------------------------------------------------------------------------ +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for Various --> Various: STRUCT<BOOLEAN, TINYINT, SMALLINT, INT, LONG, FLOAT, DOUBLE, DECIMAL(38,18), CHAR, VARCHAR, TIMESTAMP, DATE, BINARY> --> STRUCT<STRING... +-- +CREATE TABLE part_change_various_various_struct1(insert_num int, s1 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>, b STRING) PARTITIONED BY(part INT); + +CREATE TABLE complex_struct1_a_txt(insert_num int, s1 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>, b STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct1_a.txt' overwrite into table complex_struct1_a_txt; + +insert into table part_change_various_various_struct1 partition(part=1) select * from complex_struct1_a_txt; + +select insert_num,part,s1,b from part_change_various_various_struct1; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_various_various_struct1 replace columns (insert_num int, s1 STRUCT<c1:STRING, c2:STRING, c3:STRING, c4:STRING, c5:STRING, c6:STRING, c7:STRING, c8:STRING, c9:STRING, c10:STRING, c11:STRING, c12:STRING, c13:STRING>, b STRING); + +CREATE TABLE complex_struct1_b_txt(insert_num int, s1 STRUCT<c1:STRING, c2:STRING, c3:STRING, c4:STRING, c5:STRING, c6:STRING, c7:STRING, c8:STRING, c9:STRING, c10:STRING, c11:STRING, c12:STRING, c13:STRING>, b STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct1_b.txt' overwrite into table complex_struct1_b_txt; + +insert into table part_change_various_various_struct1 partition(part=2) select * from complex_struct1_b_txt; + +CREATE TABLE complex_struct1_c_txt(insert_num int, s1 STRUCT<c1:STRING, c2:STRING, c3:STRING, c4:STRING, c5:STRING, c6:STRING, c7:STRING, c8:STRING, c9:STRING, c10:STRING, c11:STRING, c12:STRING, c13:STRING>, b STRING) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct1_c.txt' overwrite into table complex_struct1_c_txt; + +insert into table part_change_various_various_struct1 partition(part=1) select * from complex_struct1_c_txt; + +explain +select insert_num,part,s1,b from part_change_various_various_struct1; + +select insert_num,part,s1,b from part_change_various_various_struct1; + +drop table part_change_various_various_struct1; + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: STRUCT +-- +CREATE TABLE part_add_various_various_struct2(insert_num int, b STRING) PARTITIONED BY(part INT); + +insert into table part_add_various_various_struct2 partition(part=1) + values(1, 'original'), + (2, 'original'); + +select insert_num,part,b from part_add_various_various_struct2; + +-- Table-Non-Cascade ADD COLUMN ... +alter table part_add_various_various_struct2 ADD columns (s2 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>); + +CREATE TABLE complex_struct2_a_txt(insert_num int, b STRING, s2 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct2_a.txt' overwrite into table complex_struct2_a_txt; + +insert into table part_add_various_various_struct2 partition(part=1) select * from complex_struct2_a_txt; + +CREATE TABLE complex_struct2_b_txt(insert_num int, b STRING, s2 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct2_b.txt' overwrite into table complex_struct2_b_txt; + +insert into table part_add_various_various_struct2 partition(part=2) select * from complex_struct2_b_txt; + +select insert_num,part,b,s2 from part_add_various_various_struct2; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_various_various_struct2 REPLACE columns (insert_num int, b STRING, s2 STRUCT<c1:STRING, c2:STRING, c3:STRING, c4:STRING, c5:STRING, c6:STRING, c7:STRING, c8:STRING, c9:STRING, c10:STRING, c11:STRING, c12:STRING, c13:STRING>); + +CREATE TABLE complex_struct2_c_txt(insert_num int, b STRING, s2 STRUCT<c1:STRING, c2:STRING, c3:STRING, c4:STRING, c5:STRING, c6:STRING, c7:STRING, c8:STRING, c9:STRING, c10:STRING, c11:STRING, c12:STRING, c13:STRING>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct2_c.txt' overwrite into table complex_struct2_c_txt; + +insert into table part_add_various_various_struct2 partition(part=2) select * from complex_struct2_c_txt; + +CREATE TABLE complex_struct2_d_txt(insert_num int, b STRING, s2 STRUCT<c1:STRING, c2:STRING, c3:STRING, c4:STRING, c5:STRING, c6:STRING, c7:STRING, c8:STRING, c9:STRING, c10:STRING, c11:STRING, c12:STRING, c13:STRING>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct2_d.txt' overwrite into table complex_struct2_d_txt; + +insert into table part_add_various_various_struct2 partition(part=1) select * from complex_struct2_d_txt; + +explain +select insert_num,part,b,s2 from part_add_various_various_struct2; + +select insert_num,part,b,s2 from part_add_various_various_struct2; + +drop table part_add_various_various_struct2; + + + + +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS for Various --> Various: ADD COLUMNS to STRUCT type as LAST column of 3 columns +-- +CREATE TABLE part_add_to_various_various_struct4(insert_num int, b STRING, s3 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT>) PARTITIONED BY(part INT); + +CREATE TABLE complex_struct4_a_txt(insert_num int, b STRING, s3 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct4_a.txt' overwrite into table complex_struct4_a_txt; + +insert into table part_add_to_various_various_struct4 partition(part=1) select * from complex_struct4_a_txt; + +select insert_num,part,b,s3 from part_add_to_various_various_struct4; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_add_to_various_various_struct4 replace columns (insert_num int, b STRING, s3 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>); + +CREATE TABLE complex_struct4_b_txt(insert_num int, b STRING, s3 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct4_b.txt' overwrite into table complex_struct4_b_txt; + +insert into table part_add_to_various_various_struct4 partition(part=2) select * from complex_struct4_b_txt; + +CREATE TABLE complex_struct4_c_txt(insert_num int, b STRING, s3 STRUCT<c1:BOOLEAN, c2:TINYINT, c3:SMALLINT, c4:INT, c5:BIGINT, c6:FLOAT, c7:DOUBLE, c8:DECIMAL(38,18), c9:CHAR(25), c10:VARCHAR(25), c11:TIMESTAMP, c12:DATE, c13:BINARY>) +row format delimited fields terminated by '|' +collection items terminated by ',' +map keys terminated by ':' stored as textfile; +load data local inpath '../../data/files/schema_evolution/complex_struct4_c.txt' overwrite into table complex_struct4_c_txt; + +insert into table part_add_to_various_various_struct4 partition(part=1) select * from complex_struct4_c_txt; + +explain +select insert_num,part,b,s3 from part_add_to_various_various_struct4; + +select insert_num,part,b,s3 from part_add_to_various_various_struct4; + +drop table part_add_to_various_various_struct4; http://git-wip-us.apache.org/repos/asf/hive/blob/cb534ab0/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q new file mode 100644 index 0000000..c9d90c3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_part_all_primitive.q @@ -0,0 +1,208 @@ +set hive.explain.user=true; +set hive.mapred.mode=nonstrict; +set hive.cli.print.header=true; +SET hive.exec.schema.evolution=true; +SET hive.vectorized.use.vectorized.input.format=false; +SET hive.vectorized.use.vector.serde.deserialize=true; +SET hive.vectorized.use.row.serde.deserialize=false; +set hive.fetch.task.conversion=none; +SET hive.vectorized.execution.enabled=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.metastore.disallow.incompatible.col.type.changes=false; +set hive.default.fileformat=textfile; +set hive.llap.io.enabled=false; + +-- SORT_QUERY_RESULTS +-- +-- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned --> all primitive conversions +-- NOTE: the use of hive.vectorized.use.vector.serde.deserialize above which enables doing +-- vectorized reading of TEXTFILE format files using the vector SERDE methods. +-- +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS Various --> Various +-- +-- + +CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) +row format delimited fields terminated by '|' stored as textfile; +load data local inpath '../../data/files/schema_evolution/schema_evolution_data.txt' overwrite into table schema_evolution_data; + +CREATE TABLE schema_evolution_data_2(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) +row format delimited fields terminated by '|' stored as textfile; +load data local inpath '../../data/files/schema_evolution/schema_evolution_data_2.txt' overwrite into table schema_evolution_data_2; + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for Various --> Various: +-- (BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DECIMAL, STRING, TIMESTAMP) --> BOOLEAN and +-- (BOOLEAN, SHORT, INT, LONG, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, TIMESTAMP) --> BYTE 128 and a maximum value of 127 and +-- (BOOLEAN, TINYINT, INT, LONG, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, TIMESTAMP) --> SMALLINT -32768 and a maximum value of 32767 and +-- (BOOLEAN, TINYINT, SMALLINT, LONG, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, TIMESTAMP) --> INT â2147483648 to 2147483647 and +-- (BOOLEAN, TINYINT, SMALLINT, INT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, TIMESTAMP) --> BIGINT -9223372036854775808 to 9223372036854775807 +-- +CREATE TABLE part_change_various_various_boolean_to_bigint(insert_num int, + c1 TINYINT, c2 SMALLINT, c3 INT, c4 BIGINT, c5 FLOAT, c6 DOUBLE, c7 DECIMAL(38,18), c8 STRING, c9 TIMESTAMP, + c10 BOOLEAN, c11 SMALLINT, c12 INT, c13 BIGINT, c14 FLOAT, c15 DOUBLE, c16 DECIMAL(38,18), c17 STRING, c18 CHAR(25), c19 VARCHAR(25), c20 TIMESTAMP, + c21 BOOLEAN, c22 TINYINT, c23 INT, c24 BIGINT, c25 FLOAT, c26 DOUBLE, c27 DECIMAL(38,18), c28 STRING, c29 CHAR(25), c30 VARCHAR(25), c31 TIMESTAMP, + c32 BOOLEAN, c33 TINYINT, c34 SMALLINT, c35 BIGINT, c36 FLOAT, c37 DOUBLE, c38 DECIMAL(38,18), c39 STRING, c40 CHAR(25), c41 VARCHAR(25), c42 TIMESTAMP, + c43 BOOLEAN, c44 TINYINT, c45 SMALLINT, c46 INT, c47 FLOAT, c48 DOUBLE, c49 DECIMAL(38,18), c50 STRING, c51 CHAR(25), c52 VARCHAR(25), c53 TIMESTAMP, + b STRING) PARTITIONED BY(part INT); + +insert into table part_change_various_various_boolean_to_bigint partition(part=1) SELECT insert_num, + tinyint1, smallint1, int1, bigint1, float1, double1, decimal1, boolean_str, timestamp1, + boolean1, smallint1, int1, bigint1, float1, double1, decimal1, tinyint_str, tinyint_str, tinyint_str, timestamp1, + boolean1, tinyint1, int1, bigint1, float1, double1, decimal1, smallint_str, smallint_str, smallint_str, timestamp1, + boolean1, tinyint1, smallint1, bigint1, float1, double1, decimal1, int_str, int_str, int_str, timestamp1, + boolean1, tinyint1, smallint1, int1, float1, double1, decimal1, bigint_str, bigint_str, bigint_str, timestamp1, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_various_various_boolean_to_bigint replace columns (insert_num int, + c1 BOOLEAN, c2 BOOLEAN, c3 BOOLEAN, c4 BOOLEAN, c5 BOOLEAN, c6 BOOLEAN, c7 BOOLEAN, c8 BOOLEAN, c9 BOOLEAN, + c10 TINYINT, c11 TINYINT, c12 TINYINT, c13 TINYINT, c14 TINYINT, c15 TINYINT, c16 TINYINT, c17 TINYINT, c18 TINYINT, c19 TINYINT, c20 TINYINT, + c21 SMALLINT, c22 SMALLINT, c23 SMALLINT, c24 SMALLINT, c25 SMALLINT, c26 SMALLINT, c27 SMALLINT, c28 SMALLINT, c29 SMALLINT, c30 SMALLINT, c31 SMALLINT, + c32 INT, c33 INT, c34 INT, c35 INT, c36 INT, c37 INT, c38 INT, c39 INT, c40 INT, c41 INT, c42 INT, + c43 BIGINT, c44 BIGINT, c45 BIGINT, c46 BIGINT, c47 BIGINT, c48 BIGINT, c49 BIGINT, c50 BIGINT, c51 BIGINT, c52 BIGINT, c53 BIGINT, + b STRING); + +insert into table part_change_various_various_boolean_to_bigint partition(part=1) SELECT insert_num, + boolean1, boolean1, boolean1, boolean1, boolean1, boolean1, boolean1, boolean1, boolean1, + tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, + smallint1, smallint1, smallint1, smallint1, smallint1, smallint1, smallint1, smallint1, smallint1, smallint1, smallint1, + int1, int1, int1, int1, int1, int1, int1, int1, int1, int1, int1, + bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, bigint1, + 'new' FROM schema_evolution_data; + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint; + +drop table part_change_various_various_boolean_to_bigint; + + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for Various --> Various: +-- (BOOLEAN, TINYINT, SMALLINT, INT, LONG, FLOAT, DOUBLE, STRING, CHAR, VARCHAR, TIMESTAMP) --> DECIMAL +-- (BOOLEAN, TINYINT, SMALLINT, INT, LONG, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, TIMESTAMP) --> FLOAT and +-- (BOOLEAN, TINYINT, SMALLINT, INT, LONG, FLOAT, DECIMAL, STRING, CHAR, VARCHAR, TIMESTAMP) --> DOUBLE and +-- +CREATE TABLE part_change_various_various_decimal_to_double(insert_num int, + c1 BOOLEAN, c2 TINYINT, c3 SMALLINT, c4 INT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 STRING, c9 CHAR(25), c10 VARCHAR(25), c11 TIMESTAMP, + c12 BOOLEAN, c13 TINYINT, c14 SMALLINT, c15 INT, c16 BIGINT, c17 DECIMAL(38,18), c18 DOUBLE, c19 STRING, c20 CHAR(25), c21 VARCHAR(25), c22 TIMESTAMP, + c23 BOOLEAN, c24 TINYINT, c25 SMALLINT, c26 INT, c27 BIGINT, c28 DECIMAL(38,18), c29 FLOAT, c30 STRING, c31 CHAR(25), c32 VARCHAR(25), c33 TIMESTAMP, + b STRING) PARTITIONED BY(part INT); + +insert into table part_change_various_various_decimal_to_double partition(part=1) SELECT insert_num, + boolean1, tinyint1, smallint1, int1, bigint1, float1, double1, decimal_str, decimal_str, decimal_str, timestamp1, + boolean1, tinyint1, smallint1, int1, bigint1, decimal1, double1, float_str, float_str, float_str, timestamp1, + boolean1, tinyint1, smallint1, int1, bigint1, decimal1, float1, double_str, double_str, double_str, timestamp1, + 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_various_various_decimal_to_double replace columns (insert_num int, + c1 DECIMAL(38,18), c2 DECIMAL(38,18), c3 DECIMAL(38,18), c4 DECIMAL(38,18), c5 DECIMAL(38,18), c6 DECIMAL(38,18), c7 DECIMAL(38,18), c8 DECIMAL(38,18), c9 DECIMAL(38,18), c10 DECIMAL(38,18), c11 DECIMAL(38,18), + c12 FLOAT, c13 FLOAT, c14 FLOAT, c15 FLOAT, c16 FLOAT, c17 FLOAT, c18 FLOAT, c19 FLOAT, c20 FLOAT, c21 FLOAT, c22 FLOAT, + c23 DOUBLE, c24 DOUBLE, c25 DOUBLE, c26 DOUBLE, c27 DOUBLE, c28 DOUBLE, c29 DOUBLE, c30 DOUBLE, c31 DOUBLE, c32 DOUBLE, c33 DOUBLE, + b STRING); + +insert into table part_change_various_various_decimal_to_double partition(part=1) SELECT insert_num, + decimal1, decimal1, decimal1, decimal1, decimal1, decimal1, decimal1, decimal1, decimal1, decimal1, decimal1, + float1, float1, float1, float1, float1, float1, float1, float1, float1, float1, float1, + double1, double1, double1, double1, double1, double1, double1, double1, double1, double1, double1, + 'new' FROM schema_evolution_data_2 WHERE insert_num=111; + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double; + +drop table part_change_various_various_decimal_to_double; + + + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for Various --> Various: (BOOLEAN, TINYINT, SMALLINT, INT, LONG, FLOAT, DOUBLE, DECIMAL(38,18), STRING, CHAR, VARCHAR, DATE) --> TIMESTAMP +-- +CREATE TABLE part_change_various_various_timestamp(insert_num int, c1 BOOLEAN, c2 TINYINT, c3 SMALLINT, c4 INT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 DECIMAL(38,18), c9 STRING, c10 CHAR(25), c11 VARCHAR(25), c12 DATE, b STRING) PARTITIONED BY(part INT); + +insert into table part_change_various_various_timestamp partition(part=1) SELECT insert_num, boolean1, tinyint1, smallint1, int1, bigint1, float1, double1, decimal1, timestamp_str, timestamp_str, timestamp_str, date1, 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_various_various_timestamp replace columns (insert_num int, c1 TIMESTAMP, c2 TIMESTAMP, c3 TIMESTAMP, c4 TIMESTAMP, c5 TIMESTAMP, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, c11 TIMESTAMP, c12 TIMESTAMP, b STRING); + +insert into table part_change_various_various_timestamp partition(part=1) SELECT insert_num, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; + +select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp; + +drop table part_change_various_various_timestamp; + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for Various --> Various: (STRING, CHAR, VARCHAR, TIMESTAMP --> DATE +-- +CREATE TABLE part_change_various_various_date(insert_num int, c1 STRING, c2 CHAR(25), c3 VARCHAR(25), c4 TIMESTAMP, b STRING) PARTITIONED BY(part INT); + +insert into table part_change_various_various_date partition(part=1) SELECT insert_num, date_str, date_str, date_str, timestamp1, 'original' FROM schema_evolution_data; + +select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_various_various_date replace columns (insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, b STRING); + +insert into table part_change_various_various_date partition(part=1) SELECT insert_num, date1, date1, date1, date1, 'new' FROM schema_evolution_data_2 WHERE insert_num=111; + +explain +select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; + +select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date; + +drop table part_change_various_various_date; + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for Same Type (CHAR, VARCHAR, DECIMAL) --> Different maxLength or precision/scale +-- +CREATE TABLE part_change_same_type_different_params(insert_num int, c1 CHAR(12), c2 CHAR(25), c3 VARCHAR(25), c4 VARCHAR(10), c5 DECIMAL(12,4), c6 DECIMAL(20,10), b STRING) PARTITIONED BY(part INT); + +CREATE TABLE same_type1_a_txt(insert_num int, c1 CHAR(12), c2 CHAR(25), c3 VARCHAR(25), c4 VARCHAR(10), c5 DECIMAL(12,4), c6 DECIMAL(20,10), b STRING) +row format delimited fields terminated by '|' +stored as textfile; +load data local inpath '../../data/files/schema_evolution/same_type1_a.txt' overwrite into table same_type1_a_txt; + +insert into table part_change_same_type_different_params partition(part=1) select * from same_type1_a_txt; + +select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table part_change_same_type_different_params replace columns (insert_num int, c1 CHAR(8), c2 CHAR(32), c3 VARCHAR(15), c4 VARCHAR(18), c5 DECIMAL(10,2), c6 DECIMAL(25,15), b STRING); + +CREATE TABLE same_type1_b_txt(insert_num int, c1 CHAR(8), c2 CHAR(32), c3 VARCHAR(15), c4 VARCHAR(18), c5 DECIMAL(10,2), c6 DECIMAL(25,15), b STRING) +row format delimited fields terminated by '|' +stored as textfile; +load data local inpath '../../data/files/schema_evolution/same_type1_b.txt' overwrite into table same_type1_b_txt; + +insert into table part_change_same_type_different_params partition(part=1) select * from same_type1_b_txt; + +CREATE TABLE same_type1_c_txt(insert_num int, c1 CHAR(8), c2 CHAR(32), c3 VARCHAR(15), c4 VARCHAR(18), c5 DECIMAL(10,2), c6 DECIMAL(25,15), b STRING) +row format delimited fields terminated by '|' +stored as textfile; +load data local inpath '../../data/files/schema_evolution/same_type1_c.txt' overwrite into table same_type1_c_txt; + +insert into table part_change_same_type_different_params partition(part=2) select * from same_type1_c_txt; + +explain +select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; + +select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params; + +drop table part_change_same_type_different_params; http://git-wip-us.apache.org/repos/asf/hive/blob/cb534ab0/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q b/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q new file mode 100644 index 0000000..7785f87 --- /dev/null +++ b/ql/src/test/queries/clientpositive/schema_evol_text_vec_table.q @@ -0,0 +1,325 @@ +set hive.explain.user=true; +set hive.cli.print.header=true; +SET hive.exec.schema.evolution=true; +SET hive.vectorized.use.vectorized.input.format=false; +SET hive.vectorized.use.vector.serde.deserialize=true; +SET hive.vectorized.use.row.serde.deserialize=false; +set hive.fetch.task.conversion=none; +SET hive.vectorized.execution.enabled=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.metastore.disallow.incompatible.col.type.changes=true; +set hive.default.fileformat=textfile; +set hive.llap.io.enabled=false; + +-- SORT_QUERY_RESULTS +-- +-- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Table +-- NOTE: the use of hive.vectorized.use.vector.serde.deserialize above which enables doing +-- vectorized reading of TEXTFILE format files using the vector SERDE methods. +-- + +CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) +row format delimited fields terminated by '|' stored as textfile; +load data local inpath '../../data/files/schema_evolution/schema_evolution_data.txt' overwrite into table schema_evolution_data; + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE ADD COLUMNS +-- +-- +-- SUBSECTION: ALTER TABLE ADD COLUMNS: INT PERMUTE SELECT +-- +-- +CREATE TABLE table_add_int_permute_select(insert_num int, a INT, b STRING); + +insert into table table_add_int_permute_select SELECT insert_num, int1, 'original' FROM schema_evolution_data; + +-- Table-Non-Cascade ADD COLUMNS ... +alter table table_add_int_permute_select add columns(c int); + +insert into table table_add_int_permute_select VALUES (111, 80000, 'new', 80000); + +explain +select insert_num,a,b from table_add_int_permute_select; + +-- SELECT permutation columns to make sure NULL defaulting works right +select insert_num,a,b from table_add_int_permute_select; +select insert_num,a,b,c from table_add_int_permute_select; +select insert_num,c from table_add_int_permute_select; + +drop table table_add_int_permute_select; + + +-- SUBSECTION: ALTER TABLE ADD COLUMNS: INT, STRING, PERMUTE SELECT +-- +-- +CREATE TABLE table_add_int_string_permute_select(insert_num int, a INT, b STRING); + +insert into table table_add_int_string_permute_select SELECT insert_num, int1, 'original' FROM schema_evolution_data; + +-- Table-Non-Cascade ADD COLUMNS ... +alter table table_add_int_string_permute_select add columns(c int, d string); + +insert into table table_add_int_string_permute_select VALUES (111, 80000, 'new', 80000, 'filler'); + +explain +select insert_num,a,b from table_add_int_string_permute_select; + +-- SELECT permutation columns to make sure NULL defaulting works right +select insert_num,a,b from table_add_int_string_permute_select; +select insert_num,a,b,c from table_add_int_string_permute_select; +select insert_num,a,b,c,d from table_add_int_string_permute_select; +select insert_num,a,c,d from table_add_int_string_permute_select; +select insert_num,a,d from table_add_int_string_permute_select; +select insert_num,c from table_add_int_string_permute_select; +select insert_num,d from table_add_int_string_permute_select; + +drop table table_add_int_string_permute_select; + + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE: (STRING, CHAR, VARCHAR) +-- +CREATE TABLE table_change_string_group_double(insert_num int, c1 STRING, c2 CHAR(50), c3 VARCHAR(50), b STRING); + +insert into table table_change_string_group_double SELECT insert_num, double_str, double_str, double_str, 'original' FROM schema_evolution_data; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_string_group_double replace columns (insert_num int, c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, b STRING); + +insert into table table_change_string_group_double VALUES (111, 789.321, 789.321, 789.321, 'new'); + +explain +select insert_num,c1,c2,c3,b from table_change_string_group_double; + +select insert_num,c1,c2,c3,b from table_change_string_group_double; + +drop table table_change_string_group_double; + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP: DATE,TIMESTAMP, (STRING, CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) +-- +CREATE TABLE table_change_date_group_string_group_date_group(insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, c5 DATE, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, b STRING); + +insert into table table_change_date_group_string_group_date_group SELECT insert_num, date1, date1, date1, date1, date1, timestamp1, timestamp1, timestamp1, timestamp1, timestamp1, 'original' FROM schema_evolution_data; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_date_group_string_group_date_group replace columns(insert_num int, c1 STRING, c2 CHAR(50), c3 CHAR(15), c4 VARCHAR(50), c5 VARCHAR(15), c6 STRING, c7 CHAR(50), c8 CHAR(15), c9 VARCHAR(50), c10 VARCHAR(15), b STRING); + +insert into table table_change_date_group_string_group_date_group VALUES (111, 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'new'); + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from table_change_date_group_string_group_date_group; + +drop table table_change_date_group_string_group_date_group; + + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: +-- (TINYINT, SMALLINT, INT, BIGINT), STRING and +-- (TINYINT, SMALLINT, INT, BIGINT), CHAR and CHAR trunc and +-- (TINYINT, SMALLINT, INT, BIGINT), VARCHAR and VARCHAR trunc +-- +-- +CREATE TABLE table_change_numeric_group_string_group_multi_ints_string_group(insert_num int, + c1 tinyint, c2 smallint, c3 int, c4 bigint, + c5 tinyint, c6 smallint, c7 int, c8 bigint, c9 tinyint, c10 smallint, c11 int, c12 bigint, + c13 tinyint, c14 smallint, c15 int, c16 bigint, c17 tinyint, c18 smallint, c19 int, c20 bigint, + b STRING); + +insert into table table_change_numeric_group_string_group_multi_ints_string_group SELECT insert_num, + tinyint1, smallint1, int1, bigint1, + tinyint1, smallint1, int1, bigint1, tinyint1, smallint1, int1, bigint1, + tinyint1, smallint1, int1, bigint1, tinyint1, smallint1, int1, bigint1, + 'original' FROM schema_evolution_data; + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_numeric_group_string_group_multi_ints_string_group replace columns (insert_num int, + c1 STRING, c2 STRING, c3 STRING, c4 STRING, + c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), c8 CHAR(50), c9 CHAR(5), c10 CHAR(5), c11 CHAR(5), c12 CHAR(5), + c13 VARCHAR(50), c14 VARCHAR(50), c15 VARCHAR(50), c16 VARCHAR(50), c17 VARCHAR(5), c18 VARCHAR(5), c19 VARCHAR(5), c20 VARCHAR(5), + b STRING) ; + +insert into table table_change_numeric_group_string_group_multi_ints_string_group VALUES (111, + 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'new'); + +explain +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from table_change_numeric_group_string_group_multi_ints_string_group; + +drop table table_change_numeric_group_string_group_multi_ints_string_group; + + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: +-- (DECIMAL, FLOAT, DOUBLE), STRING and +-- (DECIMAL, FLOAT, DOUBLE), CHAR and CHAR trunc and +-- (DECIMAL, FLOAT, DOUBLE), VARCHAR and VARCHAR trunc +-- +-- +CREATE TABLE table_change_numeric_group_string_group_floating_string_group(insert_num int, + c1 decimal(38,18), c2 float, c3 double, + c4 decimal(38,18), c5 float, c6 double, c7 decimal(38,18), c8 float, c9 double, + c10 decimal(38,18), c11 float, c12 double, c13 decimal(38,18), c14 float, c15 double, + b STRING); + +insert into table table_change_numeric_group_string_group_floating_string_group SELECT insert_num, + decimal1, float1, double1, + decimal1, float1, double1, decimal1, float1, double1, + decimal1, float1, double1, decimal1, float1, double1, + 'original' FROM schema_evolution_data; + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_numeric_group_string_group_floating_string_group replace columns (insert_num int, + c1 STRING, c2 STRING, c3 STRING, + c4 CHAR(50), c5 CHAR(50), c6 CHAR(50), c7 CHAR(7), c8 CHAR(7), c9 CHAR(7), + c10 VARCHAR(50), c11 VARCHAR(50), c12 VARCHAR(50), c13 VARCHAR(7), c14 VARCHAR(7), c15 VARCHAR(7), + b STRING); + +insert into table table_change_numeric_group_string_group_floating_string_group VALUES (111, + 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', 'filler', 'filler', 'filler', + 'new'); + +explain +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group; + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from table_change_numeric_group_string_group_floating_string_group; + +drop table table_change_numeric_group_string_group_floating_string_group; + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP: STRING, (CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) and +-- CHAR, (VARCHAR, VARCHAR trunc, STRING) and VARCHAR, (CHAR, CHAR trunc, STRING) +-- +CREATE TABLE table_change_string_group_string_group_string(insert_num int, + c1 string, c2 string, c3 string, c4 string, + c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), + c8 VARCHAR(50), c9 VARCHAR(50), c10 VARCHAR(50), b STRING); + +insert into table table_change_string_group_string_group_string SELECT insert_num, + string2, string2, string2, string2, + string2, string2, string2, + string2, string2, string2, + 'original' FROM schema_evolution_data; + +select insert_num,c1,c2,c3,c4,b from table_change_string_group_string_group_string; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_string_group_string_group_string replace columns (insert_num int, + c1 CHAR(50), c2 CHAR(9), c3 VARCHAR(50), c4 CHAR(9), + c5 VARCHAR(50), c6 VARCHAR(9), c7 STRING, + c8 CHAR(50), c9 CHAR(9), c10 STRING, b STRING) ; + +insert into table table_change_string_group_string_group_string VALUES (111, + 'filler', 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', + 'filler', 'filler', 'filler', + 'new'); + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from table_change_string_group_string_group_string; + +drop table table_change_string_group_string_group_string; + + + +------------------------------------------------------------------------------------------ +-- SECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP +-- +-- +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: +-- TINYINT, (SMALLINT, INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and +-- SMALLINT, (INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and +-- INT, (BIGINT, DECIMAL, FLOAT, DOUBLE) and +-- BIGINT, (DECIMAL, FLOAT, DOUBLE) +-- +CREATE TABLE table_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_num int, + c1 tinyint, c2 tinyint, c3 tinyint, c4 tinyint, c5 tinyint, c6 tinyint, + c7 smallint, c8 smallint, c9 smallint, c10 smallint, c11 smallint, + c12 int, c13 int, c14 int, c15 int, + c16 bigint, c17 bigint, c18 bigint, + b STRING); + +insert into table table_change_lower_to_higher_numeric_group_tinyint_to_bigint SELECT insert_num, + tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, tinyint1, + smallint1, smallint1, smallint1, smallint1, smallint1, + int1, int1, int1, int1, + bigint1, bigint1, bigint1, + 'original' FROM schema_evolution_data; + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from table_change_lower_to_higher_numeric_group_tinyint_to_bigint; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_lower_to_higher_numeric_group_tinyint_to_bigint replace columns (insert_num int, + c1 SMALLINT, c2 INT, c3 BIGINT, c4 decimal(38,18), c5 FLOAT, c6 DOUBLE, + c7 INT, c8 BIGINT, c9 decimal(38,18), c10 FLOAT, c11 DOUBLE, + c12 BIGINT, c13 decimal(38,18), c14 FLOAT, c15 DOUBLE, + c16 decimal(38,18), c17 FLOAT, c18 DOUBLE, + b STRING) ; + +insert into table table_change_lower_to_higher_numeric_group_tinyint_to_bigint VALUES (111, + 7000, 80000, 90000000, 1234.5678, 9876.543, 789.321, + 80000, 90000000, 1234.5678, 9876.543, 789.321, + 90000000, 1234.5678, 9876.543, 789.321, + 1234.5678, 9876.543, 789.321, + 'new'); + +select insert_num,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,b from table_change_lower_to_higher_numeric_group_tinyint_to_bigint; + +drop table table_change_lower_to_higher_numeric_group_tinyint_to_bigint; + + + +-- +-- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: +-- DECIMAL, (FLOAT, DOUBLE) and +-- FLOAT, (DOUBLE) +-- +CREATE TABLE table_change_lower_to_higher_numeric_group_decimal_to_float(insert_num int, + c1 decimal(38,18), c2 decimal(38,18), + c3 float, + b STRING); + +insert into table table_change_lower_to_higher_numeric_group_decimal_to_float SELECT insert_num, + decimal1, decimal1, + float1, + 'original' FROM schema_evolution_data; + +select insert_num,c1,c2,c3,b from table_change_lower_to_higher_numeric_group_decimal_to_float; + +-- Table-Non-Cascade CHANGE COLUMNS ... +alter table table_change_lower_to_higher_numeric_group_decimal_to_float replace columns (insert_num int, c1 float, c2 double, c3 DOUBLE, b STRING) ; + +insert into table table_change_lower_to_higher_numeric_group_decimal_to_float VALUES (111, 1234.5678, 9876.543, 1234.5678, 'new'); + +select insert_num,c1,c2,c3,b from table_change_lower_to_higher_numeric_group_decimal_to_float; + +drop table table_change_lower_to_higher_numeric_group_decimal_to_float;
