This is an automated email from the ASF dual-hosted git repository. mbod pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new a550003 HIVE-25658: Addendum: Fix regex for masking totalSize table properties in Iceberg q-tests (Marton Bod, reviewed by Peter Vary) a550003 is described below commit a5500035c12d94de9094d880822776d276817439 Author: Marton Bod <m...@cloudera.com> AuthorDate: Fri Nov 19 23:41:17 2021 +0100 HIVE-25658: Addendum: Fix regex for masking totalSize table properties in Iceberg q-tests (Marton Bod, reviewed by Peter Vary) --- .../test/queries/positive/create_iceberg_table.q | 3 +++ .../create_iceberg_table_stored_as_fileformat.q | 3 +++ .../create_iceberg_table_stored_by_iceberg.q | 3 +++ ...g_table_stored_by_iceberg_with_serdeproperties.q | 3 +++ .../positive/describe_iceberg_metadata_tables.q | 3 +++ .../test/queries/positive/describe_iceberg_table.q | 3 +++ .../query_iceberg_metadata_of_partitioned_table.q | 4 ++++ .../query_iceberg_metadata_of_unpartitioned_table.q | 4 ++++ .../positive/query_iceberg_metadata_with_joins.q | 3 +++ .../queries/positive/truncate_force_iceberg_table.q | 4 ++-- .../test/queries/positive/truncate_iceberg_table.q | 4 ++-- .../results/positive/create_iceberg_table.q.out | 2 +- .../create_iceberg_table_stored_as_fileformat.q.out | 10 +++++----- .../create_iceberg_table_stored_by_iceberg.q.out | 2 +- ...ble_stored_by_iceberg_with_serdeproperties.q.out | 2 +- .../results/positive/describe_iceberg_table.q.out | 8 ++++---- ...ry_iceberg_metadata_of_unpartitioned_table.q.out | Bin 29346 -> 29566 bytes .../positive/truncate_force_iceberg_table.q.out | 2 +- .../results/positive/truncate_iceberg_table.q.out | 4 ++-- 19 files changed, 48 insertions(+), 19 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q index a86d642..880b92b 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q @@ -1,3 +1,6 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ + set hive.vectorized.execution.enabled=false; CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY ICEBERG; DESCRIBE FORMATTED ice_t; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q index bc6adb8..ab1a309 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q @@ -1,3 +1,6 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ + set hive.vectorized.execution.enabled=false; DROP TABLE IF EXISTS ice_orc; CREATE EXTERNAL TABLE ice_orc (i int, s string, ts timestamp, d date) STORED BY ICEBERG STORED AS ORC; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q index f932901..7b6f12a 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q @@ -1,3 +1,6 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ + set hive.vectorized.execution.enabled=false; CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY ICEBERG; DESCRIBE FORMATTED ice_t; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q index 7c44eb6..47c1f52 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q @@ -1,3 +1,6 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ + set hive.vectorized.execution.enabled=false; CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY ICEBERG WITH SERDEPROPERTIES('write.format.default'='orc'); DESCRIBE FORMATTED ice_t; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q index 6fd7daa..fbab1c9 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q @@ -1,3 +1,6 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ + drop table if exists ice_meta_desc; create external table ice_meta_desc (id int, value string) stored by iceberg stored as orc; insert into ice_meta_desc values (1, 'one'),(2,'two'),(3,'three'),(4,'four'),(5,'five'); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q index 53ee461..371c41c 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q @@ -1,3 +1,6 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ + DROP TABLE IF EXISTS ice_t; CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY ICEBERG; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q index 2f456d3..1c1ef7a 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q @@ -1,4 +1,8 @@ -- SORT_QUERY_RESULTS +-- Mask the file size values as it can have slight variability, causing test flakiness +--! qt:replace:/("file_size_in_bytes":)\d+/$1#Masked#/ +--! qt:replace:/("total-files-size":)\d+/$1#Masked#/ +--! qt:replace:/((ORC|PARQUET|AVRO)\s+\d+\s+)\d+/$1#Masked#/ set hive.vectorized.execution.enabled = false; set tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q index f6d0ed0..2272417 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q @@ -1,4 +1,8 @@ -- SORT_QUERY_RESULTS +-- Mask the file size values as it can have slight variability, causing test flakiness +--! qt:replace:/("file_size_in_bytes":)\d+/$1#Masked#/ +--! qt:replace:/("total-files-size":)\d+/$1#Masked#/ +--! qt:replace:/((ORC|PARQUET|AVRO)\s+\d+\s+)\d+/$1#Masked#/ set hive.vectorized.execution.enabled = false; set tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q index ef745e8..3c6d83c 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q @@ -1,4 +1,7 @@ -- SORT_QUERY_RESULTS +-- Mask the file size values as it can have slight variability, causing test flakiness +--! qt:replace:/("file_size_in_bytes":)\d+/$1#Masked#/ +--! qt:replace:/("total-files-size":)\d+/$1#Masked#/ set hive.vectorized.execution.enabled = false; set tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q index 77e394d..b1c002e 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q @@ -1,6 +1,6 @@ -- SORT_QUERY_RESULTS --- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes ---! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ set hive.vectorized.execution.enabled=false; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q index 277f1c1..a6a0e4e 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q @@ -1,6 +1,6 @@ -- SORT_QUERY_RESULTS --- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes ---! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ set hive.vectorized.execution.enabled=false; diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out index eedb423..42a347a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out @@ -37,7 +37,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out index 78f4205..c7393a6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out @@ -41,7 +41,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### write.format.default ORC @@ -104,7 +104,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### write.format.default PARQUET @@ -167,7 +167,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### write.format.default AVRO @@ -230,7 +230,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### write.format.default AVRO @@ -290,7 +290,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### write.format.default ORC diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out index eedb423..42a347a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out @@ -37,7 +37,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out index d7b43ed..d4a62cd 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out @@ -37,7 +37,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### write.format.default orc diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out index 6bfe2f6..a20fec6 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out @@ -77,7 +77,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -132,7 +132,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -188,7 +188,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -232,7 +232,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information diff --git a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out index d65def6..e06df09 100644 Binary files a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out and b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out differ diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out index 9ad42d5..1fd52a5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out @@ -99,7 +99,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 1998 + totalSize #Masked# #### A masked pattern was here #### write.format.default parquet diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out index a170985..af44a6d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out @@ -99,7 +99,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 1096 + totalSize #Masked# #### A masked pattern was here #### write.format.default orc @@ -229,7 +229,7 @@ Table Parameters: serialization.format 1 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG - totalSize 371 + totalSize #Masked# #### A masked pattern was here #### write.format.default orc