HIVE-20423: Set NULLS LAST as the default null ordering (Teddy Choi, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0427ffa5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0427ffa5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0427ffa5 Branch: refs/heads/branch-3 Commit: 0427ffa57b5fc8a7e399fa6b6d868f9b989c1373 Parents: fdc12f3 Author: Teddy Choi <[email protected]> Authored: Wed Sep 26 16:51:12 2018 +0900 Committer: Teddy Choi <[email protected]> Committed: Wed Sep 26 16:51:15 2018 +0900 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 + .../write_final_output_blobstore.q.out | 4 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 9 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 4 +- .../apache/hadoop/hive/ql/parse/HiveParser.g | 14 +- .../hadoop/hive/ql/plan/PTFDeserializer.java | 4 +- .../hive/ql/udf/generic/GenericUDAFAverage.java | 16 +- .../ql/udf/generic/GenericUDAFEvaluator.java | 11 +- .../hive/ql/udf/generic/GenericUDAFSum.java | 18 +- .../hive/ql/udf/ptf/BasePartitionEvaluator.java | 49 +- .../hive/ql/udf/ptf/TableFunctionEvaluator.java | 9 + .../hive/ql/udf/ptf/TableFunctionResolver.java | 5 +- .../hive/ql/udf/ptf/ValueBoundaryScanner.java | 93 +- .../hive/ql/udf/ptf/WindowingTableFunction.java | 4 +- .../clientpositive/beeline/smb_mapjoin_13.q.out | 4 +- ...names_with_leading_and_trailing_spaces.q.out | 2 +- .../clientpositive/correlationoptimizer14.q.out | 37 - .../results/clientpositive/ctas_colname.q.out | 4 +- .../test/results/clientpositive/decimal_3.q.out | 8 +- .../test/results/clientpositive/decimal_4.q.out | 4 +- .../test/results/clientpositive/decimal_5.q.out | 8 +- .../test/results/clientpositive/decimal_6.q.out | 12 +- .../clientpositive/decimal_precision.q.out | 286 ++--- .../results/clientpositive/decimal_serde.q.out | 4 +- .../clientpositive/delete_all_partitioned.q.out | 2 +- .../clientpositive/distinct_windowing.q.out | 6 +- .../distinct_windowing_no_cbo.q.out | 10 +- .../groupby_grouping_window.q.out | 2 +- .../results/clientpositive/input_part7.q.out | 4 +- .../insert_values_non_partitioned.q.out | 2 +- .../clientpositive/limit_pushdown2.q.out | 40 +- .../clientpositive/llap/acid_no_buckets.q.out | 16 +- .../llap/acid_vectorization_original.q.out | 2 +- .../llap/bucket_map_join_tez2.q.out | 2 - .../clientpositive/llap/bucketmapjoin7.q.out | 2 +- .../results/clientpositive/llap/cbo_limit.q.out | 8 +- .../clientpositive/llap/cbo_rp_limit.q.out | 8 +- ...names_with_leading_and_trailing_spaces.q.out | 2 +- .../llap/delete_all_partitioned.q.out | 2 +- .../clientpositive/llap/explainuser_1.q.out | 88 +- .../llap/external_jdbc_table.q.out | 1 - .../llap/groupby_resolution.q.out | 2 +- .../llap/insert_into_with_schema.q.out | 4 +- .../llap/insert_values_non_partitioned.q.out | 2 +- .../clientpositive/llap/limit_pushdown.q.out | 8 +- .../clientpositive/llap/limit_pushdown3.q.out | 8 +- .../results/clientpositive/llap/lineage2.q.out | 2 +- .../results/clientpositive/llap/lineage3.q.out | 10 +- .../results/clientpositive/llap/llap_acid.q.out | 6 +- .../clientpositive/llap/llap_acid_fast.q.out | 6 +- .../clientpositive/llap/llap_smb_ptf.q.out | 4 +- .../llap/materialized_view_partitioned_2.q.out | 3 - .../llap/materialized_view_partitioned_3.q.out | 1 - .../llap/materialized_view_rewrite_part_2.q.out | 30 +- .../llap/offset_limit_ppd_optimizer.q.out | 8 +- .../clientpositive/llap/orc_llap_counters.q.out | 82 +- .../llap/orc_llap_counters1.q.out | 10 +- .../clientpositive/llap/orc_ppd_basic.q.out | 100 +- .../llap/orc_ppd_schema_evol_3a.q.out | 118 +-- .../clientpositive/llap/order_null.q.out | 4 +- .../test/results/clientpositive/llap/ptf.q.out | 140 +-- .../clientpositive/llap/ptf_matchpath.q.out | 6 +- .../clientpositive/llap/ptf_streaming.q.out | 94 +- .../llap/results_cache_with_masking.q.out | 1 - .../clientpositive/llap/sharedworkext.q.out | 4 +- .../clientpositive/llap/skewjoinopt15.q.out | 4 +- .../clientpositive/llap/smb_mapjoin_15.q.out | 8 +- .../clientpositive/llap/subquery_in.q.out | 6 +- .../llap/subquery_in_having.q.out | 2 +- .../clientpositive/llap/subquery_notin.q.out | 14 +- .../clientpositive/llap/subquery_scalar.q.out | 4 +- .../results/clientpositive/llap/sysdb.q.out | 14 +- .../llap/tez_dynpart_hashjoin_1.q.out | 4 +- .../llap/tez_dynpart_hashjoin_2.q.out | 12 +- .../llap/tez_fixed_bucket_pruning.q.out | 24 +- .../llap/tez_vector_dynpart_hashjoin_1.q.out | 4 +- .../llap/tez_vector_dynpart_hashjoin_2.q.out | 12 +- .../llap/union_assertion_type.q.out | 1 - .../llap/update_all_partitioned.q.out | 2 +- .../clientpositive/llap/update_tmp_table.q.out | 2 +- .../llap/update_where_partitioned.q.out | 4 +- .../clientpositive/llap/vector_between_in.q.out | 16 +- .../llap/vector_case_when_2.q.out | 12 +- .../clientpositive/llap/vector_coalesce.q.out | 60 +- .../clientpositive/llap/vector_coalesce_4.q.out | 4 +- .../clientpositive/llap/vector_data_types.q.out | 16 +- .../clientpositive/llap/vector_date_1.q.out | 20 +- .../clientpositive/llap/vector_decimal_1.q.out | 36 +- .../llap/vector_decimal_10_0.q.out | 8 +- .../clientpositive/llap/vector_decimal_2.q.out | 32 +- .../clientpositive/llap/vector_decimal_3.q.out | 8 +- .../clientpositive/llap/vector_decimal_4.q.out | 8 +- .../clientpositive/llap/vector_decimal_5.q.out | 8 +- .../clientpositive/llap/vector_decimal_6.q.out | 42 +- .../llap/vector_decimal_expressions.q.out | 4 +- .../llap/vector_decimal_precision.q.out | 510 ++++----- .../llap/vector_decimal_round.q.out | 12 +- .../llap/vector_decimal_round_2.q.out | 8 +- .../llap/vector_decimal_trailing.q.out | 2 +- .../llap/vector_decimal_udf.q.out | 4 +- .../vector_groupby_grouping_sets_grouping.q.out | 4 +- .../vector_groupby_grouping_sets_limit.q.out | 50 +- .../llap/vector_groupby_grouping_window.q.out | 4 +- .../clientpositive/llap/vector_interval_2.q.out | 8 +- .../llap/vector_leftsemi_mapjoin.q.out | 114 +- .../clientpositive/llap/vector_like_2.q.out | 6 +- .../llap/vector_llap_io_data_conversion.q.out | 2 +- .../llap/vector_llap_text_1.q.out | 2 +- .../clientpositive/llap/vector_order_null.q.out | 8 +- .../llap/vector_outer_join1.q.out | 44 +- .../llap/vector_outer_join2.q.out | 44 +- .../llap/vector_outer_join3.q.out | 26 +- .../llap/vector_outer_join4.q.out | 60 +- .../llap/vector_outer_reference_windowed.q.out | 28 +- .../llap/vector_ptf_part_simple.q.out | 60 +- .../llap/vector_string_concat.q.out | 2 +- .../clientpositive/llap/vector_topnkey.q.out | 10 +- .../clientpositive/llap/vector_windowing.q.out | 246 +++-- .../llap/vector_windowing_expressions.q.out | 96 +- .../llap/vector_windowing_gby.q.out | 4 +- .../llap/vector_windowing_gby2.q.out | 22 +- .../vector_windowing_multipartitioning.q.out | 20 +- .../llap/vector_windowing_navfn.q.out | 38 +- .../llap/vector_windowing_order_null.q.out | 12 +- .../vector_windowing_range_multiorder.q.out | 24 +- .../llap/vector_windowing_rank.q.out | 18 +- .../llap/vector_windowing_streaming.q.out | 110 +- .../llap/vector_windowing_windowspec.q.out | 32 +- .../llap/vector_windowing_windowspec4.q.out | 2 +- .../clientpositive/llap/vectorization_0.q.out | 14 +- .../clientpositive/llap/vectorization_12.q.out | 2 +- .../clientpositive/llap/vectorization_13.q.out | 162 +-- .../clientpositive/llap/vectorization_14.q.out | 2 +- .../clientpositive/llap/vectorization_17.q.out | 2 +- .../clientpositive/llap/vectorization_7.q.out | 52 +- .../clientpositive/llap/vectorization_8.q.out | 2 +- .../llap/vectorization_div0.q.out | 260 ++--- .../llap/vectorization_limit.q.out | 16 +- .../llap/vectorization_part_project.q.out | 22 +- .../llap/vectorization_short_regress.q.out | 252 ++--- .../clientpositive/llap/vectorized_ptf.q.out | 144 +-- .../llap/vectorized_timestamp_funcs.q.out | 260 ++--- .../results/clientpositive/llap/windowing.q.out | 2 +- .../clientpositive/llap/windowing_gby.q.out | 2 +- .../results/clientpositive/localtimezone.q.out | 12 +- .../results/clientpositive/masking_mv.q.out | 6 - .../offset_limit_global_optimizer.q.out | 16 +- .../outer_reference_windowed.q.out | 8 +- .../parquet_vectorization_0.q.out | 2 +- .../parquet_vectorization_13.q.out | 160 +-- .../parquet_vectorization_7.q.out | 50 +- .../parquet_vectorization_div0.q.out | 10 +- .../parquet_vectorization_limit.q.out | 4 +- .../parquet_vectorization_part_project.q.out | 20 +- .../partition_vs_table_metadata.q.out | 1000 +++++++++--------- ql/src/test/results/clientpositive/pcr.q.out | 34 +- ql/src/test/results/clientpositive/pcs.q.out | 6 +- .../clientpositive/perf/spark/query47.q.out | 6 +- .../clientpositive/perf/spark/query49.q.out | 12 +- .../clientpositive/perf/spark/query51.q.out | 6 +- .../clientpositive/perf/spark/query57.q.out | 6 +- .../clientpositive/perf/tez/query51.q.out | 6 +- .../results/clientpositive/pointlookup2.q.out | 20 +- .../results/clientpositive/pointlookup3.q.out | 20 +- .../results/clientpositive/pointlookup4.q.out | 4 +- ql/src/test/results/clientpositive/ppd_vc.q.out | 2 +- .../results/clientpositive/ptf_matchpath.q.out | 6 +- .../test/results/clientpositive/push_or.q.out | 2 +- .../results/clientpositive/quotedid_basic.q.out | 4 +- .../test/results/clientpositive/sample6.q.out | 14 +- .../test/results/clientpositive/semijoin2.q.out | 2 +- .../test/results/clientpositive/semijoin4.q.out | 2 +- .../test/results/clientpositive/semijoin5.q.out | 2 +- .../results/clientpositive/serde_regex.q.out | 2 +- .../clientpositive/skewjoin_mapjoin1.q.out | 4 +- .../clientpositive/skewjoin_mapjoin10.q.out | 4 +- .../clientpositive/skewjoin_mapjoin2.q.out | 4 +- .../skewjoin_union_remove_1.q.out | 8 +- .../results/clientpositive/skewjoinopt1.q.out | 4 +- .../results/clientpositive/skewjoinopt3.q.out | 4 +- .../results/clientpositive/smb_mapjoin_13.q.out | 4 +- .../spark/bucket_map_join_tez2.q.out | 12 +- .../clientpositive/spark/bucketmapjoin7.q.out | 2 +- .../spark/bucketmapjoin7.q.out_spark | 2 +- .../clientpositive/spark/cbo_limit.q.out | 8 +- .../spark/constprog_semijoin.q.out | 52 +- .../spark/groupby_resolution.q.out | 2 +- .../clientpositive/spark/limit_pushdown.q.out | 8 +- .../clientpositive/spark/limit_pushdown2.q.out | 40 +- .../spark/parquet_vectorization_0.q.out | 14 +- .../spark/parquet_vectorization_12.q.out | 2 +- .../spark/parquet_vectorization_13.q.out | 162 +-- .../spark/parquet_vectorization_14.q.out | 2 +- .../spark/parquet_vectorization_17.q.out | 2 +- .../spark/parquet_vectorization_7.q.out | 52 +- .../spark/parquet_vectorization_8.q.out | 2 +- .../spark/parquet_vectorization_div0.q.out | 10 +- .../spark/parquet_vectorization_limit.q.out | 12 +- .../parquet_vectorization_part_project.q.out | 20 +- .../test/results/clientpositive/spark/pcr.q.out | 34 +- .../test/results/clientpositive/spark/ptf.q.out | 142 +-- .../clientpositive/spark/ptf_matchpath.q.out | 6 +- .../clientpositive/spark/ptf_streaming.q.out | 94 +- .../results/clientpositive/spark/sample6.q.out | 14 +- .../spark/skewjoin_union_remove_1.q.out | 8 +- .../clientpositive/spark/skewjoinopt1.q.out | 4 +- .../clientpositive/spark/skewjoinopt15.q.out | 4 +- .../clientpositive/spark/skewjoinopt3.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_13.q.out | 4 +- .../clientpositive/spark/smb_mapjoin_15.q.out | 8 +- .../spark/spark_explainuser_1.q.out | 90 +- .../clientpositive/spark/subquery_in.q.out | 4 +- .../clientpositive/spark/subquery_notin.q.out | 14 +- .../clientpositive/spark/subquery_scalar.q.out | 4 +- .../clientpositive/spark/union_ppr.q.out | 2 +- .../spark/union_remove_6_subq.q.out | 2 +- .../spark/vector_between_in.q.out | 16 +- .../spark/vector_data_types.q.out | 16 +- .../spark/vector_outer_join1.q.out | 70 +- .../spark/vector_outer_join2.q.out | 58 +- .../spark/vector_outer_join3.q.out | 26 +- .../spark/vector_outer_join4.q.out | 60 +- .../spark/vector_string_concat.q.out | 2 +- .../clientpositive/spark/vectorization_0.q.out | 14 +- .../clientpositive/spark/vectorization_12.q.out | 2 +- .../clientpositive/spark/vectorization_13.q.out | 162 +-- .../clientpositive/spark/vectorization_14.q.out | 2 +- .../clientpositive/spark/vectorization_17.q.out | 2 +- .../spark/vectorization_div0.q.out | 260 ++--- .../spark/vectorization_part_project.q.out | 32 +- .../spark/vectorization_short_regress.q.out | 252 ++--- .../clientpositive/spark/vectorized_ptf.q.out | 146 +-- .../spark/vectorized_timestamp_funcs.q.out | 288 ++--- .../clientpositive/spark/windowing.q.out | 2 +- .../subquery_unqualcolumnrefs.q.out | 4 +- .../tez/acid_vectorization_original_tez.q.out | 2 +- .../clientpositive/udtf_json_tuple.q.out | 6 +- .../clientpositive/udtf_parse_url_tuple.q.out | 6 +- .../test/results/clientpositive/union_ppr.q.out | 4 +- .../clientpositive/union_remove_6_subq.q.out | 2 +- .../clientpositive/update_all_partitioned.q.out | 2 +- .../clientpositive/update_tmp_table.q.out | 2 +- .../update_where_partitioned.q.out | 4 +- .../clientpositive/vector_case_when_2.q.out | 6 +- .../clientpositive/vector_coalesce.q.out | 60 +- .../clientpositive/vector_coalesce_4.q.out | 2 +- .../clientpositive/vector_data_types.q.out | 16 +- .../results/clientpositive/vector_date_1.q.out | 10 +- .../clientpositive/vector_decimal_1.q.out | 18 +- .../clientpositive/vector_decimal_10_0.q.out | 4 +- .../clientpositive/vector_decimal_3.q.out | 8 +- .../clientpositive/vector_decimal_4.q.out | 8 +- .../clientpositive/vector_decimal_5.q.out | 8 +- .../clientpositive/vector_decimal_6.q.out | 34 +- .../vector_decimal_precision.q.out | 510 ++++----- .../clientpositive/vector_outer_join1.q.out | 70 +- .../clientpositive/vector_outer_join2.q.out | 58 +- .../clientpositive/vector_outer_join3.q.out | 32 +- .../clientpositive/vector_outer_join4.q.out | 66 +- .../clientpositive/vector_string_concat.q.out | 2 +- .../clientpositive/vectorization_13.q.out | 160 +-- .../clientpositive/vectorization_7.q.out | 50 +- .../clientpositive/vectorization_limit.q.out | 6 +- .../vectorization_part_project.q.out | 32 +- .../vectorized_timestamp_funcs.q.out | 288 ++--- .../results/clientpositive/windowing_gby2.q.out | 16 +- .../clientpositive/windowing_navfn.q.out | 16 +- .../clientpositive/windowing_order_null.q.out | 6 +- .../clientpositive/windowing_streaming.q.out | 98 +- .../clientpositive/windowing_windowspec3.q.out | 56 +- 270 files changed, 4997 insertions(+), 5081 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index fefa864..f9bb551 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1702,6 +1702,8 @@ public class HiveConf extends Configuration { "If the bucketing/sorting properties of the table exactly match the grouping key, whether to perform \n" + "the group by in the mapper by using BucketizedHiveInputFormat. The only downside to this\n" + "is that it limits the number of mappers to the number of files."), + HIVE_DEFAULT_NULLS_LAST("hive.default.nulls.last", true, + "Whether to set NULLS LAST as the default null ordering"), HIVE_GROUPBY_POSITION_ALIAS("hive.groupby.position.alias", false, "Whether to enable using Column Position Alias in Group By"), HIVE_ORDERBY_POSITION_ALIAS("hive.orderby.position.alias", true, http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out ---------------------------------------------------------------------- diff --git a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out index b6fff60..eb08d57 100644 --- a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out +++ b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out @@ -150,7 +150,7 @@ STAGE PLANS: GatherStats: false Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE tag: -1 @@ -425,7 +425,7 @@ STAGE PLANS: GatherStats: false Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE tag: -1 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index ebea31d..c9a4696 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1326,13 +1326,8 @@ public abstract class BaseSemanticAnalyzer { ASTNode child = (ASTNode) ast.getChild(i); if (child.getToken().getType() == HiveParser.TOK_TABSORTCOLNAMEASC) { child = (ASTNode) child.getChild(0); - if (child.getToken().getType() == HiveParser.TOK_NULLS_FIRST) { - colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()).toLowerCase(), - HIVE_COLUMN_ORDER_ASC)); - } else { - throw new SemanticException("create/alter table: " - + "not supported NULLS LAST for ORDER BY in ASC order"); - } + colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()).toLowerCase(), + HIVE_COLUMN_ORDER_ASC)); } else { child = (ASTNode) child.getChild(0); if (child.getToken().getType() == HiveParser.TOK_NULLS_LAST) { http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 3a096f7..5dec70f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1442,12 +1442,14 @@ public class CalcitePlanner extends SemanticAnalyzer { * @return Optimized SQL text (or null, if failed) */ public String getOptimizedSql(RelNode optimizedOptiqPlan) { + boolean nullsLast = HiveConf.getBoolVar(conf, ConfVars.HIVE_DEFAULT_NULLS_LAST); + NullCollation nullCollation = nullsLast ? NullCollation.LAST : NullCollation.LOW; SqlDialect dialect = new HiveSqlDialect(SqlDialect.EMPTY_CONTEXT .withDatabaseProduct(SqlDialect.DatabaseProduct.HIVE) .withDatabaseMajorVersion(4) // TODO: should not be hardcoded .withDatabaseMinorVersion(0) .withIdentifierQuoteString("`") - .withNullCollation(NullCollation.LOW)) { + .withNullCollation(nullCollation)) { @Override protected boolean allowsAs() { return true; http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 445a978..d8c66f9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -760,6 +760,12 @@ import org.apache.hadoop.hive.conf.HiveConf; public void setHiveConf(Configuration hiveConf) { this.hiveConf = hiveConf; } + protected boolean nullsLast() { + if(hiveConf == null){ + return false; + } + return HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_DEFAULT_NULLS_LAST); + } } @rulecatch { @@ -2352,7 +2358,9 @@ columnNameOrder @init { pushMsg("column name order", state); } @after { popMsg(state); } : identifier orderSpec=orderSpecification? nullSpec=nullOrdering? - -> {$orderSpec.tree == null && $nullSpec.tree == null}? + -> {$orderSpec.tree == null && $nullSpec.tree == null && nullsLast()}? + ^(TOK_TABSORTCOLNAMEASC ^(TOK_NULLS_LAST identifier)) + -> {$orderSpec.tree == null && $nullSpec.tree == null && !nullsLast()}? ^(TOK_TABSORTCOLNAMEASC ^(TOK_NULLS_FIRST identifier)) -> {$orderSpec.tree == null}? ^(TOK_TABSORTCOLNAMEASC ^($nullSpec identifier)) @@ -2382,7 +2390,9 @@ columnRefOrder @init { pushMsg("column order", state); } @after { popMsg(state); } : expression orderSpec=orderSpecification? nullSpec=nullOrdering? - -> {$orderSpec.tree == null && $nullSpec.tree == null}? + -> {$orderSpec.tree == null && $nullSpec.tree == null && nullsLast()}? + ^(TOK_TABSORTCOLNAMEASC ^(TOK_NULLS_LAST expression)) + -> {$orderSpec.tree == null && $nullSpec.tree == null && !nullsLast()}? ^(TOK_TABSORTCOLNAMEASC ^(TOK_NULLS_FIRST expression)) -> {$orderSpec.tree == null}? ^(TOK_TABSORTCOLNAMEASC ^($nullSpec expression)) http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java index 1cd6b95..863a34e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java @@ -110,7 +110,7 @@ public class PTFDeserializer { TableFunctionEvaluator tEval = def.getTFunction(); WindowingTableFunctionResolver tResolver = (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName()); - tResolver.initialize(ptfDesc, def, tEval); + tResolver.initialize(hConf, ptfDesc, def, tEval); /* @@ -171,7 +171,7 @@ public class PTFDeserializer { TableFunctionEvaluator tEval = def.getTFunction(); // TableFunctionResolver tResolver = FunctionRegistry.getTableFunctionResolver(def.getName()); TableFunctionResolver tResolver = constructResolver(def.getResolverClassName()); - tResolver.initialize(ptfDesc, def, tEval); + tResolver.initialize(hConf, ptfDesc, def, tEval); /* * 3. give Evaluator chance to setup for RawInput execution; setup RawInput shape http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java index d170d86..b7f333d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java @@ -235,11 +235,13 @@ public class GenericUDAFAverage extends AbstractGenericUDAFResolver { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { + ObjectInspector outputOI, + boolean nullsLast) { try { - return new BasePartitionEvaluator.AvgPartitionDoubleEvaluator(this, winFrame, partition, parameters, inputOI, outputOI); + return new BasePartitionEvaluator.AvgPartitionDoubleEvaluator(this, winFrame, partition, + parameters, inputOI, outputOI, nullsLast); } catch(HiveException e) { - return super.createPartitionEvaluator(winFrame, partition, parameters, outputOI); + return super.createPartitionEvaluator(winFrame, partition, parameters, outputOI, nullsLast); } } } @@ -393,11 +395,13 @@ public class GenericUDAFAverage extends AbstractGenericUDAFResolver { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { + ObjectInspector outputOI, + boolean nullsLast) { try { - return new BasePartitionEvaluator.AvgPartitionHiveDecimalEvaluator(this, winFrame, partition, parameters, inputOI, outputOI); + return new BasePartitionEvaluator.AvgPartitionHiveDecimalEvaluator(this, winFrame, + partition, parameters, inputOI, outputOI, nullsLast); } catch(HiveException e) { - return super.createPartitionEvaluator(winFrame, partition, parameters, outputOI); + return super.createPartitionEvaluator(winFrame, partition, parameters, outputOI, nullsLast); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java index 3a3e4b6..09e2583 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java @@ -304,6 +304,7 @@ public abstract class GenericUDAFEvaluator implements Closeable { * @param partition the partition data * @param parameters the list of the expressions in the function * @param outputOI the output object inspector + * @param nullsLast the nulls last configuration * @return the evaluator, default to BasePartitionEvaluator which * implements the naive approach */ @@ -311,9 +312,10 @@ public abstract class GenericUDAFEvaluator implements Closeable { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { + ObjectInspector outputOI, boolean nullsLast) { if (partitionEvaluator == null) { - partitionEvaluator = createPartitionEvaluator(winFrame, partition, parameters, outputOI); + partitionEvaluator = createPartitionEvaluator(winFrame, partition, parameters, outputOI, + nullsLast); } return partitionEvaluator; @@ -327,7 +329,8 @@ public abstract class GenericUDAFEvaluator implements Closeable { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { - return new BasePartitionEvaluator(this, winFrame, partition, parameters, outputOI); + ObjectInspector outputOI, + boolean nullsLast) { + return new BasePartitionEvaluator(this, winFrame, partition, parameters, outputOI, nullsLast); } } http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 1439b64..32e44da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -367,8 +367,10 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { - return new BasePartitionEvaluator.SumPartitionHiveDecimalEvaluator(this, winFrame, partition, parameters, outputOI); + ObjectInspector outputOI, + boolean nullsLast) { + return new BasePartitionEvaluator.SumPartitionHiveDecimalEvaluator(this, winFrame, + partition, parameters, outputOI, nullsLast); } } @@ -498,8 +500,10 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { - return new BasePartitionEvaluator.SumPartitionDoubleEvaluator(this, winFrame, partition, parameters, outputOI); + ObjectInspector outputOI, + boolean nullsLast) { + return new BasePartitionEvaluator.SumPartitionDoubleEvaluator(this, winFrame, partition, + parameters, outputOI, nullsLast); } } @@ -624,8 +628,10 @@ public class GenericUDAFSum extends AbstractGenericUDAFResolver { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { - return new BasePartitionEvaluator.SumPartitionLongEvaluator(this, winFrame, partition, parameters, outputOI); + ObjectInspector outputOI, + boolean nullsLast) { + return new BasePartitionEvaluator.SumPartitionLongEvaluator(this, winFrame, partition, + parameters, outputOI, nullsLast); } } } http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/BasePartitionEvaluator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/BasePartitionEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/BasePartitionEvaluator.java index ac83969..d44604d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/BasePartitionEvaluator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/BasePartitionEvaluator.java @@ -54,6 +54,7 @@ public class BasePartitionEvaluator { protected final PTFPartition partition; protected final List<PTFExpressionDef> parameters; protected final ObjectInspector outputOI; + protected final boolean nullsLast; /** * Internal class to represent a window range in a partition by searching the @@ -181,12 +182,14 @@ public class BasePartitionEvaluator { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { + ObjectInspector outputOI, + boolean nullsLast) { this.wrappedEvaluator = wrappedEvaluator; this.winFrame = winFrame; this.partition = partition; this.parameters = parameters; this.outputOI = outputOI; + this.nullsLast = nullsLast; } /** @@ -206,7 +209,7 @@ public class BasePartitionEvaluator { * @throws HiveException */ public Object iterate(int currentRow, LeadLagInfo leadLagInfo) throws HiveException { - Range range = getRange(winFrame, currentRow, partition); + Range range = getRange(winFrame, currentRow, partition, nullsLast); PTFPartitionIterator<Object> pItr = range.iterator(); return calcFunctionValue(pItr, leadLagInfo); } @@ -242,8 +245,8 @@ public class BasePartitionEvaluator { return ObjectInspectorUtils.copyToStandardObject(wrappedEvaluator.evaluate(aggBuffer), outputOI); } - protected static Range getRange(WindowFrameDef winFrame, int currRow, PTFPartition p) - throws HiveException { + protected static Range getRange(WindowFrameDef winFrame, int currRow, PTFPartition p, + boolean nullsLast) throws HiveException { BoundaryDef startB = winFrame.getStart(); BoundaryDef endB = winFrame.getEnd(); @@ -252,7 +255,7 @@ public class BasePartitionEvaluator { start = getRowBoundaryStart(startB, currRow); end = getRowBoundaryEnd(endB, currRow, p); } else { - ValueBoundaryScanner vbs = ValueBoundaryScanner.getScanner(winFrame); + ValueBoundaryScanner vbs = ValueBoundaryScanner.getScanner(winFrame, nullsLast); start = vbs.computeStart(currRow, p); end = vbs.computeEnd(currRow, p); } @@ -323,8 +326,9 @@ public class BasePartitionEvaluator { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + ObjectInspector outputOI, + boolean nullsLast) { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); sumAgg = new WindowSumAgg<ResultType>(); } @@ -336,7 +340,7 @@ public class BasePartitionEvaluator { return super.iterate(currentRow, leadLagInfo); } - Range currentRange = getRange(winFrame, currentRow, partition); + Range currentRange = getRange(winFrame, currentRow, partition, nullsLast); ResultType result; if (currentRow == 0 || // Reset for the new partition sumAgg.prevRange == null || @@ -365,8 +369,8 @@ public class BasePartitionEvaluator { public static class SumPartitionDoubleEvaluator extends SumPartitionEvaluator<DoubleWritable> { public SumPartitionDoubleEvaluator(GenericUDAFEvaluator wrappedEvaluator, WindowFrameDef winFrame, PTFPartition partition, - List<PTFExpressionDef> parameters, ObjectInspector outputOI) { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + List<PTFExpressionDef> parameters, ObjectInspector outputOI, boolean nullsLast) { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); this.typeOperation = new TypeOperationDoubleWritable(); } } @@ -374,8 +378,8 @@ public class BasePartitionEvaluator { public static class SumPartitionLongEvaluator extends SumPartitionEvaluator<LongWritable> { public SumPartitionLongEvaluator(GenericUDAFEvaluator wrappedEvaluator, WindowFrameDef winFrame, PTFPartition partition, - List<PTFExpressionDef> parameters, ObjectInspector outputOI) { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + List<PTFExpressionDef> parameters, ObjectInspector outputOI, boolean nullsLast) { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); this.typeOperation = new TypeOperationLongWritable(); } } @@ -383,8 +387,8 @@ public class BasePartitionEvaluator { public static class SumPartitionHiveDecimalEvaluator extends SumPartitionEvaluator<HiveDecimalWritable> { public SumPartitionHiveDecimalEvaluator(GenericUDAFEvaluator wrappedEvaluator, WindowFrameDef winFrame, PTFPartition partition, - List<PTFExpressionDef> parameters, ObjectInspector outputOI) { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + List<PTFExpressionDef> parameters, ObjectInspector outputOI, boolean nullsLast) { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); this.typeOperation = new TypeOperationHiveDecimalWritable(); } } @@ -411,8 +415,9 @@ public class BasePartitionEvaluator { WindowFrameDef winFrame, PTFPartition partition, List<PTFExpressionDef> parameters, - ObjectInspector outputOI) { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + ObjectInspector outputOI, + boolean nullsLast) { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); } /** @@ -453,7 +458,7 @@ public class BasePartitionEvaluator { return super.iterate(currentRow, leadLagInfo); } - Range currentRange = getRange(winFrame, currentRow, partition); + Range currentRange = getRange(winFrame, currentRow, partition, nullsLast); if (currentRow == 0 || // Reset for the new partition avgAgg.prevRange == null || currentRange.getSize() <= currentRange.getDiff(avgAgg.prevRange)) { @@ -485,8 +490,9 @@ public class BasePartitionEvaluator { public AvgPartitionDoubleEvaluator(GenericUDAFEvaluator wrappedEvaluator, WindowFrameDef winFrame, PTFPartition partition, - List<PTFExpressionDef> parameters, ObjectInspector inputOI, ObjectInspector outputOI) throws HiveException { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + List<PTFExpressionDef> parameters, ObjectInspector inputOI, ObjectInspector outputOI, + boolean nullsLast) throws HiveException { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); this.typeOperation = new TypeOperationDoubleWritable(); } } @@ -495,8 +501,9 @@ public class BasePartitionEvaluator { public AvgPartitionHiveDecimalEvaluator(GenericUDAFEvaluator wrappedEvaluator, WindowFrameDef winFrame, PTFPartition partition, - List<PTFExpressionDef> parameters, ObjectInspector inputOI, ObjectInspector outputOI) throws HiveException { - super(wrappedEvaluator, winFrame, partition, parameters, outputOI); + List<PTFExpressionDef> parameters, ObjectInspector inputOI, ObjectInspector outputOI, + boolean nullsLast) throws HiveException { + super(wrappedEvaluator, winFrame, partition, parameters, outputOI, nullsLast); this.typeOperation = new TypeOperationHiveDecimalWritable(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java index 7d5f92c..e2b7035 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java @@ -93,6 +93,15 @@ public abstract class TableFunctionEvaluator { boolean transformsRawInput; transient protected PTFPartition outputPartition; transient protected boolean canAcceptInputAsStream; + protected boolean nullsLast; + + public boolean getNullsLast() { + return nullsLast; + } + + public void setNullsLast(boolean nullsLast) { + this.nullsLast = nullsLast; + } public StructObjectInspector getOutputOI() { return OI; http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java index 5b81a43..dbc7693 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf.ptf; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -71,17 +72,19 @@ public abstract class TableFunctionResolver { evaluator.setTransformsRawInput(transformsRawInput()); evaluator.setTableDef(tDef); evaluator.setQueryDef(ptfDesc); + evaluator.setNullsLast(HiveConf.getBoolVar(cfg, HiveConf.ConfVars.HIVE_DEFAULT_NULLS_LAST)); } /* * called during deserialization of a QueryDef during runtime. */ - public void initialize(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator) + public void initialize(Configuration cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator) throws HiveException { this.evaluator = evaluator; this.ptfDesc = ptfDesc; evaluator.setTableDef(tDef); evaluator.setQueryDef(ptfDesc); + evaluator.setNullsLast(HiveConf.getBoolVar(cfg, HiveConf.ConfVars.HIVE_DEFAULT_NULLS_LAST)); } public TableFunctionEvaluator getEvaluator() { http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java index b34c4d6..e633edb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/ValueBoundaryScanner.java @@ -36,24 +36,28 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn public abstract class ValueBoundaryScanner { BoundaryDef start, end; + protected final boolean nullsLast; - public ValueBoundaryScanner(BoundaryDef start, BoundaryDef end) { + public ValueBoundaryScanner(BoundaryDef start, BoundaryDef end, boolean nullsLast) { this.start = start; this.end = end; + this.nullsLast = nullsLast; } public abstract int computeStart(int rowIdx, PTFPartition p) throws HiveException; public abstract int computeEnd(int rowIdx, PTFPartition p) throws HiveException; - public static ValueBoundaryScanner getScanner(WindowFrameDef winFrameDef) + public static ValueBoundaryScanner getScanner(WindowFrameDef winFrameDef, boolean nullsLast) throws HiveException { OrderDef orderDef = winFrameDef.getOrderDef(); int numOrders = orderDef.getExpressions().size(); if (numOrders != 1) { - return new MultiValueBoundaryScanner(winFrameDef.getStart(), winFrameDef.getEnd(), orderDef); + return new MultiValueBoundaryScanner(winFrameDef.getStart(), winFrameDef.getEnd(), orderDef, + nullsLast); } else { - return SingleValueBoundaryScanner.getScanner(winFrameDef.getStart(), winFrameDef.getEnd(), orderDef); + return SingleValueBoundaryScanner.getScanner(winFrameDef.getStart(), winFrameDef.getEnd(), + orderDef, nullsLast); } } } @@ -65,8 +69,9 @@ public abstract class ValueBoundaryScanner { abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { OrderExpressionDef expressionDef; - public SingleValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end); + public SingleValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, nullsLast); this.expressionDef = expressionDef; } @@ -125,11 +130,8 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { Object sortKey = computeValue(p.getAt(rowIdx)); if ( sortKey == null ) { - // Use Case 2. - if ( expressionDef.getOrder() == Order.ASC ) { - return 0; - } - else { // Use Case 3. + // Use Case 3. + if (nullsLast || expressionDef.getOrder() == Order.DESC) { while ( sortKey == null && rowIdx >= 0 ) { --rowIdx; if ( rowIdx >= 0 ) { @@ -138,6 +140,11 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { } return rowIdx+1; } + else { // Use Case 2. + if ( expressionDef.getOrder() == Order.ASC ) { + return 0; + } + } } Object rowVal = sortKey; @@ -200,7 +207,7 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { if ( sortKey == null ) { // Use Case 9. - if ( expressionDef.getOrder() == Order.DESC) { + if (nullsLast || expressionDef.getOrder() == Order.DESC) { return p.size(); } else { // Use Case 10. @@ -289,7 +296,7 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { if ( sortKey == null ) { // Use Case 2. - if ( expressionDef.getOrder() == Order.DESC ) { + if (nullsLast || expressionDef.getOrder() == Order.DESC ) { return p.size(); } else { // Use Case 3. @@ -362,7 +369,7 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { if ( sortKey == null ) { // Use Case 9. - if ( expressionDef.getOrder() == Order.DESC) { + if (nullsLast || expressionDef.getOrder() == Order.DESC) { return p.size(); } else { // Use Case 10. @@ -416,8 +423,8 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { @SuppressWarnings("incomplete-switch") - public static SingleValueBoundaryScanner getScanner(BoundaryDef start, BoundaryDef end, OrderDef orderDef) - throws HiveException { + public static SingleValueBoundaryScanner getScanner(BoundaryDef start, BoundaryDef end, + OrderDef orderDef, boolean nullsLast) throws HiveException { if (orderDef.getExpressions().size() != 1) { throw new HiveException("Internal error: initializing SingleValueBoundaryScanner with" + " multiple expression for sorting"); @@ -429,20 +436,20 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { case INT: case LONG: case SHORT: - return new LongValueBoundaryScanner(start, end, exprDef); + return new LongValueBoundaryScanner(start, end, exprDef, nullsLast); case TIMESTAMP: - return new TimestampValueBoundaryScanner(start, end, exprDef); + return new TimestampValueBoundaryScanner(start, end, exprDef, nullsLast); case TIMESTAMPLOCALTZ: - return new TimestampLocalTZValueBoundaryScanner(start, end, exprDef); + return new TimestampLocalTZValueBoundaryScanner(start, end, exprDef, nullsLast); case DOUBLE: case FLOAT: - return new DoubleValueBoundaryScanner(start, end, exprDef); + return new DoubleValueBoundaryScanner(start, end, exprDef, nullsLast); case DECIMAL: - return new HiveDecimalValueBoundaryScanner(start, end, exprDef); + return new HiveDecimalValueBoundaryScanner(start, end, exprDef, nullsLast); case DATE: - return new DateValueBoundaryScanner(start, end, exprDef); + return new DateValueBoundaryScanner(start, end, exprDef, nullsLast); case STRING: - return new StringValueBoundaryScanner(start, end, exprDef); + return new StringValueBoundaryScanner(start, end, exprDef, nullsLast); } throw new HiveException( String.format("Internal Error: attempt to setup a Window for datatype %s", @@ -451,8 +458,9 @@ abstract class SingleValueBoundaryScanner extends ValueBoundaryScanner { } class LongValueBoundaryScanner extends SingleValueBoundaryScanner { - public LongValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public LongValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, expressionDef, nullsLast); } @Override @@ -483,8 +491,9 @@ class LongValueBoundaryScanner extends SingleValueBoundaryScanner { } class DoubleValueBoundaryScanner extends SingleValueBoundaryScanner { - public DoubleValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public DoubleValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, expressionDef, nullsLast); } @Override @@ -515,8 +524,9 @@ class DoubleValueBoundaryScanner extends SingleValueBoundaryScanner { } class HiveDecimalValueBoundaryScanner extends SingleValueBoundaryScanner { - public HiveDecimalValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public HiveDecimalValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, expressionDef, nullsLast); } @Override @@ -547,8 +557,9 @@ class HiveDecimalValueBoundaryScanner extends SingleValueBoundaryScanner { } class DateValueBoundaryScanner extends SingleValueBoundaryScanner { - public DateValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public DateValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, expressionDef, nullsLast); } @Override @@ -574,8 +585,9 @@ class DateValueBoundaryScanner extends SingleValueBoundaryScanner { } class TimestampValueBoundaryScanner extends SingleValueBoundaryScanner { - public TimestampValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public TimestampValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end,expressionDef, nullsLast); } @Override @@ -604,8 +616,9 @@ class TimestampValueBoundaryScanner extends SingleValueBoundaryScanner { } class TimestampLocalTZValueBoundaryScanner extends SingleValueBoundaryScanner { - public TimestampLocalTZValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public TimestampLocalTZValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, expressionDef, nullsLast); } @Override @@ -634,8 +647,9 @@ class TimestampLocalTZValueBoundaryScanner extends SingleValueBoundaryScanner { } class StringValueBoundaryScanner extends SingleValueBoundaryScanner { - public StringValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderExpressionDef expressionDef) { - super(start, end,expressionDef); + public StringValueBoundaryScanner(BoundaryDef start, BoundaryDef end, + OrderExpressionDef expressionDef, boolean nullsLast) { + super(start, end, expressionDef, nullsLast); } @Override @@ -662,8 +676,9 @@ class StringValueBoundaryScanner extends SingleValueBoundaryScanner { class MultiValueBoundaryScanner extends ValueBoundaryScanner { OrderDef orderDef; - public MultiValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderDef orderDef) { - super(start, end); + public MultiValueBoundaryScanner(BoundaryDef start, BoundaryDef end, OrderDef orderDef, + boolean nullsLast) { + super(start, end, nullsLast); this.orderDef = orderDef; } http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java index 07b1e2e..5f9009c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java @@ -143,7 +143,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator { private Object evaluateWindowFunction(WindowFunctionDef wFn, int rowToProcess, PTFPartition partition) throws HiveException { BasePartitionEvaluator partitionEval = wFn.getWFnEval() - .getPartitionWindowingEvaluator(wFn.getWindowFrame(), partition, wFn.getArgs(), wFn.getOI()); + .getPartitionWindowingEvaluator(wFn.getWindowFrame(), partition, wFn.getArgs(), wFn.getOI(), nullsLast); return partitionEval.iterate(rowToProcess, ptfDesc.getLlInfo()); } @@ -151,7 +151,7 @@ public class WindowingTableFunction extends TableFunctionEvaluator { private Object evaluateFunctionOnPartition(WindowFunctionDef wFn, PTFPartition partition) throws HiveException { BasePartitionEvaluator partitionEval = wFn.getWFnEval() - .getPartitionWindowingEvaluator(wFn.getWindowFrame(), partition, wFn.getArgs(), wFn.getOI()); + .getPartitionWindowingEvaluator(wFn.getWindowFrame(), partition, wFn.getArgs(), wFn.getOI(), nullsLast); return partitionEval.getPartitionAgg(); } http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out index 2dd921f..7caa2ee 100644 --- a/ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out +++ b/ql/src/test/results/clientpositive/beeline/smb_mapjoin_13.q.out @@ -96,7 +96,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + tag: -1 TopN: 10 @@ -275,7 +275,7 @@ STAGE PLANS: Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) - null sort order: a + null sort order: z sort order: + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE tag: -1 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out b/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out index e83a370..a9206f7 100644 --- a/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out +++ b/ql/src/test/results/clientpositive/column_names_with_leading_and_trailing_spaces.q.out @@ -119,5 +119,5 @@ POSTHOOK: query: select * from space order by ` left` POSTHOOK: type: QUERY POSTHOOK: Input: default@space #### A masked pattern was here #### -NULL 2 NULL 1 2 3 +NULL 2 NULL http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/correlationoptimizer14.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/correlationoptimizer14.q.out b/ql/src/test/results/clientpositive/correlationoptimizer14.q.out index da30b48..c4a7dcb 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer14.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer14.q.out @@ -1518,40 +1518,3 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Input: default@src1 #### A masked pattern was here #### -128 val_128 128 1 -128 val_128 128 1 -128 val_128 128 1 -146 val_146 146 1 -146 val_146 146 1 -150 val_150 150 1 -213 val_213 213 1 -213 val_213 213 1 -224 val_224 224 1 -224 val_224 224 1 -238 val_238 238 1 -238 val_238 238 1 -255 val_255 255 1 -255 val_255 255 1 -273 val_273 273 1 -273 val_273 273 1 -273 val_273 273 1 -278 val_278 278 1 -278 val_278 278 1 -311 val_311 311 1 -311 val_311 311 1 -311 val_311 311 1 -369 val_369 369 1 -369 val_369 369 1 -369 val_369 369 1 -401 val_401 401 1 -401 val_401 401 1 -401 val_401 401 1 -401 val_401 401 1 -401 val_401 401 1 -406 val_406 406 1 -406 val_406 406 1 -406 val_406 406 1 -406 val_406 406 1 -66 val_66 66 1 -98 val_98 98 1 -98 val_98 98 1 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/ctas_colname.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/ctas_colname.q.out b/ql/src/test/results/clientpositive/ctas_colname.q.out index b3a4992..05e3f76 100644 --- a/ql/src/test/results/clientpositive/ctas_colname.q.out +++ b/ql/src/test/results/clientpositive/ctas_colname.q.out @@ -184,7 +184,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col1 ASC NULLS FIRST + order by: _col1 ASC NULLS LAST partition by: _col0 raw input shape: window functions: @@ -350,7 +350,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col1 ASC NULLS FIRST + order by: _col1 ASC NULLS LAST partition by: _col0 raw input shape: window functions: http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/decimal_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_3.q.out b/ql/src/test/results/clientpositive/decimal_3.q.out index 3ded9a7..d2e3957 100644 --- a/ql/src/test/results/clientpositive/decimal_3.q.out +++ b/ql/src/test/results/clientpositive/decimal_3.q.out @@ -32,7 +32,6 @@ POSTHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_3 #### A masked pattern was here #### -NULL 0 -1234567890.123456789000000000 -1234567890 -4400.000000000000000000 4400 -1255.490000000000000000 -1255 @@ -70,6 +69,7 @@ NULL 0 125.200000000000000000 125 200.000000000000000000 200 1234567890.123456780000000000 1234567890 +NULL 0 PREHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key DESC, value DESC PREHOOK: type: QUERY PREHOOK: Input: default@decimal_3 @@ -124,7 +124,6 @@ POSTHOOK: query: SELECT * FROM DECIMAL_3 ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_3 #### A masked pattern was here #### -NULL 0 -1234567890.123456789000000000 -1234567890 -4400.000000000000000000 4400 -1255.490000000000000000 -1255 @@ -162,6 +161,7 @@ NULL 0 125.200000000000000000 125 200.000000000000000000 200 1234567890.123456780000000000 1234567890 +NULL 0 PREHOOK: query: SELECT DISTINCT key FROM DECIMAL_3 ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@decimal_3 @@ -170,7 +170,6 @@ POSTHOOK: query: SELECT DISTINCT key FROM DECIMAL_3 ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_3 #### A masked pattern was here #### -NULL -1234567890.123456789000000000 -4400.000000000000000000 -1255.490000000000000000 @@ -199,6 +198,7 @@ NULL 125.200000000000000000 200.000000000000000000 1234567890.123456780000000000 +NULL PREHOOK: query: SELECT key, sum(value) FROM DECIMAL_3 GROUP BY key ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@decimal_3 @@ -207,7 +207,6 @@ POSTHOOK: query: SELECT key, sum(value) FROM DECIMAL_3 GROUP BY key ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_3 #### A masked pattern was here #### -NULL 0 -1234567890.123456789000000000 -1234567890 -4400.000000000000000000 4400 -1255.490000000000000000 -1255 @@ -236,6 +235,7 @@ NULL 0 125.200000000000000000 125 200.000000000000000000 200 1234567890.123456780000000000 1234567890 +NULL 0 PREHOOK: query: SELECT value, sum(key) FROM DECIMAL_3 GROUP BY value ORDER BY value PREHOOK: type: QUERY PREHOOK: Input: default@decimal_3 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/decimal_4.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_4.q.out b/ql/src/test/results/clientpositive/decimal_4.q.out index 8eb1de4..9d3ee84 100644 --- a/ql/src/test/results/clientpositive/decimal_4.q.out +++ b/ql/src/test/results/clientpositive/decimal_4.q.out @@ -56,7 +56,6 @@ POSTHOOK: query: SELECT * FROM DECIMAL_4_1 ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_4_1 #### A masked pattern was here #### -NULL 0 -1234567890.1234567890000000000000000 -1234567890 -4400.0000000000000000000000000 4400 -1255.4900000000000000000000000 -1255 @@ -94,6 +93,7 @@ NULL 0 125.2000000000000000000000000 125 200.0000000000000000000000000 200 1234567890.1234567800000000000000000 1234567890 +NULL 0 PREHOOK: query: SELECT * FROM DECIMAL_4_2 ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@decimal_4_2 @@ -102,7 +102,6 @@ POSTHOOK: query: SELECT * FROM DECIMAL_4_2 ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_4_2 #### A masked pattern was here #### -NULL NULL -1234567890.1234567890000000000000000 -3703703670.3703703670000000000000000 -4400.0000000000000000000000000 -13200.0000000000000000000000000 -1255.4900000000000000000000000 -3766.4700000000000000000000000 @@ -140,6 +139,7 @@ NULL NULL 125.2000000000000000000000000 375.6000000000000000000000000 200.0000000000000000000000000 600.0000000000000000000000000 1234567890.1234567800000000000000000 3703703670.3703703400000000000000000 +NULL NULL PREHOOK: query: DROP TABLE DECIMAL_4_1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@decimal_4_1 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/decimal_5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_5.q.out b/ql/src/test/results/clientpositive/decimal_5.q.out index d94f5f2..f24588c 100644 --- a/ql/src/test/results/clientpositive/decimal_5.q.out +++ b/ql/src/test/results/clientpositive/decimal_5.q.out @@ -32,9 +32,6 @@ POSTHOOK: query: SELECT key FROM DECIMAL_5_n0 ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_5_n0 #### A masked pattern was here #### -NULL -NULL -NULL -4400.00000 -1255.49000 -1.12200 @@ -70,6 +67,9 @@ NULL 124.00000 125.20000 200.00000 +NULL +NULL +NULL PREHOOK: query: SELECT DISTINCT key FROM DECIMAL_5_n0 ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@decimal_5_n0 @@ -78,7 +78,6 @@ POSTHOOK: query: SELECT DISTINCT key FROM DECIMAL_5_n0 ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_5_n0 #### A masked pattern was here #### -NULL -4400.00000 -1255.49000 -1.12200 @@ -105,6 +104,7 @@ NULL 124.00000 125.20000 200.00000 +NULL PREHOOK: query: SELECT cast(key as decimal) FROM DECIMAL_5_n0 PREHOOK: type: QUERY PREHOOK: Input: default@decimal_5_n0 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/decimal_6.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_6.q.out b/ql/src/test/results/clientpositive/decimal_6.q.out index 1959dd9..83cadce 100644 --- a/ql/src/test/results/clientpositive/decimal_6.q.out +++ b/ql/src/test/results/clientpositive/decimal_6.q.out @@ -72,12 +72,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_6_1_n0 POSTHOOK: Input: default@decimal_6_2_n0 #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL -NULL -1234567890.12350 -4400.00000 -4400.00000 @@ -126,6 +120,12 @@ NULL 2389432.23750 2389432.23750 1234567890.12350 +NULL +NULL +NULL +NULL +NULL +NULL PREHOOK: query: CREATE TABLE DECIMAL_6_3_n0 AS SELECT key + 5.5 AS k, value * 11 AS v from DECIMAL_6_1_n0 ORDER BY v PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@decimal_6_1_n0 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/decimal_precision.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_precision.q.out b/ql/src/test/results/clientpositive/decimal_precision.q.out index 921d86b..fb59c4f 100644 --- a/ql/src/test/results/clientpositive/decimal_precision.q.out +++ b/ql/src/test/results/clientpositive/decimal_precision.q.out @@ -32,6 +32,37 @@ POSTHOOK: query: SELECT * FROM DECIMAL_PRECISION_n0 ORDER BY `dec` POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_precision_n0 #### A masked pattern was here #### +0.0000000000 +0.0000000000 +0.0000000000 +0.0000000000 +0.0000000000 +0.1234567890 +0.1234567890 +1.2345678901 +1.2345678901 +1.2345678901 +12.3456789012 +12.3456789012 +12.3456789012 +123.4567890123 +123.4567890123 +123.4567890123 +1234.5678901235 +1234.5678901235 +1234.5678901235 +12345.6789012346 +12345.6789012346 +123456.7890123456 +123456.7890123457 +1234567.8901234560 +1234567.8901234568 +12345678.9012345600 +12345678.9012345679 +123456789.0123456000 +123456789.0123456789 +1234567890.1234560000 +1234567890.1234567890 NULL NULL NULL @@ -76,37 +107,6 @@ NULL NULL NULL NULL -0.0000000000 -0.0000000000 -0.0000000000 -0.0000000000 -0.0000000000 -0.1234567890 -0.1234567890 -1.2345678901 -1.2345678901 -1.2345678901 -12.3456789012 -12.3456789012 -12.3456789012 -123.4567890123 -123.4567890123 -123.4567890123 -1234.5678901235 -1234.5678901235 -1234.5678901235 -12345.6789012346 -12345.6789012346 -123456.7890123456 -123456.7890123457 -1234567.8901234560 -1234567.8901234568 -12345678.9012345600 -12345678.9012345679 -123456789.0123456000 -123456789.0123456789 -1234567890.1234560000 -1234567890.1234567890 PREHOOK: query: SELECT `dec`, `dec` + 1, `dec` - 1 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` PREHOOK: type: QUERY PREHOOK: Input: default@decimal_precision_n0 @@ -115,50 +115,6 @@ POSTHOOK: query: SELECT `dec`, `dec` + 1, `dec` - 1 FROM DECIMAL_PRECISION_n0 OR POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_precision_n0 #### A masked pattern was here #### -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL -NULL NULL NULL 0.0000000000 1.0000000000 -1.0000000000 0.0000000000 1.0000000000 -1.0000000000 0.0000000000 1.0000000000 -1.0000000000 @@ -190,14 +146,6 @@ NULL NULL NULL 123456789.0123456789 123456790.0123456789 123456788.0123456789 1234567890.1234560000 1234567891.1234560000 1234567889.1234560000 1234567890.1234567890 1234567891.1234567890 1234567889.1234567890 -PREHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_precision_n0 -#### A masked pattern was here #### -POSTHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_precision_n0 -#### A masked pattern was here #### NULL NULL NULL NULL NULL NULL NULL NULL NULL @@ -242,6 +190,14 @@ NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +PREHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision_n0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT `dec`, `dec` * 2, `dec` / 3 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision_n0 +#### A masked pattern was here #### 0.0000000000 0.0000000000 0.000000000000 0.0000000000 0.0000000000 0.000000000000 0.0000000000 0.0000000000 0.000000000000 @@ -273,6 +229,50 @@ NULL NULL NULL 123456789.0123456789 246913578.0246913578 41152263.004115226300 1234567890.1234560000 2469135780.2469120000 411522630.041152000000 1234567890.1234567890 2469135780.2469135780 411522630.041152263000 +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL +NULL NULL NULL PREHOOK: query: SELECT `dec`, `dec` / 9 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` PREHOOK: type: QUERY PREHOOK: Input: default@decimal_precision_n0 @@ -281,50 +281,6 @@ POSTHOOK: query: SELECT `dec`, `dec` / 9 FROM DECIMAL_PRECISION_n0 ORDER BY `dec POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_precision_n0 #### A masked pattern was here #### -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL -NULL NULL 0.0000000000 0.000000000000 0.0000000000 0.000000000000 0.0000000000 0.000000000000 @@ -356,14 +312,6 @@ NULL NULL 123456789.0123456789 13717421.001371742100 1234567890.1234560000 137174210.013717333333 1234567890.1234567890 137174210.013717421000 -PREHOOK: query: SELECT `dec`, `dec` / 27 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_precision_n0 -#### A masked pattern was here #### -POSTHOOK: query: SELECT `dec`, `dec` / 27 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_precision_n0 -#### A masked pattern was here #### NULL NULL NULL NULL NULL NULL @@ -408,6 +356,14 @@ NULL NULL NULL NULL NULL NULL NULL NULL +PREHOOK: query: SELECT `dec`, `dec` / 27 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision_n0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT `dec`, `dec` / 27 FROM DECIMAL_PRECISION_n0 ORDER BY `dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision_n0 +#### A masked pattern was here #### 0.0000000000 0.0000000000000 0.0000000000 0.0000000000000 0.0000000000 0.0000000000000 @@ -439,14 +395,6 @@ NULL NULL 123456789.0123456789 4572473.6671239140333 1234567890.1234560000 45724736.6712391111111 1234567890.1234567890 45724736.6712391403333 -PREHOOK: query: SELECT `dec`, `dec` * `dec` FROM DECIMAL_PRECISION_n0 ORDER BY `dec` -PREHOOK: type: QUERY -PREHOOK: Input: default@decimal_precision_n0 -#### A masked pattern was here #### -POSTHOOK: query: SELECT `dec`, `dec` * `dec` FROM DECIMAL_PRECISION_n0 ORDER BY `dec` -POSTHOOK: type: QUERY -POSTHOOK: Input: default@decimal_precision_n0 -#### A masked pattern was here #### NULL NULL NULL NULL NULL NULL @@ -491,6 +439,14 @@ NULL NULL NULL NULL NULL NULL NULL NULL +PREHOOK: query: SELECT `dec`, `dec` * `dec` FROM DECIMAL_PRECISION_n0 ORDER BY `dec` +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_precision_n0 +#### A masked pattern was here #### +POSTHOOK: query: SELECT `dec`, `dec` * `dec` FROM DECIMAL_PRECISION_n0 ORDER BY `dec` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_precision_n0 +#### A masked pattern was here #### 0.0000000000 0.00000000000000000 0.0000000000 0.00000000000000000 0.0000000000 0.00000000000000000 @@ -522,6 +478,50 @@ NULL NULL 123456789.0123456789 15241578753238836.75019051998750191 1234567890.1234560000 1524157875323881726.87092138393600000 1234567890.1234567890 1524157875323883675.01905199875019052 +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL +NULL NULL PREHOOK: query: EXPLAIN SELECT avg(`dec`), sum(`dec`) FROM DECIMAL_PRECISION_n0 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT avg(`dec`), sum(`dec`) FROM DECIMAL_PRECISION_n0 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/decimal_serde.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/decimal_serde.q.out b/ql/src/test/results/clientpositive/decimal_serde.q.out index fb74324..ec90dd3 100644 --- a/ql/src/test/results/clientpositive/decimal_serde.q.out +++ b/ql/src/test/results/clientpositive/decimal_serde.q.out @@ -44,7 +44,6 @@ POSTHOOK: query: SELECT * FROM DECIMAL_TEXT ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_text #### A masked pattern was here #### -NULL 0 -1234567890 -1234567890 -4400 4400 -1255 -1255 @@ -82,6 +81,7 @@ NULL 0 125 125 200 200 1234567890 1234567890 +NULL 0 PREHOOK: query: CREATE TABLE DECIMAL_RC STORED AS RCFile AS SELECT * FROM DECIMAL_TEXT @@ -218,7 +218,6 @@ POSTHOOK: query: SELECT * FROM DECIMAL_SEQUENCE ORDER BY key, value POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_sequence #### A masked pattern was here #### -NULL 0 -1234567890 -1234567890 -4400 4400 -1255 -1255 @@ -256,6 +255,7 @@ NULL 0 125 125 200 200 1234567890 1234567890 +NULL 0 PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_TEXT PREHOOK: type: DROPTABLE PREHOOK: Input: default@decimal_text http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/delete_all_partitioned.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/delete_all_partitioned.q.out b/ql/src/test/results/clientpositive/delete_all_partitioned.q.out index 90f8753..4c1a024 100644 --- a/ql/src/test/results/clientpositive/delete_all_partitioned.q.out +++ b/ql/src/test/results/clientpositive/delete_all_partitioned.q.out @@ -48,7 +48,6 @@ POSTHOOK: Input: default@acid_dap@ds=tomorrow -1070883071 0ruyd6Y50JpdGRf6HqD today -1070551679 iUR3Q today -1069736047 k17Am8uPHWk02cEf1jet today -6981 NULL tomorrow 6981 1FNNhmiFLGw425NA13g tomorrow 6981 4KhrrQ0nJ7bMNTvhSCA tomorrow 6981 K630vaVf tomorrow @@ -58,6 +57,7 @@ POSTHOOK: Input: default@acid_dap@ds=tomorrow 6981 o4lvY20511w0EOX3P3I82p63 tomorrow 6981 o5mb0QP5Y48Qd4vdB0 tomorrow 6981 sF2CRfgt2K tomorrow +6981 NULL tomorrow PREHOOK: query: delete from acid_dap PREHOOK: type: QUERY PREHOOK: Input: default@acid_dap http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/distinct_windowing.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/distinct_windowing.q.out b/ql/src/test/results/clientpositive/distinct_windowing.q.out index 39b87e8..b32786e 100644 --- a/ql/src/test/results/clientpositive/distinct_windowing.q.out +++ b/ql/src/test/results/clientpositive/distinct_windowing.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: @@ -202,7 +202,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: @@ -326,7 +326,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/distinct_windowing_no_cbo.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/distinct_windowing_no_cbo.q.out b/ql/src/test/results/clientpositive/distinct_windowing_no_cbo.q.out index c1fe3a2..6196b28 100644 --- a/ql/src/test/results/clientpositive/distinct_windowing_no_cbo.q.out +++ b/ql/src/test/results/clientpositive/distinct_windowing_no_cbo.q.out @@ -83,7 +83,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: @@ -202,7 +202,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: @@ -326,7 +326,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: @@ -539,7 +539,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col1 ASC NULLS FIRST + order by: _col1 ASC NULLS LAST partition by: _col0 raw input shape: window functions: @@ -680,7 +680,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col2 ASC NULLS FIRST + order by: _col2 ASC NULLS LAST partition by: _col1 raw input shape: window functions: http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/groupby_grouping_window.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out index f50b8dc..63a9d0c 100644 --- a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out +++ b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out @@ -103,7 +103,7 @@ STAGE PLANS: Windowing table definition input alias: ptf_1 name: windowingtablefunction - order by: _col3 ASC NULLS FIRST + order by: _col3 ASC NULLS LAST partition by: _col0 raw input shape: window functions: http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/input_part7.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/input_part7.q.out b/ql/src/test/results/clientpositive/input_part7.q.out index c4f0b11..e40627f 100644 --- a/ql/src/test/results/clientpositive/input_part7.q.out +++ b/ql/src/test/results/clientpositive/input_part7.q.out @@ -42,7 +42,7 @@ STAGE PLANS: Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) - null sort order: aaaa + null sort order: zzzz sort order: ++++ Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE tag: -1 @@ -67,7 +67,7 @@ STAGE PLANS: Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), '2008-04-08' (type: string), _col3 (type: string) - null sort order: aaaa + null sort order: zzzz sort order: ++++ Statistics: Num rows: 666 Data size: 7074 Basic stats: COMPLETE Column stats: NONE tag: -1 http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out b/ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out index 3819351..513f7e5 100644 --- a/ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out +++ b/ql/src/test/results/clientpositive/insert_values_non_partitioned.q.out @@ -65,6 +65,6 @@ POSTHOOK: query: select * from acid_ivnp order by ti POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_ivnp #### A masked pattern was here #### -NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 257 65537 4294967297 3.14 3.141592654 109.23 2014-08-25 17:21:30 2014-08-25 true mary had a little lamb ring around the rosie red 3 25 6553 NULL 0.14 1923.141592654 1.23 2014-08-24 17:21:30 2014-08-26 false its fleece was white as snow a pocket full of posies blue +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL http://git-wip-us.apache.org/repos/asf/hive/blob/0427ffa5/ql/src/test/results/clientpositive/limit_pushdown2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/limit_pushdown2.q.out b/ql/src/test/results/clientpositive/limit_pushdown2.q.out index 87be772..edfc042 100644 --- a/ql/src/test/results/clientpositive/limit_pushdown2.q.out +++ b/ql/src/test/results/clientpositive/limit_pushdown2.q.out @@ -1143,23 +1143,23 @@ order by key, value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -NULL NULL 261.182 -NULL val_0 1.0 -NULL val_10 11.0 -NULL val_100 101.0 -NULL val_103 104.0 -NULL val_104 105.0 -NULL val_105 106.0 -NULL val_11 12.0 -NULL val_111 112.0 -NULL val_113 114.0 -NULL val_114 115.0 -NULL val_116 117.0 -NULL val_118 119.0 -NULL val_119 120.0 -NULL val_12 13.0 -NULL val_120 121.0 -NULL val_125 126.0 -NULL val_126 127.0 -NULL val_128 129.0 -NULL val_129 130.0 +0 val_0 1.0 +10 val_10 11.0 +100 val_100 101.0 +103 val_103 104.0 +104 val_104 105.0 +105 val_105 106.0 +11 val_11 12.0 +111 val_111 112.0 +113 val_113 114.0 +114 val_114 115.0 +116 val_116 117.0 +118 val_118 119.0 +119 val_119 120.0 +12 val_12 13.0 +120 val_120 121.0 +125 val_125 126.0 +126 val_126 127.0 +128 val_128 129.0 +129 val_129 130.0 +131 val_131 132.0
