Author: gunther Date: Tue Jul 29 23:15:13 2014 New Revision: 1614514 URL: http://svn.apache.org/r1614514 Log: Merge latest trunk into cbo branch. (Gunther Hagleitner)
Added: hive/branches/cbo/ql/src/test/queries/clientpositive/vector_string_concat.q - copied unchanged from r1614505, hive/trunk/ql/src/test/queries/clientpositive/vector_string_concat.q hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_string_concat.q.out - copied unchanged from r1614505, hive/trunk/ql/src/test/results/clientpositive/tez/vector_string_concat.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vector_string_concat.q.out - copied unchanged from r1614505, hive/trunk/ql/src/test/results/clientpositive/vector_string_concat.q.out Modified: hive/branches/cbo/ (props changed) hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java hive/branches/cbo/conf/hive-default.xml.template hive/branches/cbo/itests/qtest/testconfiguration.properties hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatColScalar.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatScalarCol.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_filter.q.out hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out hive/branches/cbo/ql/src/test/results/clientpositive/join_nullsafe.q.out hive/branches/cbo/ql/src/test/results/clientpositive/metadataonly1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/metadataonly1.q.out Propchange: hive/branches/cbo/ ------------------------------------------------------------------------------ Merged /hive/trunk:r1614212-1614505 Modified: hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original) +++ hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Jul 29 23:15:13 2014 @@ -745,7 +745,7 @@ public class HiveConf extends Configurat // HWI HIVEHWILISTENHOST("hive.hwi.listen.host", "0.0.0.0", "This is the host address the Hive Web Interface will listen on"), HIVEHWILISTENPORT("hive.hwi.listen.port", "9999", "This is the port the Hive Web Interface will listen on"), - HIVEHWIWARFILE("hive.hwi.war.file", "${system:HWI_WAR_FILE}", + HIVEHWIWARFILE("hive.hwi.war.file", "${env:HWI_WAR_FILE}", "This sets the path to the HWI war file, relative to ${HIVE_HOME}. "), HIVEHADOOPMAXMEM("hive.mapred.local.mem", 0, "mapper/reducer memory in local mode"), @@ -1394,10 +1394,9 @@ public class HiveConf extends Configurat "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."), // Hive global init file location - HIVE_GLOBAL_INIT_FILE_LOCATION("hive.global.init.file.location", System.getenv("HIVE_CONF_DIR"), + HIVE_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}", "The location of HS2 global init file (.hiverc).\n" + - "If the property is not set, then HS2 will search for the file in $HIVE_CONF_DIR/.\n" + - "If the property is set, the value must be a valid path where the init file is located."), + "If the property is reset, the value must be a valid path where the init file is located."), // prefix used to auto generated column aliases (this should be started with '_') HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c", Modified: hive/branches/cbo/conf/hive-default.xml.template URL: http://svn.apache.org/viewvc/hive/branches/cbo/conf/hive-default.xml.template?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/conf/hive-default.xml.template (original) +++ hive/branches/cbo/conf/hive-default.xml.template Tue Jul 29 23:15:13 2014 @@ -1191,7 +1191,7 @@ </property> <property> <key>hive.hwi.war.file</key> - <value>${system:HWI_WAR_FILE}</value> + <value>${env:HWI_WAR_FILE}</value> <description>This sets the path to the HWI war file, relative to ${HIVE_HOME}. </description> </property> <property> @@ -2480,12 +2480,11 @@ </description> </property> <property> - <key>hive.global.init.file.location</key> - <value/> + <key>hive.server2.global.init.file.location</key> + <value>${env:HIVE_CONF_DIR}</value> <description> The location of HS2 global init file (.hiverc). - If the property is not set, then HS2 will search for the file in $HIVE_CONF_DIR/. - If the property is set, the value must be a valid path where the init file is located. + If the property is reset, the value must be a valid path where the init file is located. </description> </property> <property> Modified: hive/branches/cbo/itests/qtest/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/qtest/testconfiguration.properties?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/itests/qtest/testconfiguration.properties (original) +++ hive/branches/cbo/itests/qtest/testconfiguration.properties Tue Jul 29 23:15:13 2014 @@ -1,5 +1,5 @@ minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q -minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t able.q,vectorized_ptf.q,optimize_nullscan.q,vector_cast_constant.q,cbo_correctness.q +minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t able.q,vectorized_ptf.q,optimize_nullscan.q,vector_cast_constant.q,vector_string_concat.q,cbo_correctness.q beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q, exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwr ite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatColScalar.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatColScalar.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatColScalar.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatColScalar.java Tue Jul 29 23:15:13 2014 @@ -45,6 +45,11 @@ public class StringConcatColScalar exten @Override public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatScalarCol.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatScalarCol.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatScalarCol.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringConcatScalarCol.java Tue Jul 29 23:15:13 2014 @@ -45,6 +45,11 @@ public class StringConcatScalarCol exten @Override public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + BytesColumnVector inputColVector = (BytesColumnVector) batch.cols[colNum]; BytesColumnVector outV = (BytesColumnVector) batch.cols[outputColumn]; int[] sel = batch.selected; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDateString.java Tue Jul 29 23:15:13 2014 @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; import org.apache.hadoop.io.Text; import java.sql.Date; @@ -25,6 +28,9 @@ import java.sql.Date; public class VectorUDFDateString extends StringUnaryUDF { private static final long serialVersionUID = 1L; + private static final Log LOG = LogFactory.getLog( + VectorUDFDateString.class.getName()); + public VectorUDFDateString(int colNum, int outputColumn) { super(colNum, outputColumn, new StringUnaryUDF.IUDFUnaryString() { Text t = new Text(); @@ -39,7 +45,9 @@ public class VectorUDFDateString extends t.set(date.toString()); return t; } catch (IllegalArgumentException e) { - e.printStackTrace(); + if (LOG.isDebugEnabled()) { + LOG.info("VectorUDFDateString passed bad string for Date.valueOf '" + s.toString() + "'"); + } return null; } } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldLong.java Tue Jul 29 23:15:13 2014 @@ -88,6 +88,11 @@ public abstract class VectorUDFTimestamp @Override public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + LongColumnVector outV = (LongColumnVector) batch.cols[outputColumn]; LongColumnVector inputCol = (LongColumnVector)batch.cols[this.colNum]; /* every line below this is identical for evaluateLong & evaluateString */ Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java Tue Jul 29 23:15:13 2014 @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Stack; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; @@ -50,6 +51,7 @@ import org.apache.hadoop.hive.ql.parse.Q import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TableAccessAnalyzer; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -456,10 +458,19 @@ abstract public class AbstractBucketJoin public static List<String> toColumns(List<ExprNodeDesc> keys) { List<String> columns = new ArrayList<String>(); for (ExprNodeDesc key : keys) { - if (!(key instanceof ExprNodeColumnDesc)) { + if (key instanceof ExprNodeColumnDesc) { + columns.add(((ExprNodeColumnDesc) key).getColumn()); + } else if ((key instanceof ExprNodeConstantDesc)) { + ExprNodeConstantDesc constant = (ExprNodeConstantDesc) key; + String colName = constant.getFoldedFromCol(); + if (colName == null){ + return null; + } else { + columns.add(colName); + } + } else { return null; } - columns.add(((ExprNodeColumnDesc) key).getColumn()); } return columns; } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java Tue Jul 29 23:15:13 2014 @@ -82,9 +82,7 @@ public class ConstantPropagate implement // if the later is enabled. return pactx; } - if (pactx.getConf().getBoolVar(ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN)) { - return pactx; - } + pGraphContext = pactx; opToParseCtxMap = pGraphContext.getOpParseCtx(); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java Tue Jul 29 23:15:13 2014 @@ -552,6 +552,7 @@ public final class ConstantPropagateProc * conditional expressions and extract assignment expressions and propagate them. */ public static class ConstantPropagateFilterProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { FilterOperator op = (FilterOperator) nd; @@ -594,6 +595,7 @@ public final class ConstantPropagateProc * Node Processor for Constant Propagate for Group By Operators. */ public static class ConstantPropagateGroupByProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { GroupByOperator op = (GroupByOperator) nd; @@ -630,6 +632,7 @@ public final class ConstantPropagateProc * The Default Node Processor for Constant Propagation. */ public static class ConstantPropagateDefaultProc implements NodeProcessor { + @Override @SuppressWarnings("unchecked") public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { @@ -658,6 +661,7 @@ public final class ConstantPropagateProc * The Node Processor for Constant Propagation for Select Operators. */ public static class ConstantPropagateSelectProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { SelectOperator op = (SelectOperator) nd; @@ -691,6 +695,7 @@ public final class ConstantPropagateProc * propagation, this processor also prunes dynamic partitions to static partitions if possible. */ public static class ConstantPropagateFileSinkProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { FileSinkOperator op = (FileSinkOperator) nd; @@ -743,6 +748,7 @@ public final class ConstantPropagateProc * Currently these kinds of Operators include UnionOperator and ScriptOperator. */ public static class ConstantPropagateStopProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { Operator<?> op = (Operator<?>) nd; @@ -763,6 +769,7 @@ public final class ConstantPropagateProc * join (left table for left outer join and vice versa) can be propagated. */ public static class ConstantPropagateReduceSinkProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ReduceSinkOperator op = (ReduceSinkOperator) nd; @@ -795,7 +802,11 @@ public final class ConstantPropagateProc // key columns ArrayList<ExprNodeDesc> newKeyEpxrs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc desc : rsDesc.getKeyCols()) { - newKeyEpxrs.add(foldExpr(desc, constants, cppCtx, op, 0, false)); + ExprNodeDesc newDesc = foldExpr(desc, constants, cppCtx, op, 0, false); + if (newDesc != desc && desc instanceof ExprNodeColumnDesc && newDesc instanceof ExprNodeConstantDesc) { + ((ExprNodeConstantDesc)newDesc).setFoldedFromCol(((ExprNodeColumnDesc)desc).getColumn()); + } + newKeyEpxrs.add(newDesc); } rsDesc.setKeyCols(newKeyEpxrs); @@ -854,6 +865,7 @@ public final class ConstantPropagateProc * The Node Processor for Constant Propagation for Join Operators. */ public static class ConstantPropagateJoinProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { JoinOperator op = (JoinOperator) nd; @@ -916,6 +928,7 @@ public final class ConstantPropagateProc * The Node Processor for Constant Propagation for Table Scan Operators. */ public static class ConstantPropagateTableScanProc implements NodeProcessor { + @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { TableScanOperator op = (TableScanOperator) nd; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java Tue Jul 29 23:15:13 2014 @@ -18,17 +18,13 @@ package org.apache.hadoop.hive.ql.optimizer; -import java.util.HashMap; -import java.util.Map; import java.util.Stack; import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.parse.ParseContext; -import org.apache.hadoop.hive.ql.parse.QBJoinTree; import org.apache.hadoop.hive.ql.parse.SemanticException; public class SortedMergeJoinProc extends AbstractSMBJoinProc implements NodeProcessor { @@ -46,12 +42,6 @@ public class SortedMergeJoinProc extends JoinOperator joinOp = (JoinOperator) nd; SortBucketJoinProcCtx smbJoinContext = (SortBucketJoinProcCtx) procCtx; - Map<MapJoinOperator, QBJoinTree> mapJoinMap = pGraphContext.getMapJoinContext(); - if (mapJoinMap == null) { - mapJoinMap = new HashMap<MapJoinOperator, QBJoinTree>(); - pGraphContext.setMapJoinContext(mapJoinMap); - } - boolean convert = canConvertJoinToSMBJoin( joinOp, smbJoinContext, pGraphContext); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java Tue Jul 29 23:15:13 2014 @@ -401,7 +401,6 @@ public class StatsRulesProcFactory { long numRows = stats.getNumRows(); - // evaluate similar to "col = constant" expr if (pred instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc genFunc = (ExprNodeGenericFuncDesc) pred; @@ -413,9 +412,7 @@ public class StatsRulesProcFactory { String tabAlias = colDesc.getTabAlias(); ColStatistics cs = stats.getColumnStatisticsForColumn(tabAlias, colName); if (cs != null) { - long dvs = cs.getCountDistint(); - numRows = dvs == 0 ? numRows / 2 : numRows / dvs; - return numRows; + return cs.getNumNulls(); } } } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Jul 29 23:15:13 2014 @@ -928,7 +928,7 @@ alterStatement | KW_INDEX! alterIndexStatementSuffix | - KW_DATABASE! alterDatabaseStatementSuffix + (KW_DATABASE|KW_SCHEMA)! alterDatabaseStatementSuffix ) ; @@ -1299,7 +1299,7 @@ descStatement @after { popMsg(state); } : (KW_DESCRIBE|KW_DESC) (descOptions=KW_FORMATTED|descOptions=KW_EXTENDED|descOptions=KW_PRETTY)? (parttype=descPartTypeExpr) -> ^(TOK_DESCTABLE $parttype $descOptions?) | (KW_DESCRIBE|KW_DESC) KW_FUNCTION KW_EXTENDED? (name=descFuncNames) -> ^(TOK_DESCFUNCTION $name KW_EXTENDED?) - | (KW_DESCRIBE|KW_DESC) KW_DATABASE KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) + | (KW_DESCRIBE|KW_DESC) (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) ; analyzeStatement @@ -1324,7 +1324,7 @@ showStatement -> ^(TOK_SHOW_TABLESTATUS showStmtIdentifier $db_name? partitionSpec?) | KW_SHOW KW_TBLPROPERTIES tblName=identifier (LPAREN prptyName=StringLiteral RPAREN)? -> ^(TOK_SHOW_TBLPROPERTIES $tblName $prptyName?) | KW_SHOW KW_LOCKS (parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?) - | KW_SHOW KW_LOCKS KW_DATABASE (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) + | KW_SHOW KW_LOCKS (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWDBLOCKS $dbName $isExtended?) | KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)? -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?) | KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS) @@ -1341,7 +1341,7 @@ lockStatement lockDatabase @init { pushMsg("lock database statement", state); } @after { popMsg(state); } - : KW_LOCK KW_DATABASE (dbName=Identifier) lockMode -> ^(TOK_LOCKDB $dbName lockMode) + : KW_LOCK (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) lockMode -> ^(TOK_LOCKDB $dbName lockMode) ; lockMode @@ -1359,7 +1359,7 @@ unlockStatement unlockDatabase @init { pushMsg("unlock database statement", state); } @after { popMsg(state); } - : KW_UNLOCK KW_DATABASE (dbName=Identifier) -> ^(TOK_UNLOCKDB $dbName) + : KW_UNLOCK (KW_DATABASE|KW_SCHEMA) (dbName=Identifier) -> ^(TOK_UNLOCKDB $dbName) ; createRoleStatement @@ -1471,7 +1471,7 @@ privilegeObject privObjectType @init {pushMsg("privilege object type type", state);} @after {popMsg(state);} - : KW_DATABASE -> ^(TOK_DB_TYPE) + : (KW_DATABASE|KW_SCHEMA) -> ^(TOK_DB_TYPE) | KW_TABLE? -> ^(TOK_TABLE_TYPE) ; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Tue Jul 29 23:15:13 2014 @@ -36,6 +36,17 @@ public class ExprNodeConstantDesc extend private static final long serialVersionUID = 1L; final protected transient static char[] hexArray = "0123456789ABCDEF".toCharArray(); private Object value; + // If this constant was created while doing constant folding, foldedFromCol holds the name of + // original column from which it was folded. + private transient String foldedFromCol; + + public String getFoldedFromCol() { + return foldedFromCol; + } + + public void setFoldedFromCol(String foldedFromCol) { + this.foldedFromCol = foldedFromCol; + } public ExprNodeConstantDesc() { } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Tue Jul 29 23:15:13 2014 @@ -437,6 +437,8 @@ public class StatsUtils { cs.setAvgColLen(JavaDataModel.get().lengthOfTimestamp()); } else if (colType.startsWith(serdeConstants.DECIMAL_TYPE_NAME)) { cs.setAvgColLen(JavaDataModel.get().lengthOfDecimal()); + cs.setCountDistint(csd.getDecimalStats().getNumDVs()); + cs.setNumNulls(csd.getDecimalStats().getNumNulls()); } else if (colType.equalsIgnoreCase(serdeConstants.DATE_TYPE_NAME)) { cs.setAvgColLen(JavaDataModel.get().lengthOfDate()); } else { Modified: hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_filter.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_filter.q.out?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== Files hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_filter.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_filter.q.out Tue Jul 29 23:15:13 2014 differ Modified: hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== Files hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_join.q.out Tue Jul 29 23:15:13 2014 differ Modified: hive/branches/cbo/ql/src/test/results/clientpositive/join_nullsafe.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/join_nullsafe.q.out?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/join_nullsafe.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/join_nullsafe.q.out Tue Jul 29 23:15:13 2014 @@ -1519,9 +1519,8 @@ STAGE PLANS: predicate: value is null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: value (type: int) + key expressions: null (type: void) sort order: + - Map-reduce partition columns: value (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: key (type: int) TableScan @@ -1531,9 +1530,8 @@ STAGE PLANS: predicate: key is null (type: boolean) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: key (type: int) + key expressions: null (type: void) sort order: + - Map-reduce partition columns: key (type: int) Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: value (type: int) Reduce Operator Tree: @@ -1541,13 +1539,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {KEY.reducesinkkey0} {VALUE._col0} - 1 {VALUE._col0} {KEY.reducesinkkey0} + 0 {VALUE._col0} + 1 {VALUE._col0} nullSafes: [true] - outputColumnNames: _col0, _col1, _col4, _col5 + outputColumnNames: _col1, _col4 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int), _col5 (type: int) + expressions: null (type: void), _col1 (type: int), _col4 (type: int), null (type: void) outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator Modified: hive/branches/cbo/ql/src/test/results/clientpositive/metadataonly1.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/metadataonly1.q.out?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== Files hive/branches/cbo/ql/src/test/results/clientpositive/metadataonly1.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/metadataonly1.q.out Tue Jul 29 23:15:13 2014 differ Modified: hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out Tue Jul 29 23:15:13 2014 @@ -219,14 +219,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {key} + 0 1 keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 + 0 5 (type: int) + 1 5 (type: int) Select Operator - expressions: _col0 (type: int) + expressions: 5 (type: int) outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) @@ -242,14 +241,13 @@ STAGE PLANS: condition map: Inner Join 0 to 1 condition expressions: - 0 {key} + 0 1 keys: - 0 key (type: int) - 1 key (type: int) - outputColumnNames: _col0 + 0 5 (type: int) + 1 5 (type: int) Select Operator - expressions: _col0 (type: int) + expressions: 5 (type: int) outputColumnNames: _col0 Reduce Output Operator key expressions: _col0 (type: int) @@ -266,7 +264,7 @@ STAGE PLANS: Filter Operator predicate: (_col1 = 5) (type: boolean) Select Operator - expressions: _col0 (type: int), _col1 (type: int) + expressions: _col0 (type: int), 5 (type: int) outputColumnNames: _col0, _col1 File Output Operator compressed: false Modified: hive/branches/cbo/ql/src/test/results/clientpositive/tez/metadataonly1.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/tez/metadataonly1.q.out?rev=1614514&r1=1614513&r2=1614514&view=diff ============================================================================== Files hive/branches/cbo/ql/src/test/results/clientpositive/tez/metadataonly1.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/tez/metadataonly1.q.out Tue Jul 29 23:15:13 2014 differ