http://git-wip-us.apache.org/repos/asf/hive/blob/a0df0ace/ql/src/test/results/clientpositive/llap/vector_windowing_windowspec.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing_windowspec.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing_windowspec.q.out new file mode 100644 index 0000000..9521468 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_windowing_windowspec.q.out @@ -0,0 +1,2374 @@ +PREHOOK: query: drop table over10k +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table over10k +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table over10k( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal, + bin binary) + row format delimited + fields terminated by '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over10k +POSTHOOK: query: create table over10k( + t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + `dec` decimal, + bin binary) + row format delimited + fields terminated by '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over10k +PREHOOK: query: load data local inpath '../../data/files/over10k' into table over10k +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@over10k +POSTHOOK: query: load data local inpath '../../data/files/over10k' into table over10k +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@over10k +PREHOOK: query: explain vectorization detail +select s, sum(b) over (partition by i order by s,b rows unbounded preceding) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, sum(b) over (partition by i order by s,b rows unbounded preceding) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: i (type: int), s (type: string), b (type: bigint) + sort order: +++ + Map-reduce partition columns: i (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [2, 7, 3] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [2] + valueColumns: [] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [2, 3, 7] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: sum UNBOUNDED end frame is not supported for ROWS window type + vectorized: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey2 (type: bigint), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col2, _col3, _col7 + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: int, _col3: bigint, _col7: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST, _col3 ASC NULLS FIRST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col3 + name: sum + window function: GenericUDAFSumLong + window frame: ROWS PRECEDING(MAX)~CURRENT + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), sum_window_0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select s, sum(b) over (partition by i order by s,b rows unbounded preceding) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, sum(b) over (partition by i order by s,b rows unbounded preceding) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s sum_window_0 +alice ichabod 4294967441 +alice robinson 8589934917 +bob robinson 12884902266 +calvin thompson 17179869602 +david johnson 21474837092 +david laertes 25769804523 +david nixon 30064771904 +david nixon 34359739395 +ethan johnson 38654706752 +ethan ovid 42949674180 +ethan underhill 47244641690 +fred miller 51539609102 +fred miller 55834576592 +gabriella garcia 60129544023 +gabriella underhill 64424511330 +holly white 68719478650 +irene johnson 73014446110 +katie ellison 77309413485 +luke allen 81604380948 +mike quirinius 85899348426 +mike white 90194315855 +nick davidson 94489283385 +oscar allen 98784250693 +oscar garcia 103079218190 +oscar ichabod 107374185594 +oscar ovid 111669153102 +oscar steinbeck 115964120553 +priscilla garcia 120259087901 +priscilla white 124554055390 +priscilla xylophone 128849022850 +priscilla young 133143990191 +rachel brown 137438957640 +rachel ichabod 141733924974 +rachel xylophone 146028892291 +sarah thompson 150323859590 +sarah thompson 154618826928 +tom johnson 158913794359 +tom steinbeck 163208761724 +ulysses polk 167503729208 +victor johnson 171798696592 +wendy polk 176093663918 +xavier davidson 180388631312 +yuri ellison 184683598825 +zach allen 188978566334 +zach hernandez 193273533646 +alice ellison 4294967446 +bob carson 8589934892 +calvin brown 12884902329 +david xylophone 17179869748 +ethan white 21474837241 +fred johnson 25769804704 +fred van buren 30064772167 +gabriella ichabod 34359739606 +holly laertes 38654707054 +holly quirinius 42949674584 +jessica hernandez 47244642120 +katie robinson 51539609539 +katie thompson 55834576895 +luke nixon 60129544345 +mike garcia 64424511764 +mike hernandez 68719479285 +nick carson 73014446621 +nick davidson 77309414083 +oscar carson 81604381543 +oscar robinson 85899348869 +priscilla white 90194316274 +sarah falkner 94489283722 +sarah ichabod 98784251271 +ulysses falkner 103079218819 +victor xylophone 107374186359 +wendy garcia 111669153733 +wendy van buren 115964121147 +xavier underhill 120259088561 +yuri garcia 124554056001 +yuri quirinius 128849023443 +yuri white 133143990852 +zach falkner 137438958357 +zach ichabod 141733925776 +zach nixon 146028893205 +zach ovid 150323860576 +alice ichabod 4294967451 +alice king 8589934958 +alice robinson 12884902278 +calvin allen 17179869612 +gabriella johnson 21474837108 +gabriella nixon 25769804436 +holly falkner 30064771905 +holly hernandez 34359739256 +holly thompson 38654706595 +katie nixon 42949674112 +luke brown 47244641636 +luke davidson 51539608978 +luke white 55834576299 +mike brown 60129543641 +nick quirinius 64424511126 +oscar white 68719478551 +priscilla xylophone 73014446004 +quinn garcia 77309413317 +quinn laertes 81604380656 +rachel young 85899348171 +PREHOOK: query: explain vectorization detail +select s, sum(f) over (partition by d order by s,f rows unbounded preceding) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, sum(f) over (partition by d order by s,f rows unbounded preceding) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: d (type: double), s (type: string), f (type: float) + sort order: +++ + Map-reduce partition columns: d (type: double) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [5, 7, 4] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [5] + valueColumns: [] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [4, 5, 7] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: sum UNBOUNDED end frame is not supported for ROWS window type + vectorized: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey2 (type: float), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col4, _col5, _col7 + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col4: float, _col5: double, _col7: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST, _col4 ASC NULLS FIRST + partition by: _col5 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col4 + name: sum + window function: GenericUDAFSumDouble + window frame: ROWS PRECEDING(MAX)~CURRENT + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), sum_window_0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select s, sum(f) over (partition by d order by s,f rows unbounded preceding) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, sum(f) over (partition by d order by s,f rows unbounded preceding) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s sum_window_0 +calvin miller 8.390000343322754 +holly polk 5.289999961853027 +wendy quirinius 30.789999961853027 +yuri laertes 68.38000011444092 +nick steinbeck 79.23999786376953 +katie brown 60.0 +priscilla quirinius 137.83999633789062 +tom young 186.33999633789062 +gabriella quirinius 14.359999656677246 +katie falkner 65.92999935150146 +xavier robinson 153.84000301361084 +ethan carson 40.90999984741211 +victor johnson 100.0 +jessica king 92.70999908447266 +jessica white 124.16999816894531 +zach white 170.71999740600586 +holly falkner 97.3499984741211 +quinn falkner 196.23999786376953 +victor davidson 255.95999908447266 +holly young 19.110000610351562 +nick robinson 13.329999923706055 +xavier steinbeck 48.53999900817871 +irene king 30.469999313354492 +quinn zipper 90.04000091552734 +priscilla miller 15.359999656677246 +wendy zipper 92.8000020980835 +yuri miller 153.5600004196167 +zach steinbeck 9.069999694824219 +fred nixon 50.08000183105469 +katie brown 13.300000190734863 +nick davidson 87.05000305175781 +gabriella davidson 3.940000057220459 +zach carson 70.88999700546265 +holly hernandez 48.52000045776367 +jessica quirinius 90.18000030517578 +tom xylophone 166.11000061035156 +wendy king 184.76000022888184 +gabriella brown 84.83000183105469 +quinn johnson 134.9800033569336 +yuri zipper 205.75 +david robinson 64.79000091552734 +mike nixon 153.7300033569336 +gabriella white 1.4199999570846558 +rachel davidson 98.12999904155731 +yuri garcia 9.880000114440918 +yuri zipper 104.01999950408936 +alice king 85.72000122070312 +jessica steinbeck 111.41000175476074 +katie hernandez 178.9699993133545 +katie ovid 40.0 +priscilla young 101.72999954223633 +quinn davidson 196.8400001525879 +quinn van buren 279.6400032043457 +victor steinbeck 309.6400032043457 +gabriella brown 80.6500015258789 +jessica ichabod 96.54000091552734 +zach laertes 104.50000095367432 +ethan miller 49.61000061035156 +irene carson 110.68000030517578 +irene falkner 131.42000007629395 +priscilla zipper 201.39000129699707 +tom robinson 290.75000190734863 +katie polk 38.689998626708984 +nick white 96.93999862670898 +sarah davidson 99.59999871253967 +xavier laertes 161.30999779701233 +alice ichabod 32.689998626708984 +nick polk 130.97999954223633 +gabriella robinson 90.0999984741211 +luke brown 90.71999847888947 +wendy allen 116.34999763965607 +calvin ichabod 29.059999465942383 +holly steinbeck 98.4799976348877 +gabriella carson 38.09000015258789 +holly van buren 106.89999771118164 +tom nixon 191.92999649047852 +katie laertes 75.75 +mike brown 163.97000122070312 +oscar nixon 24.020000457763672 +zach garcia 101.61999893188477 +tom polk 76.98999786376953 +mike allen 96.44999694824219 +alice johnson 1.090000033378601 +holly robinson 26.209999084472656 +priscilla thompson 111.12999725341797 +yuri young 168.73999786376953 +rachel carson 80.98999786376953 +gabriella laertes 39.81999969482422 +victor brown 78.97999954223633 +bob carson 24.149999618530273 +holly allen 68.71999931335449 +fred nixon 38.04999923706055 +rachel carson 119.60000228881836 +alice nixon 49.130001068115234 +priscilla brown 123.57999801635742 +victor falkner 42.4900016784668 +david garcia 67.27999877929688 +holly hernandez 116.36999893188477 +tom white 154.0 +rachel ellison 10.600000381469727 +PREHOOK: query: explain vectorization detail +select s, sum(f) over (partition by ts order by f range between current row and unbounded following) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, sum(f) over (partition by ts order by f range between current row and unbounded following) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: ts (type: timestamp), f (type: float) + sort order: ++ + Map-reduce partition columns: ts (type: timestamp) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [8, 4] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [8] + valueColumns: [7] + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + value expressions: s (type: string) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [4, 7, 8] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: sum only UNBOUNDED start frame is supported + vectorized: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: float), VALUE._col6 (type: string), KEY.reducesinkkey0 (type: timestamp) + outputColumnNames: _col4, _col7, _col8 + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col4: float, _col7: string, _col8: timestamp + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col4 ASC NULLS FIRST + partition by: _col8 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col4 + name: sum + window function: GenericUDAFSumDouble + window frame: RANGE CURRENT~FOLLOWING(MAX) + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), sum_window_0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 14400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 14400 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select s, sum(f) over (partition by ts order by f range between current row and unbounded following) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, sum(f) over (partition by ts order by f range between current row and unbounded following) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s sum_window_0 +gabriella xylophone 1276.850001335144 +calvin brown 1273.68000125885 +jessica laertes 1262.7900009155273 +yuri allen 1248.2500009536743 +tom johnson 1233.4700012207031 +bob ovid 1215.6200008392334 +fred nixon 1195.0100002288818 +oscar brown 1166.3199996948242 +calvin laertes 1137.1000003814697 +david falkner 1105.9300003051758 +calvin steinbeck 1067.5800018310547 +katie white 1028.9700012207031 +sarah falkner 989.4900016784668 +mike laertes 948.9500007629395 +victor ellison 907.3500022888184 +luke zipper 861.2700004577637 +rachel garcia 806.9099998474121 +wendy steinbeck 749.9700012207031 +priscilla zipper 685.0100021362305 +rachel thompson 611.4900054931641 +victor van buren 532.9100036621094 +fred zipper 451.5 +gabriella van buren 366.79000091552734 +nick carson 279.36000061035156 +katie king 188.0 +jessica polk 95.04000091552734 +oscar davidson 2368.430002987385 +xavier johnson 2367.600003004074 +rachel ovid 2365.6100029945374 +xavier davidson 2361.880002975464 +nick ellison 2353.0200033187866 +jessica robinson 2342.4000034332275 +bob king 2331.0800037384033 +ulysses xylophone 2318.2500038146973 +wendy thompson 2303.550004005432 +yuri brown 2288.590003967285 +ethan ovid 2271.010004043579 +rachel robinson 2251.9100036621094 +holly falkner 2230.9000034332275 +calvin nixon 2203.950002670288 +luke thompson 2176.7200031280518 +gabriella johnson 2147.6500034332275 +jessica brown 2117.940004348755 +quinn allen 2086.100004196167 +irene brown 2054.1600036621094 +katie zipper 2018.8400039672852 +gabriella steinbeck 1981.520004272461 +priscilla brown 1943.020004272461 +zach young 1900.9400024414062 +alice miller 1856.6400032043457 +priscilla zipper 1811.9800033569336 +rachel young 1765.1400032043457 +holly thompson 1716.2500038146973 +calvin white 1666.6100044250488 +priscilla hernandez 1616.330005645752 +fred polk 1564.240005493164 +sarah van buren 1510.9800071716309 +rachel ovid 1456.890007019043 +luke xylophone 1400.4400062561035 +yuri hernandez 1343.6800079345703 +oscar van buren 1282.2700080871582 +quinn ovid 1220.390007019043 +victor underhill 1157.360008239746 +luke king 1092.8100051879883 +calvin carson 1024.1900024414062 +jessica brown 948.0600051879883 +jessica nixon 869.0100021362305 +katie davidson 788.5800018310547 +fred king 707.1699981689453 +wendy johnson 624.3199996948242 +ulysses johnson 540.3399963378906 +katie xylophone 456.12999725341797 +ethan young 370.57999420166016 +gabriella underhill 282.6499938964844 +luke steinbeck 193.7199935913086 +bob falkner 99.44999694824219 +holly allen 1607.950005441904 +rachel ichabod 1607.590005427599 +bob carson 1607.1100054383278 +wendy miller 1606.3200054168701 +nick king 1605.0500054359436 +rachel ellison 1600.5700054168701 +yuri garcia 1591.5700054168701 +victor hernandez 1568.3000049591064 +wendy underhill 1543.1700057983398 +alice underhill 1517.830005645752 +rachel polk 1491.9200057983398 +holly nixon 1462.910005569458 +ethan nixon 1432.4400062561035 +sarah falkner 1394.490005493164 +tom hernandez 1355.1900062561035 +rachel ichabod 1309.2800064086914 +priscilla thompson 1256.8400077819824 +jessica thompson 1202.7400093078613 +ulysses carson 1146.0400085449219 +wendy falkner 1087.2700080871582 +calvin white 1025.1800079345703 +jessica ovid 956.9800109863281 +jessica johnson 885.3000106811523 +priscilla garcia 805.8400115966797 +PREHOOK: query: explain vectorization detail +select s, avg(f) over (partition by ts order by s,f rows between current row and 5 following) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, avg(f) over (partition by ts order by s,f rows between current row and 5 following) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: ts (type: timestamp), s (type: string), f (type: float) + sort order: +++ + Map-reduce partition columns: ts (type: timestamp) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [8, 7, 4] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [8] + valueColumns: [] + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [4, 7, 8] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: avg only UNBOUNDED start frame is supported + vectorized: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey2 (type: float), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: timestamp) + outputColumnNames: _col4, _col7, _col8 + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col4: float, _col7: string, _col8: timestamp + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST, _col4 ASC NULLS FIRST + partition by: _col8 + raw input shape: + window functions: + window function definition + alias: avg_window_0 + arguments: _col4 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: ROWS CURRENT~FOLLOWING(5) + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), avg_window_0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 14400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 14400 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select s, avg(f) over (partition by ts order by s,f rows between current row and 5 following) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, avg(f) over (partition by ts order by s,f rows between current row and 5 following) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s avg_window_0 +bob ovid 28.053333441416424 +calvin brown 38.73666652043661 +calvin laertes 51.493333180745445 +calvin steinbeck 46.826666514078774 +david falkner 42.81499973932902 +fred nixon 52.26333347956339 +fred zipper 62.97499990463257 +gabriella van buren 55.43666664759318 +gabriella xylophone 49.925000031789146 +jessica laertes 56.32999976476034 +jessica polk 69.13333320617676 +katie king 58.16333293914795 +katie white 54.92333253224691 +luke zipper 57.83333237965902 +mike laertes 61.86999924977621 +nick carson 61.69333299001058 +oscar brown 49.44166628519694 +priscilla zipper 52.25166670481364 +rachel garcia 53.56666787465414 +rachel thompson 54.903334617614746 +sarah falkner 44.27000093460083 +tom johnson 45.01600093841553 +victor ellison 51.80750107765198 +victor van buren 53.71666749318441 +wendy steinbeck 39.869999408721924 +yuri allen 14.779999732971191 +alice miller 51.76333204905192 +bob falkner 47.50333213806152 +bob king 45.58333269755045 +calvin carson 57.253332455952965 +calvin nixon 53.441665967305504 +calvin white 53.85499922434489 +ethan ovid 51.891666094462074 +ethan young 63.52999941507975 +fred king 53.36666615804037 +fred polk 47.83166631062826 +gabriella johnson 44.84166653951009 +gabriella steinbeck 45.1966667175293 +gabriella underhill 51.95500055948893 +holly falkner 50.538333892822266 +holly thompson 47.93333371480306 +irene brown 53.22833442687988 +jessica brown 61.600001653035484 +jessica brown 62.51333491007487 +jessica nixon 60.775001525878906 +jessica robinson 63.08166758219401 +katie davidson 66.04000091552734 +katie xylophone 61.931666692097984 +katie zipper 49.44333283106486 +luke king 43.36166621247927 +luke steinbeck 42.238332599401474 +luke thompson 33.54000013073286 +luke xylophone 37.376666873693466 +nick ellison 35.72333384553591 +oscar davidson 39.27666728695234 +oscar van buren 49.643333752950035 +priscilla brown 39.95166691144308 +priscilla hernandez 42.346666733423866 +priscilla zipper 37.166666746139526 +quinn allen 37.50833328564962 +quinn ovid 41.199999888738 +rachel ovid 44.729999939600624 +rachel ovid 46.558333237965904 +rachel robinson 47.90833361943563 +rachel young 58.40333414077759 +sarah van buren 52.74833424886068 +ulysses johnson 45.21000083287557 +ulysses xylophone 31.506667653719585 +victor underhill 31.98666767279307 +wendy johnson 31.46333380540212 +wendy thompson 24.84999978542328 +xavier davidson 26.82799973487854 +xavier johnson 31.319999754428864 +yuri brown 41.09666633605957 +yuri hernandez 52.85499954223633 +zach young 44.29999923706055 +alice underhill 38.0366666217645 +bob carson 38.7966665327549 +calvin white 51.90833304325739 +ethan ichabod 52.48833360274633 +ethan nixon 46.103333373864494 +holly allen 40.5249999165535 +holly nixon 55.85333355267843 +jessica johnson 64.11166644096375 +jessica ovid 66.54166674613953 +jessica thompson 69.09166725476582 +nick king 68.65833353996277 +oscar carson 82.59166717529297 +priscilla garcia 80.75166702270508 +priscilla hernandez 68.91500091552734 +priscilla polk 53.32166742781798 +priscilla thompson 47.56499997278055 +quinn van buren 43.383333598574005 +rachel davidson 35.253333166241646 +rachel ellison 29.356666321555775 +rachel ichabod 37.651666397849716 +rachel ichabod 41.75999959309896 +rachel polk 49.56333351135254 +sarah falkner 59.53333377838135 +tom hernandez 63.331667264302574 +PREHOOK: query: explain vectorization detail +select s, avg(d) over (partition by t order by s,d desc rows between 5 preceding and 5 following) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, avg(d) over (partition by t order by s,d desc rows between 5 preceding and 5 following) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: t (type: tinyint), s (type: string), d (type: double) + sort order: ++- + Map-reduce partition columns: t (type: tinyint) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [0, 7, 5] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [0] + valueColumns: [] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [0, 5, 7] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: avg only UNBOUNDED start frame is supported + vectorized: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey2 (type: double), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col5, _col7 + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: tinyint, _col5: double, _col7: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST, _col5 DESC NULLS LAST + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: avg_window_0 + arguments: _col5 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: ROWS PRECEDING(5)~FOLLOWING(5) + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), avg_window_0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 11200 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select s, avg(d) over (partition by t order by s,d desc rows between 5 preceding and 5 following) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, avg(d) over (partition by t order by s,d desc rows between 5 preceding and 5 following) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s avg_window_0 +alice allen 33.20166666666666 +alice davidson 30.741428571428568 +alice falkner 27.742499999999996 +alice king 26.706666666666663 +alice king 26.306999999999995 +alice xylophone 24.458181818181814 +bob ellison 25.029090909090908 +bob falkner 24.216363636363635 +bob ichabod 20.173636363636362 +bob johnson 16.431818181818176 +bob polk 16.640909090909087 +bob underhill 15.266363636363632 +bob underhill 18.288181818181812 +bob van buren 18.405454545454543 +calvin ichabod 20.90363636363636 +calvin white 22.448181818181812 +david carson 24.329090909090898 +david falkner 25.01181818181817 +david garcia 22.984545454545444 +david hernandez 22.92272727272726 +ethan steinbeck 24.026363636363627 +ethan underhill 25.189090909090904 +fred ellison 27.159999999999993 +gabriella brown 25.66454545454545 +holly nixon 25.70545454545454 +holly polk 24.11818181818182 +holly steinbeck 24.49090909090909 +holly thompson 23.376363636363635 +holly underhill 19.453636363636363 +irene ellison 20.378181818181826 +irene underhill 23.510000000000012 +irene young 25.371818181818195 +jessica johnson 24.42636363636365 +jessica king 26.380000000000017 +jessica miller 23.99545454545456 +jessica white 26.866363636363655 +katie ichabod 28.520909090909115 +luke garcia 26.110909090909114 +luke ichabod 27.41909090909093 +luke king 28.713636363636375 +luke young 30.59181818181818 +mike allen 27.91545454545455 +mike king 25.526363636363644 +mike polk 24.774545454545464 +mike white 25.18363636363637 +mike xylophone 27.50818181818182 +nick nixon 26.225454545454546 +nick robinson 24.34454545454545 +oscar davidson 26.719090909090916 +oscar garcia 27.196363636363643 +oscar johnson 27.08272727272728 +oscar johnson 25.164545454545472 +oscar miller 28.059090909090916 +priscilla laertes 31.73727272727274 +priscilla quirinius 30.353636363636372 +priscilla zipper 27.961818181818195 +quinn ellison 29.40636363636366 +quinn polk 27.267272727272754 +rachel davidson 25.415454545454562 +rachel thompson 23.608181818181823 +sarah miller 21.49909090909091 +sarah robinson 23.40454545454546 +sarah xylophone 26.957272727272724 +sarah zipper 24.83545454545455 +tom hernandez 21.274545454545454 +tom hernandez 20.315454545454546 +tom polk 21.90181818181819 +tom steinbeck 20.772727272727273 +ulysses carson 21.647272727272718 +ulysses ellison 22.960909090909084 +ulysses quirinius 23.025454545454544 +ulysses robinson 23.762727272727282 +ulysses steinbeck 21.08909090909091 +victor allen 16.628181818181826 +victor hernandez 15.74909090909091 +victor robinson 18.193636363636355 +victor thompson 20.81181818181817 +victor xylophone 20.372727272727243 +wendy quirinius 20.81636363636362 +wendy robinson 19.936363636363634 +wendy xylophone 20.270909090909093 +xavier garcia 19.874000000000002 +xavier ovid 19.976666666666663 +yuri xylophone 21.89625000000001 +zach thompson 25.021428571428583 +zach young 27.77666666666668 +alice carson 18.785 +alice nixon 17.58142857142857 +alice underhill 17.072499999999998 +alice underhill 19.146666666666665 +alice xylophone 20.556 +bob falkner 19.116363636363637 +bob king 21.04 +bob ovid 20.854545454545452 +bob van buren 21.988181818181815 +bob xylophone 24.364545454545453 +calvin xylophone 26.91272727272727 +david falkner 27.31 +david laertes 28.00454545454545 +david miller 28.40090909090909 +PREHOOK: query: explain vectorization detail +select s, sum(i) over(partition by ts order by s) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, sum(i) over(partition by ts order by s) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: ts (type: timestamp), s (type: string) + sort order: ++ + Map-reduce partition columns: ts (type: timestamp) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [8, 7] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [8] + valueColumns: [2] + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + value expressions: i (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [2, 7, 8] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: aa + reduceColumnSortOrder: ++ + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 3 + dataColumns: KEY.reducesinkkey0:timestamp, KEY.reducesinkkey1:string, VALUE._col2:int + partitionColumnCount: 0 + scratchColumnTypeNames: bigint + Reduce Operator Tree: + Select Operator + expressions: VALUE._col2 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: timestamp) + outputColumnNames: _col2, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [2, 1, 0] + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: int, _col7: string, _col8: timestamp + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col7 ASC NULLS FIRST + partition by: _col8 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col2 + name: sum + window function: GenericUDAFSumLong + window frame: RANGE PRECEDING(MAX)~CURRENT + PTF Vectorization: + className: VectorPTFOperator + evaluatorClasses: [VectorPTFEvaluatorLongSum] + functionInputExpressions: [col 2] + functionNames: [sum] + keyInputColumns: [1, 0] + native: true + nonKeyInputColumns: [2] + orderExpressions: [col 1] + outputColumns: [3, 2, 1, 0] + outputTypes: [bigint, int, string, timestamp] + partitionExpressions: [col 0] + streamingColumns: [] + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), sum_window_0 (type: bigint) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [1, 3] + Statistics: Num rows: 7066 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 100 Data size: 14400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 100 Data size: 14400 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select s, sum(i) over(partition by ts order by s) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, sum(i) over(partition by ts order by s) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s sum_window_0 +bob ovid 65748 +calvin brown 131440 +calvin laertes 197097 +calvin steinbeck 262874 +david falkner 328506 +fred nixon 394118 +fred zipper 459719 +gabriella van buren 525334 +gabriella xylophone 591058 +jessica laertes 656771 +jessica polk 722558 +katie king 788310 +katie white 853920 +luke zipper 919543 +mike laertes 985277 +nick carson 1050928 +oscar brown 1116474 +priscilla zipper 1182084 +rachel garcia 1247836 +rachel thompson 1313378 +sarah falkner 1379093 +tom johnson 1444791 +victor ellison 1510421 +victor van buren 1576006 +wendy steinbeck 1641591 +yuri allen 1707256 +alice miller 65581 +bob falkner 131319 +bob king 197015 +calvin carson 262712 +calvin nixon 328407 +calvin white 393960 +ethan ovid 459504 +ethan young 525178 +fred king 590838 +fred polk 656600 +gabriella johnson 722283 +gabriella steinbeck 787886 +gabriella underhill 853497 +holly falkner 919218 +holly thompson 985000 +irene brown 1050757 +jessica brown 1182155 +jessica brown 1182155 +jessica nixon 1247815 +jessica robinson 1313437 +katie davidson 1379172 +katie xylophone 1444746 +katie zipper 1510302 +luke king 1576084 +luke steinbeck 1641724 +luke thompson 1707324 +luke xylophone 1773102 +nick ellison 1838744 +oscar davidson 1904390 +oscar van buren 1969971 +priscilla brown 2035582 +priscilla hernandez 2101353 +priscilla zipper 2166925 +quinn allen 2232487 +quinn ovid 2298060 +rachel ovid 2429366 +rachel ovid 2429366 +rachel robinson 2495140 +rachel young 2560880 +sarah van buren 2626599 +ulysses johnson 2692259 +ulysses xylophone 2757830 +victor underhill 2823401 +wendy johnson 2889058 +wendy thompson 2954831 +xavier davidson 3020367 +xavier johnson 3086050 +yuri brown 3151628 +yuri hernandez 3217338 +zach young 3283046 +alice underhill 65705 +bob carson 131461 +calvin white 197044 +ethan ichabod 262796 +ethan nixon 328501 +holly allen 394248 +holly nixon 459928 +jessica johnson 525664 +jessica ovid 591415 +jessica thompson 657122 +nick king 722691 +oscar carson 788459 +priscilla garcia 854222 +priscilla hernandez 919979 +priscilla polk 985680 +priscilla thompson 1051347 +quinn van buren 1117102 +rachel davidson 1182710 +rachel ellison 1248448 +rachel ichabod 1379923 +rachel ichabod 1379923 +rachel polk 1445518 +sarah falkner 1511234 +tom hernandez 1576947 +PREHOOK: query: explain vectorization detail +select f, sum(f) over (partition by ts order by f range between unbounded preceding and current row) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select f, sum(f) over (partition by ts order by f range between unbounded preceding and current row) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: ts (type: timestamp), f (type: float) + sort order: ++ + Map-reduce partition columns: ts (type: timestamp) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [8, 4] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [8] + valueColumns: [] + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [4, 8] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: aa + reduceColumnSortOrder: ++ + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 2 + dataColumns: KEY.reducesinkkey0:timestamp, KEY.reducesinkkey1:float + partitionColumnCount: 0 + scratchColumnTypeNames: double + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: float), KEY.reducesinkkey0 (type: timestamp) + outputColumnNames: _col4, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [1, 0] + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col4: float, _col8: timestamp + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col4 ASC NULLS FIRST + partition by: _col8 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col4 + name: sum + window function: GenericUDAFSumDouble + window frame: RANGE PRECEDING(MAX)~CURRENT + PTF Vectorization: + className: VectorPTFOperator + evaluatorClasses: [VectorPTFEvaluatorDoubleSum] + functionInputExpressions: [col 1] + functionNames: [sum] + keyInputColumns: [1, 0] + native: true + nonKeyInputColumns: [] + orderExpressions: [col 1] + outputColumns: [2, 1, 0] + outputTypes: [double, float, timestamp] + partitionExpressions: [col 0] + streamingColumns: [] + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col4 (type: float), sum_window_0 (type: double) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [1, 2] + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 100 Data size: 4400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 100 Data size: 4400 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select f, sum(f) over (partition by ts order by f range between unbounded preceding and current row) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select f, sum(f) over (partition by ts order by f range between unbounded preceding and current row) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +f sum_window_0 +3.17 3.1700000762939453 +10.89 14.0600004196167 +14.54 28.600000381469727 +14.78 43.38000011444092 +17.85 61.230000495910645 +20.61 81.8400011062622 +28.69 110.53000164031982 +29.22 139.75000095367432 +31.17 170.92000102996826 +38.35 209.26999950408936 +38.61 247.88000011444092 +39.48 287.35999965667725 +40.54 327.9000005722046 +41.6 369.4999990463257 +46.08 415.58000087738037 +54.36 469.94000148773193 +56.94 526.8800001144409 +64.96 591.8399991989136 +73.52 665.35999584198 +78.58 743.9399976730347 +81.41 825.350001335144 +84.71 910.0600004196167 +87.43 997.4900007247925 +91.36 1088.850001335144 +92.96 1181.8100004196167 +95.04 1276.850001335144 +0.83 0.8299999833106995 +1.99 2.8199999928474426 +3.73 6.550000011920929 +8.86 15.409999668598175 +10.62 26.029999554157257 +11.32 37.349999248981476 +12.83 50.17999917268753 +14.7 64.87999898195267 +14.96 79.83999902009964 +17.58 97.4199989438057 +19.1 116.51999932527542 +21.01 137.52999955415726 +26.95 164.4800003170967 +27.23 191.70999985933304 +29.07 220.77999955415726 +29.71 250.4899986386299 +31.84 282.3299987912178 +31.94 314.2699993252754 +35.32 349.58999902009964 +37.32 386.90999871492386 +38.5 425.40999871492386 +42.08 467.49000054597855 +44.3 511.7899997830391 +44.66 556.4499996304512 +46.84 603.2899997830391 +48.89 652.1799991726875 +49.64 701.819998562336 +50.28 752.0999973416328 +52.09 804.1899974942207 +53.26 857.4499958157539 +54.09 911.5399959683418 +56.45 967.9899967312813 +56.76 1024.7499950528145 +61.41 1086.1599949002266 +61.88 1148.0399959683418 +63.03 1211.0699947476387 +64.55 1275.6199977993965 +68.62 1344.2400005459785 +76.13 1420.3699977993965 +79.05 1499.4200008511543 +80.43 1579.85000115633 +81.41 1661.2600048184395 +82.85 1744.1100032925606 +83.98 1828.0900066494942 +84.21 1912.3000057339668 +85.55 1997.8500087857246 +87.93 2085.7800090909004 +88.93 2174.710009396076 +94.27 2268.9800060391426 +99.45 2368.430002987385 +0.36 0.36000001430511475 +0.48 0.8400000035762787 +0.79 1.6300000250339508 +1.27 2.9000000059604645 +4.48 7.380000025033951 +9.0 16.38000002503395 +23.27 39.65000048279762 +25.13 64.77999964356422 +25.34 90.11999979615211 +25.91 116.02999964356422 +29.01 145.03999987244606 +30.47 175.50999918580055 +37.95 213.45999994874 +39.3 252.75999918580055 +45.91 298.66999903321266 +52.44 351.10999765992165 +54.1 405.20999613404274 +56.7 461.9099968969822 +58.77 520.6799973547459 +62.09 582.7699975073338 +68.2 650.9699944555759 +71.68 722.6499947607517 +79.46 802.1099938452244 +80.02 882.1299904882908 +PREHOOK: query: explain vectorization detail +select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: ts (type: timestamp), f (type: float) + sort order: ++ + Map-reduce partition columns: ts (type: timestamp) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [8, 4] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [8] + valueColumns: [] + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [4, 8] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: sum only UNBOUNDED start frame is supported + vectorized: false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: float), KEY.reducesinkkey0 (type: timestamp) + outputColumnNames: _col4, _col8 + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col4: float, _col8: timestamp + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col4 ASC NULLS FIRST + partition by: _col8 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col4 + name: sum + window function: GenericUDAFSumDouble + window frame: ROWS PRECEDING(2)~PRECEDING(1) + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col4 (type: float), sum_window_0 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 23126 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 4400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 4400 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +f sum_window_0 +3.17 NULL +10.89 3.1700000762939453 +14.54 14.0600004196167 +14.78 25.43000030517578 +17.85 29.31999969482422 +20.61 32.63000011444092 +28.69 38.46000099182129 +29.22 49.30000114440918 +31.17 57.90999984741211 +38.35 60.38999938964844 +38.61 69.51999855041504 +39.48 76.95999908447266 +40.54 78.09000015258789 +41.6 80.02000045776367 +46.08 82.13999938964844 +54.36 87.68000030517578 +56.94 100.44000244140625 +64.96 111.29999923706055 +73.52 121.89999771118164 +78.58 138.47999572753906 +81.41 152.0999984741211 +84.71 159.99000549316406 +87.43 166.12000274658203 +91.36 172.13999938964844 +92.96 178.79000091552734 +95.04 184.31999969482422 +0.83 NULL +1.99 0.8299999833106995 +3.73 2.8199999928474426 +8.86 5.7200000286102295 +10.62 12.589999675750732 +11.32 19.479999542236328 +12.83 21.9399995803833 +14.7 24.149999618530273 +14.96 27.52999973297119 +17.58 29.65999984741211 +19.1 32.53999996185303 +21.01 36.68000030517578 +26.95 40.11000061035156 +27.23 47.96000099182129 +29.07 54.18000030517578 +29.71 56.29999923706055 +31.84 58.779998779296875 +31.94 61.54999923706055 +35.32 63.78000068664551 +37.32 67.26000022888184 +38.5 72.63999938964844 +42.08 75.81999969482422 +44.3 80.58000183105469 +44.66 86.38000106811523 +46.84 88.95999908447266 +48.89 91.5 +49.64 95.72999954223633 +50.28 98.52999877929688 +52.09 99.91999816894531 +53.26 102.36999893188477 +54.09 105.3499984741211 +56.45 107.3499984741211 +56.76 110.54000091552734 +61.41 113.20999908447266 +61.88 118.16999816894531 +63.03 123.29000091552734 +64.55 124.90999984741211 +68.62 127.58000183105469 +76.13 133.17000579833984 +79.05 144.75 +80.43 155.18000030517578 +81.41 159.4800033569336 +82.85 161.84000396728516 +83.98 164.26000213623047 +84.21 166.8300018310547 +85.55 168.19000244140625 +87.93 169.76000213623047 +88.93 173.4800033569336 +94.27 176.86000061035156 +99.45 183.1999969482422 +0.36 NULL +0.48 0.36000001430511475 +0.79 0.8400000035762787 +1.27 1.270000010728836 +4.48 2.060000002384186 +9.0 5.75 +23.27 13.480000019073486 +25.13 32.27000045776367 +25.34 48.39999961853027 +25.91 50.46999931335449 +29.01 51.25 +30.47 54.920000076293945 +37.95 59.47999954223633 +39.3 68.42000007629395 +45.91 77.25 +52.44 85.20999908447266 +54.1 98.3499984741211 +56.7 106.53999710083008 +58.77 110.79999923706055 +62.09 115.47000122070312 +68.2 120.86000061035156 +71.68 130.28999710083008 +79.46 139.87999725341797 +80.02 151.13999938964844 +PREHOOK: query: explain vectorization detail +select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: s (type: string), i (type: int) + sort order: ++ + Map-reduce partition columns: s (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [7, 2] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [7] + valueColumns: [5] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + value expressions: d (type: double) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [2, 5, 7] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: aa + reduceColumnSortOrder: ++ + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 3 + dataColumns: KEY.reducesinkkey0:string, KEY.reducesinkkey1:int, VALUE._col4:double + partitionColumnCount: 0 + scratchColumnTypeNames: double, double, double + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey1 (type: int), VALUE._col4 (type: double), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col2, _col5, _col7 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [1, 2, 0] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: int, _col5: double, _col7: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 ASC NULLS FIRST + partition by: _col7 + raw input shape: + window functions: + window function definition + alias: avg_window_0 + arguments: _col5 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: RANGE PRECEDING(MAX)~CURRENT + PTF Vectorization: + className: VectorPTFOperator + evaluatorClasses: [VectorPTFEvaluatorDoubleAvg] + functionInputExpressions: [col 2] + functionNames: [avg] + keyInputColumns: [1, 0] + native: true + nonKeyInputColumns: [2] + orderExpressions: [col 1] + outputColumns: [3, 1, 2, 0] + outputTypes: [double, int, double, string] + partitionExpressions: [col 0] + streamingColumns: [] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col7 (type: string), _col2 (type: int), round((avg_window_0 / 10.0), 2) (type: double) + outputColumnNames: _col0, _col1, _col2 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 1, 5] + selectExpressions: RoundWithNumDigitsDoubleToDouble(col 4, decimalPlaces 2)(children: DoubleColDivideDoubleScalar(col 3, val 10.0) -> 4:double) -> 5:double + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 7 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 7 Data size: 784 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 7 Data size: 784 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 7 + Processor Tree: + ListSink + +PREHOOK: query: select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7 +PREHOOK: type: QUERY +PREHOOK: Input: default@over10k +#### A masked pattern was here #### +POSTHOOK: query: select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over10k +#### A masked pattern was here #### +s i _c2 +alice allen 65545 2.22 +alice allen 65557 2.58 +alice allen 65600 3.38 +alice allen 65609 2.99 +alice allen 65662 2.7 +alice allen 65670 2.88 +alice allen 65720 2.76 +PREHOOK: query: explain vectorization detail +select s, i, round((avg(d) over w1 + 10.0) - (avg(d) over w1 - 10.0),2) from over10k window w1 as (partition by s order by i) limit 7 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select s, i, round((avg(d) over w1 + 10.0) - (avg(d) over w1 - 10.0),2) from over10k window w1 as (partition by s order by i) limit 7 +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: over10k + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + Reduce Output Operator + key expressions: s (type: string), i (type: int) + sort order: ++ + Map-reduce partition columns: s (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumns: [7, 2] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumns: [7] + valueColumns: [5] + Statistics: Num rows: 9085 Data size: 1017544 Basic stats: COMPLETE Column stats: NONE + value expressions: d (type: double) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 11 + includeColumns: [2, 5, 7] + dataColumns: t:tinyint, si:smallint, i:int, b:bigint, f:float, d:double, bo:boolean, s:string, ts:timestamp, dec:decimal(10,0), bin:binary + partitionColumnCount: 0 + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: aa + reduceColumnSortOrder: +
<TRUNCATED>
